import itertools
import logging
import time
import warnings
from collections import defaultdict, namedtuple
from contextlib import contextmanager
from copy import copy
from typing import Tuple, List, Dict, Any, Union, TYPE_CHECKING
from warnings import warn

import networkx as nx
from networkx import NetworkXNoPath, NodeNotFound
from py2neo.cypher import Cursor

from weaveio.hierarchy import Hierarchy

from .exceptions import AmbiguousPathError, CardinalityError, DisjointPathError, AttributeNameError
from .parser import QueryGraph
from .results import Table, AstropyTable
from ..path_finding import collapse_classes_to_superclasses

if TYPE_CHECKING:
    from .objects import ObjectQuery, Query, AttributeQuery
    from ..data import Data

@contextmanager
def logtime(name):
    start = time.perf_counter()
    yield
    logging.info(f"{name} took {time.perf_counter() - start: .3f} seconds")

def get_groupids(graph, params):
    groupids = []
    names = []
    for name, query in graph.groupbys.items():
        if name in params:
            groupids.append(query._iterate(limit=None, skip=0, distinct=True, no_groups=True))
            names.append(name)
    return groupids, names

def _execute_groups(groupids, names, params):
    for values in itertools.product(*groupids):
        for name, value in zip(names, values):
            params[name] = value
        yield values, params

SplitQuery = namedtuple('SplitQuery', ['index', 'query'])
SplitResult = namedtuple('SplitResult', ['index', 'result'])

class QueryFunctionBase:
    def _precompile(self):
        raise NotImplementedError

    def __call__(self, *args, **kwargs):
        raise NotImplementedError

    def __iter__(self):
        raise NotImplementedError

    def __repr__(self):
        return f'{self.__class__.__name__}()'

class BaseQuery(QueryFunctionBase):
    one_row = False
    one_column = False

    def __new__(cls, *args, **kwargs):
        from fastapi.encoders import ENCODERS_BY_TYPE
        ENCODERS_BY_TYPE[cls] = lambda x: [x._data, x._node]
        return super().__new__(cls)

    def _debug_output(self, skip=0, limit=None, distinct=False, no_cache=False, graph_export_fname=None,
                      exclusive=True):
        n = self._precompile()
        cached_params = n._get_cached_parameters()
        placeholder_params = n._G.dependency_parameters(self._node)
        params = {**placeholder_params, **cached_params}
        c = n._to_cypher(skip, limit, distinct, no_cache)
        params = n._prepare_parameters(c, params)
        if graph_export_fname is not None:
            if exclusive:
                n._G.export(graph_export_fname, n._node)
            else:
                n._G.export(graph_export_fname)
        return '\n'.join(c), self._G.G.nodes[n._node]['ordering'], params

    def raise_error_with_suggestions(self, obj, exception: Exception):
        self._data.autosuggest(obj, self._obj, exception)
        raise exception

    def __repr__(self):
        try:
            return f'<{self.__class__.__name__}({self._previous._obj}-{self._obj})>'
        except AttributeError:
            return f'<{self.__class__.__name__}({self._obj})>'

    def _precompile(self) -> 'BaseQuery':
        return self

    def _prepare_parameters(self, lines, parameters):
        params = {}
        for k, v in parameters.items():
            if any(k in l for l in lines):
                if isinstance(v, AstropyTable):
                    cols = [c for c in v.colnames if any(c in l for l in lines)]
                    assert len(cols), "No columns are used, but table is used. This should not happen"
                    params[k] = v[cols]
                else:
                    params[k] = v
        return params

    def _to_cypher(self, skip, limit, distinct, no_cache=False) -> List[str]:
        """
        returns the cypher lines, cypher parameters, names of columns, expect_one_row, expect_one_column
        """
        lines = self._G.cypher_lines(self._node, no_cache)
        if distinct:
            lines[-1] = 'RETURN DISTINCT'.join(lines[-1].rsplit('RETURN', 1))
        if skip > 0:
            lines.append(f"\nSKIP {skip}")
        if limit is not None:
            lines.append(f"\nLIMIT {limit}")
        return lines

    def _compile(self, skip, limit, distinct) -> Tuple['BaseQuery', List[str]]:
        with logtime('compiling'):
            r = self._precompile()
            return r, r._to_cypher(skip, limit, distinct)

    def _execute_single_query(self, query, cypher, params):
        return query._data.graph.execute(cypher, **{k.replace('$', ''): v for k, v in params.items()})

    def _get_cached_parameters(self):
        params = {}
        for node in self._G.restricted(self._node).nodes:
            params.update(self._G.G.nodes[node].get('cached_params', {}))
        return params

    def _execute(self, skip=0, limit=None, distinct=False, no_groups=False):
        """
        returns a new query (using _precompile) and an iterator over cursors.
        For queries that have not been `split`, this is a single cursor.
        For queries that have been `split`, there will be multiple cursors.
        :param skip: number of rows to skip
        :param limit: number of rows to limit to
        :param distinct: whether to return distinct rows
        :param no_groups: whether to return a single cursor or a generator of cursors
        """
        with logtime('executing'):
            cached_params = self._get_cached_parameters()
            placeholder_params = self._G.dependency_parameters(self._node)
            params = {**placeholder_params, **cached_params}
            if not no_groups and any(v == '<placeholder>' for v in params.values()):
                groupids, names = get_groupids(self._G, params)
                if groupids:
                    return _execute_groups(groupids, names, params), self
            new, lines = self._compile(skip, limit, distinct)
            cypher = '\n'.join(lines)
            params = self._prepare_parameters(lines, params)
            return self._execute_single_query(new, cypher, params), new

    def _iterate_groups(self, generator, query):
        for split_indexes, params in generator:
            copied = copy(query)  # copy the query, so that the parameters are not shared between the cursors
            query._G.G.nodes[copied._node]['cached_params'] = params  # store the groups for when the query is executed
            yield SplitQuery(split_indexes, copied)

    def _iterate(self, skip=0, limit=None, distinct=False, no_groups=False):
        cursor_or_gen, new = self._execute(skip, limit, distinct, no_groups)  # produces either a cursor or a generator of cursors
        if not isinstance(cursor_or_gen, Cursor):
            yield from self._iterate_groups(cursor_or_gen, new)
        else:
            rows = new._data.rowparser.iterate_cursor(cursor_or_gen, new._names, new._is_products, True)
            i = None
            with logtime('total streaming (by iteration)'):
                for i, row in enumerate(rows):
                    yield new._post_process_row(row)
            if limit == i:
                warnings.simplefilter("always")
                warn(f"This query has been capped to {limit} rows but the result is larger than that. "
                     f"Consider using the `limit` parameter to better limit the result or set `limit` to None to get "
                     f"the full result (although this is not recommended).")

    def __iter__(self):
        yield from self._iterate()

    def _to_table(self, skip=0, limit=None, distinct=False) -> Tuple[Table, 'BaseQuery']:
        with logtime('total streaming'):
            cursor_or_gen, new = self._execute(skip, limit, distinct)  # produces either a cursor or a generator of cursors
            if not isinstance(cursor_or_gen, Cursor):
                return [SplitResult(split_indexes, query._to_table(skip, limit, distinct)) for split_indexes, query in self._iterate_groups(cursor_or_gen, new)], new
            return new._data.rowparser.parse_to_table(cursor_or_gen, new._names, new._is_products), new

    def _post_process_table(self, result, squeeze):
        if isinstance(result, SplitResult):
            return result.index, self._post_process_table(result.result, squeeze)
        if not squeeze:
            return result
        if self.one_column:
            result = result[result.colnames[0]].data
        if self.one_row:
            result = result[0]
        return result

    def _post_process_row(self, row):
        if self.one_column:
            return row[row.colnames[0]]
        return row

    def __call__(self, skip=0, limit=1000, distinct=False, squeeze=True, **kwargs):
        tbl, new = self._to_table(skip, limit, distinct)
        if isinstance(tbl, list):
            return tbl
        tbl = new._post_process_table(tbl, squeeze)
        try:
            if not len(tbl):
                return tbl
        except TypeError:
            return tbl  # not a table, just a value, so return it
        if limit == 1 and squeeze:
            return tbl[0]
        try:
            if limit == len(tbl):
                warnings.simplefilter("always")
                warn(f"This query has been capped to {limit} rows but the result is larger than that. "
                     f"Consider using the `limit` parameter to better limit the result or set `limit` to None to get "
                     f"the full result (although this is not recommended).")
        except TypeError:
            pass
        return tbl

    def __init__(self, data: 'Data', G: QueryGraph = None, node=None, previous: Union['Query', 'AttributeQuery', 'ObjectQuery'] = None,
                 obj: str = None, start: 'Query' = None, index_node = None,
                 single=False, names=None, is_products=None, attrs=None, dtype=None, *args, **kwargs) -> None:
        from .objects import ObjectQuery
        self._single = single
        self._data = data
        if G is None:
            self._G = QueryGraph()
        else:
            self._G = G
        if node is None:
            self._node = self._G.start
        else:
            self._node = node
        self._previous = previous
        self.__cypher = None
        self._index_node = index_node
        self._obj = obj
        if previous is not None:
            if self._index_node is None:
                if isinstance(previous, ObjectQuery):
                    self._index_node = previous._node
                elif self._index_node != 'start':
                    self._index_node = previous._index_node
            if obj is None:
                self._obj = previous._obj
        if start is None:
            self._start = self
        else:
            self._start = start
        if self._obj is not None:
            if not self._obj.startswith('_'):  # system name for other purposes
                self._obj = self._normalise_object(self._obj)[0]
        self._names = [] if names is None else names
        self._is_products = [False]*len(self._names) if is_products is None else is_products
        self.attrs = attrs
        self.dtype = dtype

    def _get_object_of(self, maybe_attribute: str) -> Tuple[str, bool, bool]:
        """
        Given a str that might refer to an attribute, return the object that contains that attribute
        and also whether the attribute and the object are singular.
        plurality table:
            inferred obj | attr
                s        |  s      (i.e. run.obid -> run.ob.obid)
                p        |  p      (i.e. ...-> run.l1specs.snrs) - cannot be inferred just from a name
                s        |  p      (i.e. run.obids -> run.ob.obids)
                p        |  s      (i.e. run.snr -> run.l1spec.snr) - is not allowed
        attribute storage scenarios:
            * attribute is contained in this object - return self
            * attribute is contained in exactly one object - return object
            * attribute is contained in more than one object and ...
                * any of those objects don't have the same inherited class - raise AmbiguousError
                * only subclass objects and all subclass objects from a superclass contain the attribute - return the superclass
                * only subclass objects and not all subclass objects from a superclass contain the attribute - return the superclass
        If a singular path exists, use that
        :return: obj, obj_is_singular, attr_is_singular
        """
        single_name = self._data.singular_name(maybe_attribute)
        if self._obj is not None:
            if self._data.class_hierarchies[self._obj] in self._data.factor_hierarchies[single_name]:
                return self._obj, True, self._data.is_singular_name(maybe_attribute)
        want_single = single_name == maybe_attribute
        # try to get singular paths
        hs = {h for h in self._data.factor_hierarchies[single_name] if self._has_path_to_object(h, True)}
        if len(hs) == 0: # if no singular paths then get all paths
            hs = {h for h in self._data.factor_hierarchies[single_name] if self._has_path_to_object(h, want_single)}
        hs = set(collapse_classes_to_superclasses(self._data.hierarchy_graph, hs))
        if len(hs) == 1:
            obj = hs.pop().__name__
        elif not hs:
            raise AmbiguousPathError(f"There is no path to any object that contains `{single_name}`. {self._data.factor_hierarchies[single_name]} are not reachable")
        else:
            raise AmbiguousPathError(f"`{single_name}` is contained within more than one unrelated object {[h.__name__ for h in hs]}. Be specific.")
        if self._obj is None:
            return obj, True, False
        if not self._data.is_factor_name(maybe_attribute):
            raise ValueError(f"{maybe_attribute} is not a valid attribute name")
        _, obj_is_singulars = self._get_path_to_object(obj, False)
        obj_is_singular = len(obj_is_singulars) == 1 and obj_is_singulars[0]
        attr_is_singular = self._data.is_singular_name(maybe_attribute)
        if obj_is_singular and attr_is_singular:
            pass  # run.obid -> run.ob.obid
        elif not obj_is_singular and not attr_is_singular:
            pass
        elif obj_is_singular and not attr_is_singular:
            pass  # run.obids -> run.ob.obids
        else:
            # ob.runid -> Error
            plural_name = self._data.plural_name(obj)
            original = self._data.singular_name(self._obj)
            raise CardinalityError(f"Requested one `{single_name}` from `{original}` "
                                   f"when `{original}` has several `{plural_name}`")
        return obj, obj_is_singular, attr_is_singular

    def _normalise_object(self, obj: str):
        """
        returns the __name__ of the respective class of the obj and whether it is plural/singular.
        If not a class, raise KeyError
        """
        if obj in self._data.class_hierarchies:
            return obj, True
        singular = self._data.singular_name(obj)
        if singular not in self._data.singular_hierarchies:
            raise KeyError(f"{obj} is not a valid object name")
        try:
            h = self._data.plural_hierarchies[obj.lower()]
            singular = False
        except KeyError:
            h = self._data.singular_hierarchies[obj.lower()]
            singular = True
        return h.__name__, singular

    @property
    def _cypher(self):
        if self.__cypher is None:
            self.__cypher = self._G.cypher_lines(self._node)
        return self.__cypher

    @property
    def _result_graph(self):
        return self._G.restricted(self._node)

    def _plot_graph(self, fname):
        return self._G.export(fname, self._node)

    @classmethod
    def _spawn(cls, parent: 'BaseQuery', node, obj=None, index_node=None, single=False, *args, **kwargs):
        return cls(parent._data, parent._G, node, parent, obj, parent._start, index_node, single, *args, **kwargs)

    def _get_path_to_object(self, obj, want_single) -> Tuple[List[str], List[bool]]:
        return self._data.paths_to_hierarchy(self._obj, obj, want_single)

    def _has_path_to_object(self, obj, want_single) -> bool:
        if self._obj is None:
            return True
        try:
            return len(self._get_path_to_object(obj, want_single)[0]) > 0
        except NetworkXNoPath:
            return False

    def _slice(self, slc):
        """
        obj[slice]
        filter, shrinks, destructive
        filter by using HEAD/TAIL/SKIP/LIMIT
        e.g. obs.runs[:10] will return the first 10 runs for each ob (in whatever order they were in)
        you must use query(skip, limit) to request a specific number of rows in total since this is unrelated to the actual query
        """
        raise NotImplementedError

    def _filter_by_mask(self, mask, single=None, split_node=False):
        """
        obj[boolean_filter]
        filter, shrinks, destructive
        filters based on a list of True/False values constructed beforehand, parentage of the booleans must be derived from the obj
        e.g. `ob.l1stackedspectra[ob.l1stackedspectra.camera == 'red']` gives only the red stacks
             `ob.l1stackedspectra[ob.l1singlespectra == 'red']` is invalid since the lists will not be the same size or have the same parentage
        """
        if mask.dtype != 'boolean':
            raise TypeError(f"The predicate in a filter `query[predicate]`, must be of type boolean. {mask} is of type {mask.dtype}. ")
        single = single if single is not None else self._single
        try:
            n = self._G.add_filter(self._node, mask._node, direct=False, force_single=single, split_node=split_node)
        except SyntaxError:
            raise DisjointPathError(f"{self} cannot be filtered by {mask} since `{self._obj}` != `{mask._obj}`."
                                    f"This may be because a nonsensical `wrt` has specified in an aggregation.")
        return self.__class__._spawn(self, n, single=single)

    def _aggregate(self, wrt, string_op, op_name, predicate=False, expected_dtype=None, returns_dtype=None,
                   remove_infs=None, distinct=False, nulls=False):
        if wrt is None:
            wrt = self._start
        try:
            if predicate:
                n = self._G.add_predicate_aggregation(self._node, wrt._node, string_op)
            else:
                n = self._G.add_aggregation(self._node, wrt._node, string_op, remove_infs, expected_dtype, self.dtype, distinct, nulls)
        except SyntaxError:
            raise DisjointPathError(f"Cannot aggregate {self} into {wrt} since they don't share a parent query")
        from .objects import AttributeQuery
        return AttributeQuery._spawn(self, n, wrt._obj, wrt._node, dtype=returns_dtype, single=True, factor_name=op_name)


# -*- coding: utf-8 -*-
"""
Created on Sun Jan 27 19:33:37 2019

@author: myran
"""
import matplotlib.pyplot as plt
from seabird import fCNV
import numpy as np
import sys
import os
import cartopy.crs as ccrs
from shapely.geometry.polygon import LinearRing
import cartopy.feature as cfeature
sys.path.insert(0, '/Users/myran/Documents/Oceanography/Ocean 340')
from Myranda_Dang_a6_340b_cross_sections import twoDemData, calcDistance

def allProfDict(profList, varList):
    '''
    This function creates a "master dictionary" and returns a dictionary containing all the
    stations and their values only during the downcast. Also gets list of max depths of each
    stations
    '''
    ProfDict = {}
    for i in range(len(profList)):
        prof = getDownCastData(profList[i], varList)    #one station
        ProfDict['Station ' + str(i + 1)] = prof
    return(ProfDict)

def getDownCastData(dataset, varList):
    '''
    This function takes in the filepath name of the station's cnv file, creates a dictionary of
    the station variables and values only in the CTD downcasting, and returns that dictionary
    '''
    station = fCNV(dataset)
    newProf = {}
    lat, lon = latLong(station)
    newProf['LATITUDE'] = lat
    newProf['LONGITUDE'] = lon
    for var in station.keys():
        newProf[var] = []
    depth = station['DEPTH']
    maxNum = depth.max()
    end = list(depth).index(maxNum)
    minNum = depth.min()
    start = list(depth).index(minNum)
    for var in varList:    #Look at each variable
        for i in range(start, end):
            newProf[var].append(station[var][i])
    return(newProf)

def latLong(stationDict):
    '''
    This function takes in a dicionary containing a station and its variables and values, and
    returns the latitude and longitude (in that order) of where the profile was taken with rounded
    decimals of four digits after the decimal place
    '''
    head = stationDict.attributes
    lat = float(head['LATITUDE'])
    lat = round(lat, 4)
    long = float(head['LONGITUDE'])
    long = round(long, 4)
    return lat, long

def plotCrossSection(depths2d, dist2d, var2d, figNum, xlabel, ylabel, title, barLabel, maxDepth):
    '''
    This function takes in a 2-D array of depth, distance,
    variable data, title, and a figure number, and create a filled contour plot of the
    variable data array with respect to the depths and distance arrays given
    '''
    plt.figure(num = figNum)
    plt.gca().invert_yaxis()
    plt.ylabel(ylabel)
    plt.xlabel(xlabel)
    plt.title(title, fontsize = 15)
    plt.contourf(dist2d, depths2d, var2d)
    #plt.fill_between(dist2shore,bathy,1000,color='k') #Mapping bathymetry
    cbar = plt.colorbar()
    cbar.set_label(barLabel)
    plt.show()

def plotVar(staDict, varName, figNum, yVar = 'DEPTH'):
    """This function takes in a profile of a variable in the form of a list, another variable
    in a list (most likely depth), the figure number, and the label for the x-axis. It then plots
    the profile on a specific figure, adding labels, a title, inverts the y-axis for proper
    depth profiles, and caption.
    """
    plt.figure(figNum, figsize = (15,20))
    for station in staDict.keys():
        var1 = staDict[station][varName]
        var2 = staDict[station][yVar]
        plt.plot(var1, var2, label = station)
    plt.xlabel(varName)
    plt.ylabel(yVar)
    if 'DEPTH' in yVar:
        plt.gca().invert_yaxis()
        plt.title(varName + ' Profile Plot')
    else:
        plt.title(varName + ' vs '+yVar)
    plt.legend()
    plt.show()
    #Optional saving profiles
    #file = xlabel + '.png'
    #plt.savefig(file)

def mapStations(lats, longs, figSize=(20,20), res='10m', cmap='rainbow'):
    '''
    This function takes in lists of longitude and latitude and plots
    them on a map, as well as taking in optional parameters such as
    figure size, default of (20, 20), map resolution (as string),
    default of '10m', and colormap (string), default of 'rainbow'
    '''
    fig = plt.figure(1,figsize=figSize)
    fig.suptitle('Station Locations', fontsize=25)
    map1 = fig.add_subplot(1,1,1,projection=ccrs.PlateCarree())
    map2 = fig.add_subplot(1,2,1,projection=ccrs.PlateCarree())
    map1.coastlines(resolution=res)
    map1.set_extent([-124, -122, 47, 48.5]) #Puget Sound area
    pslons = [-122.10, -122.10, -122.40, -122.40]
    pslats = [47.89, 48.07, 48.07, 47.89]
    ring = LinearRing(list(zip(pslons, pslats)))
    map2.coastlines(resolution=res)
    map2.set_extent([-122.40, -122.10, 47.89, 48.07], crs=ccrs.PlateCarree())    #Posession Sound
    land = cfeature.NaturalEarthFeature('physical', 'land', res, facecolor='darkseagreen')
    map1.add_feature(land)
    map2.add_feature(land)
    map1.add_geometries([ring],ccrs.PlateCarree(),facecolor='none',edgecolor='darkred',linewidth=4)
    color = eval('plt.cm.'+cmap+'(np.linspace(0,1, len(lats)))')
    for i in range(len(lats)):
        c = color[i]
        lat = lats[i]
        long = longs[i]
        profNum = i+1
        l = "Station" + str(profNum)
        map2.plot(long,lat,'o',color=c, label = l, markersize=14, transform=ccrs.PlateCarree())
    map2.legend()
    map1.set_title('Puget Sound', fontsize=14)
    map2.set_title('Possession Sound', fontsize=14)
    plt.show

def mapDepthCrossSection(lats,longs,varName,stationName,varData,barLabel):
    '''
    This function takes in a list of latitudes and longitudes, the variable name and data that
    are being plotted and shown as changes in each depth layer for each station, label containing
    the units of the inputted variable, and the figure number.

    '''
    plt.figure(figsize=(20,20))
    ax = plt.axes(projection=ccrs.PlateCarree())
    ax.set_title(varName + ' m', fontsize=25)
    land = cfeature.NaturalEarthFeature('physical', 'land', '10m',
                                        facecolor='darkseagreen')
    ax.set_extent([-122.40, -122.10, 47.89, 48.07], crs=ccrs.PlateCarree())
    ax.coastlines(resolution='10m')
    ax.add_feature(land)
    plt.scatter(longs,lats,c=varData,s=200,transform=ccrs.PlateCarree())
    #Need to check to see if SAL/AATT = 0, then pt would be black 'k'
    for i in range(len(lats)):
        lat = lats[i]
        long = longs[i]
        if varData[i] == 0: #Check if station has data or not
            plt.plot(long,lat,'o',color='darkgray', markersize=14, transform=ccrs.PlateCarree())
        plt.text(longs[i], lats[i]+0.005, 'Sta. '+ str(stationName[i]),
                fontsize=14,transform=ccrs.PlateCarree())
    cbar = plt.colorbar()
    cbar.set_label(barLabel)
    plt.show()

def depthRange(depth, station, varName):
    '''
    This function takes in depth(as a string) that samples were collected at, a list of the station,
    their variables and values, and also the variable name of which the user
    wishs to calculate; calculates an average variable over the passed in depth and returns average
    as a float
    '''
    avgList = []
    count = 0
    for i in range(len(station['DEPTH'])):
        num = station['DEPTH'][i]
        if depth in str(num):
            if num < (float(depth) + 2) and num >= (float(depth) - 2):
                value = list(station['DEPTH']).index(num)
                avgList.append(station[varName][value])
                count += 1
    result = 0
    for j in range(len(avgList)):
        result += avgList[j]
    if result != 0 and count != 0:
        result = result/count
    else:
        result = 0
    return result

def createLayersDict(varName, stationDict, layerDepths):
    '''
    This function takes in a variable name, a dictionary containing all the stations and their
    values, and a list of depths that are considered shallow, mid, and deep layers. The function
    creates a dictionary of variable values at those depth layers and returns created dictionary.
    '''
    layerVar = {}
    for layer in layerDepths:
        layerVar[layer] = []
        for station in stationDict.keys():
            if station not in 'Everett Docks':
                layerData = depthRange(str(layer),stationDict[station],varName)
                layerVar[layer].append(layerData)
    return layerVar

def main():
    #Creating list of profile filenames for Stations 1-9
    fp = '/Users/myran/Documents/Oceanography/Ocean Senior Thesis/DATA/rc0015_CTD_MD/PROCESSED/Stations 1-9/'
    fileList = os.listdir(fp)
    profList = []
    for item in fileList:
        if '.cnv' in item:    #Looks to see if file is .cnv file
            openFile = fp + item
            profList.append(openFile)

    #List of variables
    varList = ['PSAL', 'CStarAt0', 'CStarTr0', 'DEPTH', 'flECO-AFL']

    #Attenuation and Salinity labels
    attLabel = 'Attenuation (1/m)'
    salLabel = 'Salinity (PSU)'

    #Create dictionary of all profile
    d = allProfDict(profList, varList)

    #Create list of max depths of all stations
    maxDepth = []
    for station in d.keys():
        maxNum = max(d[station]['DEPTH'])
        maxDepth.append(maxNum)

    #Append Everett Dock file to master dictionary
    dockName = '/Users/myran/Documents/Oceanography/Ocean Senior Thesis/DATA/rc0015_CTD_MD/PROCESSED/RC0015_MD_dockTest.cnv'
    dockFile = getDownCastData(dockName,varList)
    d['Everett Docks'] = dockFile

    # Create numpy array of new depths
    newDepth = np.arange(10, 60, 2)

    #Getting stations that are in a line (Stations 2 to 9) for crossections
    statLine = {}
    stationList = list(d.keys())
    for station in stationList[1:9]:
        statLine[station] = d[station]
    #Create array of latitude and longitude
    lats = []
    longs = []
    for station in d.keys():
        lat = d[station]['LATITUDE']
        lon = d[station]['LONGITUDE']
        lats.append(lat)
        longs.append(lon)

    #Get lats/longs of stations to for transect
    crossLats =[]
    crossLongs = []
    crossDepth = []
    for i in range(4):
        latCross1 = lats[1 + i]
        depth1 = lats[1 + i]
        crossLats.append(latCross1)
        crossDepth.append(depth1)
        longCross1 = longs[1 + i]
        depth2 = longs[1 + i]
        crossLongs.append(longCross1)
        crossDepth.append(depth2)
        latCross2 = lats[8 - i]
        crossLats.append(latCross2)
        longCross2 = longs[8 - i]
        crossLongs.append(longCross2)

    # Get list of distances between datapoints
    dists = calcDistance(crossLats, crossLongs)

    # Create 2-D array of data salinity and attenuation
    attData = twoDemData('CStarAt0', statLine, newDepth)
    salData = twoDemData('PSAL', statLine, newDepth)

    # Create grid of depths and distances
    depths_2d, dists_2d = np.meshgrid(newDepth, dists)

    #Plot lat/long pair of all stations
    mapStations(lats, longs)

    #Plotting cross section
    plotCrossSection(depths_2d,dists_2d,attData,2,'Distance (km)','Depth(m)',
                     'Attenuation CrossSection',attLabel,crossDepth)
    plotCrossSection(depths_2d,dists_2d,salData,3,'Distance (km)','Depth(m)',
                     'Salinity CrossSection', salLabel,crossDepth)

    #Creating "shallow, mid, deep" data of Attenuation and Salinity
    layerDepth = [10,20,30,40,50,60,70,80,90]
    attLayers = createLayersDict('CStarAt0', d, layerDepth)
    salLayers = createLayersDict('PSAL', d, layerDepth)

    #Getting lats/longs for depth layers
    depthCoorLats = lats
    depthCoorLongs = longs
    depthCoorLats.pop()
    depthCoorLongs.pop()

    #Plotting depth interval layers
    count = 4
    for layer in attLayers.keys():
        mapDepthCrossSection(depthCoorLats,depthCoorLongs,'Attenuation: '+str(layer),
                             [1,2,3,4,5,6,7,8,9],np.array(attLayers[layer]),attLabel)
        count += 1
        mapDepthCrossSection(depthCoorLats,depthCoorLongs,'Salinity: '+str(layer),
                             [1,2,3,4,5,6,7,8,9],np.array(salLayers[layer]),salLabel)
        count += 1
    '''
    #Creating depth profiles
    for var in varList: #plotting profiles for each variable
        plotVar(d, var, count)
        count += 1
    plotVar(d, 'PSAL', count, yVar = 'CStarAt0')
    '''

if __name__ == "__main__":
    main()


from django.shortcuts import render, redirect
import requests

# Create your views here.
from order.models import Order

def index(request):
    """
    IOS에서 해줄 부
    """
    if request.method == 'POST':

        URL = 'https://kapi.kakao.com/v1/payment/ready'
        headers = {
            "Authorization": "KakaoAK " + "f9f70eb192ef14919735fb40a6e599f5",
            "Content-type": "application/x-www-form-urlencoded;charset=utf-8",
        }
        try:
            order_ins = Order.objects.last()
        except Order.DoesNotExist:
            order_ins = None

        if order_ins:
            cart_ins = order_ins.items.last().cart

            params = {
                "cid": "TC0ONETIME",  # 테스트용 코드
                "partner_order_id": f'{order_ins.id}',  # 주문번호
                "partner_user_id": 'admin',  # 유저 아이디
                "item_name": f'admin의 상품 구매',  # 구매 물품 이름
                "quantity": f'{cart_ins.quantity_of_goods}',  # 구매 물품 수량
                "total_amount": f'{order_ins.discount_payment}',  # 구매 물품 가격
                "tax_free_amount": "0",  # 구매 물품 비과세
                "approval_url": f"http://localhost:8000/api/order/{order_ins.id}/approve",
                "cancel_url": "https://developers.kakao.com/fail",
                "fail_url": "https://developers.kakao.com/cancel",
            }

            res = requests.post(URL, headers=headers, params=params)
            request.session['tid'] = res.json()['tid']  # 결제 고유 번호, 20자 결제 승인시 사용할 tid를 세션에 저장
            request.session['partner_order_id'] = order_ins.id
            request.session['partner_user_id'] = 'admin'
            next_url = res.json()['next_redirect_pc_url']  # 카카오톡 결제 페이지 Redirect URL

            request.session.modified = True
        return redirect(next_url)
    return render(request, 'base.html')

def success(request):
    """
    서버에서 할 부분
    """
    # 결제 성공
    URL = 'https://kapi.kakao.com/v1/payment/approve'
    headers = {
        "Authorization": "KakaoAK " + "f9f70eb192ef14919735fb40a6e599f5",
        "Content-type": "application/x-www-form-urlencoded;charset=utf-8",
    }
    params = {
        "cid": "TC0ONETIME",  # 테스트용 코드
        "tid": request.session['tid'],  # 결제 요청시 세션에 저장한 tid
        "partner_order_id": "1",  # 주문번호
        "partner_user_id": "admin",  # 유저 아이디
        "pg_token": request.GET.get("pg_token"),  # 쿼리 스트링으로 받은 pg토큰
    }
    res = requests.post(URL, headers=headers, params=params)
    amount = res.json()['amount']['total']
    res = res.json()
    context = {
        'res': res,
        'amount': amount,
    }
    return render(request, 'success.html')

def fail(request):
    return render(request, 'fail.html')


#Author: Michael Elgin (melgin@uwyo.edu)

#File to preprocess data from ../data/cifar-10-batches-py
#Assumes cifar-10-batches-py contains the extract from cifar-10-python.tar.gz
#Extracts the wanted 32x32 images to new folder(s) with their labels.
#This file should only need to be executed once.

import os
import pickle
import matplotlib.pyplot as plt

def main():

	DATA_PATH = "../data/cifar-10-batches-py/"
	IMG_PATH = "../data/32/"
	LABELS_PATH = "../data/labels/" #To match all datasets

	#asserts will stop the file from running if it need not be.
	assert(os.path.exists(DATA_PATH))
	assert(len(os.listdir(DATA_PATH)) != 0) #stuff inside
	assert(not (os.path.exists(IMG_PATH) and os.path.exists(LABELS_PATH)))

	os.mkdir(IMG_PATH)
	os.mkdir(IMG_PATH + "train")
	os.mkdir(IMG_PATH + "test")
	os.mkdir(LABELS_PATH)

	#This list becomes the labels for horse and ship subsets of cifar-10
	train_label_subset = []
	test_label_subset = []

	#horses and ships now get gathered with their labels in sync
	batch_files = [DATA_PATH + 'data_batch_' + str(i) for i in range(1,6)] + [DATA_PATH + "test_batch"]
	item = 0
	for batch_file in batch_files:
		with open(batch_file, 'rb') as f:
			batch_dict = pickle.load(f, encoding='bytes')
		batch_data = batch_dict[b'data']
		batch_labels = batch_dict[b'labels']
		for i in range(len(batch_labels)):
			if (batch_labels[i] == 7) or (batch_labels[i] == 8): #horse or ship
				img = batch_data[i].reshape(3,32,32)
				img = img.transpose(1,2,0) #Ref binarystudy.com
				if (batch_file == DATA_PATH + "test_batch"):
					test_label_subset.append(batch_labels[i])
					plt.imsave(IMG_PATH + "test/" + str(item) + ".png", img)
				else:
					train_label_subset.append(batch_labels[i])
					plt.imsave(IMG_PATH + "train/" + str(item) + ".png", img)
				item += 1

	assert(len(train_label_subset) == 10000) #5000 horse + 5000 ship
	assert(len(test_label_subset) == 2000) #1000 horse + 1000 ship

	#Convert to 0 and 1
	train_label_subset = [(0 if label == 7 else 1) for label in train_label_subset]
	test_label_subset = [(0 if label == 7 else 1) for label in test_label_subset]

	with open(LABELS_PATH + "train.pickle", "wb") as f:
		pickle.dump(train_label_subset, f)
	with open(LABELS_PATH + "test.pickle", "wb") as f:
		pickle.dump(test_label_subset, f)

if __name__ == "__main__":
	main()


import json
import urllib.request
from datetime import date
import matplotlib.pyplot as plt
from Kfilter import KalmanFilter

"""
Usage: Gets and prepares poll data from a json file. The format used should be similar to :
https://github.com/nsppolls/nsppolls
"""

class PollData:
    def __init__(self):
        self.Data = []
        self.FormattedData = dict()
        self.Settings = dict()
    """
    Loads the data from the provided source
    :param source: URL for the data. http://, https://, ftp://, ftps://. Filepaths should be prefixed with file://.

    """
    def load(self):

        with urllib.request.urlopen(self.Settings['Source']) as s:
            self.Data = json.load(s)

    def formatdata(self, tour: str, hypothese: str):
        self.FormattedData = dict(zip([x[0] for x in self.Settings['Candidats']], [[] for _ in self.Settings['Candidats']]))
        for sondage in [x for x in self.Data if (x['nom_institut'] in self.Settings['Instituts'])]:
            for t in [x for x in sondage['tours'] if x['tour'] == tour]:
                for hypo in [x for x in t['hypotheses'] if hypothese == '*' or hypothese == x['hypothese']]:
                    for result in [x for x in hypo['candidats'] if x['candidat'] in self.FormattedData.keys()]:
                        self.FormattedData[result['candidat']].append({'candidat': result['candidat'],
                                                                       'date': date.fromisoformat(sondage['fin_enquete']),
                                                                       'score': float(result['intentions']),
                                                                       'err_inf': float(result['erreur_inf']),
                                                                       'err_sup': float(result['erreur_sup']),
                                                                       'rolling': sondage['rolling'] == 'true'})

    @staticmethod
    def setchartsettings(fig, ax):
        facecolor, axcolor = '#384f75', '#ffffffC0'
        ax.tick_params(color=axcolor, labelcolor=axcolor)
        for spine in ax.spines.values():
            spine.set_edgecolor(axcolor)
        fig.set_facecolor(facecolor)
        ax.set_ylim([0, None])

        ax.set_facecolor(facecolor)
        ax.grid(color=axcolor, alpha=0.1)
        ax.vlines(x=[date(2022, 2, 24), '2022-02-24'], ymin=0, ymax=1, transform=ax.get_xaxis_transform(),
                  color='#FF000070', label='Invasion de l\'Ukraine', linewidth=1)
        ax.vlines(x=[date(2022, 3, 7), '2022-03-07'], ymin=0, ymax=1, transform=ax.get_xaxis_transform(),
                  color='#00FF0040', label='Publication de la liste des candidats', linewidth=1)
        ax.vlines(x=[date(2022, 4, 10), '2022-04-10'], ymin=0, ymax=1, transform=ax.get_xaxis_transform(),
                  color='#036ffcF0', label='1er tour de l\'élection présidentielle', linewidth=1)
        for line in fig.legend(loc='lower center', facecolor='#25354C', edgecolor='#121A26', labelcolor=axcolor) \
                .get_lines():
            line.set_linewidth(2)
            line.set_alpha(1)

        ax.text(0, 0, '@VaeVix', transform=ax.transAxes, fontsize=60, color='white',
                alpha=0.2, ha='left', va='bottom', rotation='0')

    def showchart(self, candidatA):
        fig, ax = plt.subplots(nrows=1, ncols=1)
        tf = {'family': 'serif', 'color':  'white', 'weight': 'normal'}
        fig.suptitle('Intentions de vote à l\'élection présidentielle de 2022', fontsize=22, fontdict=tf)
        ax.set_title('Instituts pris en compte : '+', '.join(self.Settings['Instituts']), fontsize=10, color='white')
        for candidat in candidatA:
            couleur = [x[1] for x in self.Settings['Candidats'] if x[0] == candidat].pop() +\
                      [x[2] for x in self.Settings['Candidats'] if x[0] == candidat].pop()
            ckline = [x[1] for x in self.Settings['Candidats'] if x[0] == candidat].pop() + 'FF'
            cerror = [x[1] for x in self.Settings['Candidats'] if x[0] == candidat].pop() + '20'
            data = [x for x in self.FormattedData[candidat] if self.Settings['From'] <= x['date'] <= self.Settings['To']]
            data.sort(key=lambda x: x['date'])

            KF = KalmanFilter(data[0]['score'])
            dates, scores, Kscores = [x['date'] for x in data], [x['score'] for x in data],\
                                     [KF.filter(x['score']) for x in data]
            fdates = []
            fKscores = []
            acc = (dates[0], Kscores[0])
            for i in range(len(dates)):
                if dates[i] > acc[0]:
                    fdates.append(acc[0])
                    fKscores.append(acc[1])

                acc = (dates[i], Kscores[i])

            fdates.append(acc[0])
            fKscores.append(acc[1])
            ax.plot_date(dates, scores, color=couleur)

            ax.plot_date(fdates, [x[0][0] for x in fKscores], color=ckline, ls='-', marker=None)
            ax.annotate('   '+str(round(fKscores[-1][0][0], 1)) + '% - '+candidat, (dates[-1], Kscores[-1][0][0]),
                        color=ckline)

            # Marges d'erreur
            plt.fill_between(fdates, [x[2][0] for x in fKscores], [x[3][0] for x in fKscores], color=cerror)

        self.setchartsettings(fig, ax)
        plt.show()


import logging
import multiprocessing
import os
import shutil
import subprocess

SOURCE_BASE_URL = 'http://ftp.gnu.org/gnu/glibc'
GLIBC_VERSION = 'glibc-2.27'

PATH_THIS = os.path.abspath (os.path.dirname (__file__))
PATH_ARTIFACTS = os.path.join (os.path.dirname (PATH_THIS), 'artifacts')
PATH_GLIBC = os.path.join (PATH_ARTIFACTS, GLIBC_VERSION)
PATH_GLIBC_BUILD = os.path.join (PATH_ARTIFACTS, 'glibc-build')
PATH_GLIBC_OUTPUT = os.path.join (PATH_ARTIFACTS, 'glibc-output')
PATH_LINUX_OUTPUT = os.path.join (PATH_ARTIFACTS, 'linux-output')

#-----------------------------------------------------------------------------
# build

def build ():
   logging.info ('Building %s' % GLIBC_VERSION)
   fetch ()
   clean ()

   if not os.path.exists (PATH_GLIBC_BUILD):
      os.makedirs (PATH_GLIBC_BUILD)

   configure ()
   make ()

   if not os.path.exists (PATH_GLIBC_OUTPUT):
      os.makedirs (PATH_GLIBC_OUTPUT)

   install ()

#-----------------------------------------------------------------------------
# fetch

def fetch ():
   if not os.path.exists (PATH_ARTIFACTS):
      os.makedirs (PATH_ARTIFACTS)

   if not os.path.exists (PATH_GLIBC):
      logging.info ('   Downloading')

      dl_cmd = [
         'wget', '-q',
         '-O', os.path.join (PATH_ARTIFACTS, '%s.tar.bz2' % GLIBC_VERSION),
         '%s/%s.tar.bz2' % (SOURCE_BASE_URL, GLIBC_VERSION)
      ]

      subprocess.check_call (dl_cmd)

      logging.info ('   Extracting')

      tar_cmd = [
         'tar', '-xf', os.path.join (PATH_ARTIFACTS, '%s.tar.bz2' % GLIBC_VERSION)
      ]

      subprocess.check_call (tar_cmd, cwd = PATH_ARTIFACTS)

#-----------------------------------------------------------------------------
# clean

def clean ():
   logging.info ('   Cleaning')

   if os.path.exists (PATH_GLIBC_BUILD):
      shutil.rmtree (PATH_GLIBC_BUILD)

   if os.path.exists (PATH_GLIBC_OUTPUT):
      shutil.rmtree (PATH_GLIBC_OUTPUT)

#-----------------------------------------------------------------------------
# configure

def configure ():
   logging.info ('   Configuring')

   log_file = open (os.path.join (PATH_ARTIFACTS, '%s-configure-log.txt' % GLIBC_VERSION), 'w')

   cmd = [
      os.path.join (PATH_GLIBC, 'configure'),
      '--prefix=',
      '--with-headers=%s/include' % PATH_LINUX_OUTPUT,
      '--without-gd',
      '--without-selinux',
      '--disable-werror',
      'CFLAGS=-Os -s -fno-stack-protector -fomit-frame-pointer -U_FORTIFY_SOURCE'
   ]
   subprocess.check_call (cmd, cwd = PATH_GLIBC_BUILD, stdout = log_file, stderr = log_file)

#-----------------------------------------------------------------------------
# make

def make ():
   logging.info ('   Building')

   log_file = open (os.path.join (PATH_ARTIFACTS, '%s-make-log.txt' % GLIBC_VERSION), 'w')

   cmd = [
      'make',
      '-j', '%d' % multiprocessing.cpu_count ()
   ]
   subprocess.check_call (cmd, cwd = PATH_GLIBC_BUILD, stdout = log_file, stderr = log_file)

#-----------------------------------------------------------------------------
# install

def install ():
   logging.info ('   Installing')

   log_file = open (os.path.join (PATH_ARTIFACTS, '%s-make-install-log.txt' % GLIBC_VERSION), 'w')

   cmd = [
      'make', 'install',
      'DESTDIR=%s' % PATH_GLIBC_OUTPUT,
      '-j', '%d' % multiprocessing.cpu_count ()
   ]
   subprocess.check_call (cmd, cwd = PATH_GLIBC_BUILD, stdout = log_file, stderr = log_file)


from model import Transformer
from torch.utils.data import DataLoader
import torch
import torch.nn as nn
from DataLoader import SensorDataset
import logging
import time # debugging
from plot import *
from helpers import *
from joblib import load
from icecream import ic

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s %(message)s", datefmt="[%Y-%m-%d %H:%M:%S]")
logger = logging.getLogger(__name__)

def inference(path_to_save_predictions, forecast_window, dataloader, device, path_to_save_model, best_model):

    # 将device转换为torch.device类型
    device = torch.device(device)

    # 创建Transformer模型实例，将其转换为double类型，并将其移动到指定的device上
    model = Transformer().double().to(device)

    # 加载预训练模型
    model.load_state_dict(torch.load(path_to_save_model+best_model))

    # 定义损失函数为均方误差
    criterion = torch.nn.MSELoss()

    # 初始化验证集损失为0
    val_loss = 0

    # 关闭梯度计算
    with torch.no_grad():

        # 将模型设置为评估模式
        model.eval()

        # 对于25个plot
        for plot in range(25):

            for index_in, index_tar, _input, target, sensor_number in dataloader:

                # starting from 1 so that src matches with target, but has same length as when training
                # 将输入数据的维度从 (batch_size, seq_len, feature_dim) 转换为 (seq_len, batch_size, feature_dim)
                src = _input.permute(1,0,2).double().to(device)[1:, :, :] # 47, 1, 7: t1 -- t47
                # 将目标数据的维度从 (batch_size, seq_len, feature_dim) 转换为 (seq_len, batch_size, feature_dim)
                target = target.permute(1,0,2).double().to(device) # t48 - t59

                # 将模型下一个输入设置为当前输入数据
                next_input_model = src
                # 初始化所有预测结果为空列表
                all_predictions = []

                for i in range(forecast_window - 1):

                    # 用模型预测下一个时间步的值
                    prediction = model(next_input_model, device) # 47,1,1: t2' - t48'

                    if all_predictions == []:
                        all_predictions = prediction # 47,1,1: t2' - t48'
                    else:
                        # 将预测结果拼接到 all_predictions 中
                        all_predictions = torch.cat((all_predictions, prediction[-1,:,:].unsqueeze(0))) # 47+,1,1: t2' - t48', t49', t50'

                    # 获取当前时间步之后的所有位置编码
                    pos_encoding_old_vals = src[i+1:, :, 1:] # 46, 1, 6, pop 位置编码的第一个值: t2 -- t47
                    # 获取最后一个预测值的位置编码
                    pos_encoding_new_val = target[i + 1, :, 1:].unsqueeze(1) # 1, 1, 6, 添加最后一个预测值的位置编码: t48
                    # 将所有位置编码与预测值匹配
                    pos_encodings = torch.cat((pos_encoding_old_vals, pos_encoding_new_val)) # 47, 1, 6: t2 -- t48 的位置编码

                    # 将当前时间步之后的输入数据和最后一个预测值拼接在一起，作为下一个时间步的输入数据
                    next_input_model = torch.cat((src[i+1:, :, 0].unsqueeze(-1), prediction[-1,:,:].unsqueeze(0))) #t2 -- t47, t48'
                    # 将位置编码拼接在一起，作为下一个时间步的输入数据
                    next_input_model = torch.cat((next_input_model, pos_encodings), dim = 2) # 47, 1, 7 input for next round

                true = torch.cat((src[1:,:,0],target[:-1,:,0]))
                loss = criterion(true, all_predictions[:,:,0])
                val_loss += loss

            val_loss = val_loss/10 # 将验证集的损失除以10，得到平均损失
            scaler = load('scalar_item.joblib') # 加载标量
            # 将输入数据的湿度值转换为原始值
            src_humidity = scaler.inverse_transform(src[:,:,0].cpu())
            # 将目标数据的湿度值转换为原始值
            target_humidity = scaler.inverse_transform(target[:,:,0].cpu())
            # 将所有预测结果的湿度值转换为原始值
            prediction_humidity = scaler.inverse_transform(all_predictions[:,:,0].detach().cpu().numpy())
            # 绘制预测结果图像
            plot_prediction(plot, path_to_save_predictions, src_humidity, target_humidity, prediction_humidity, sensor_number, index_in, index_tar)

        logger.info(f"Loss On Unseen Dataset: {val_loss.item()}")


import os
import socket
import ssl
from logging import getLogger

import pnp.infra.utils.pnp_constants as consts
from pnp.infra.system.certificate import get_certificates_location

def load_csdl_ssl_context(cafile_='', default_certs=False, trustpool=False):
    """Returns SSLContext object initialized to meet CSDL standards.

    Args:
        cafile_ (str, optional): Path to CA file containing PEM encoded cert(s).
            Defaults to empty string.
        default_certs (bool, optional): Load default OpenSSL certificates.
            Defaults to False.
        trustpool (bool, optional): Load an existing Trustpool bundle. Defaults
            to False.

    Returns:
        SSLContext: SSL Context loaded with requested certificates.

    """
    # Important to overload ssl.match_hostname in order support verifying the
    # identity of HTTPS server's certificate through the IP address in the
    # Subject Alternate Name field
    ssl.match_hostname = _match_hostname_ip
    ssl_context = ssl.SSLContext(consts.CSDL_SSL_PROTOCOL)
    ssl_context.verify_mode = ssl.CERT_REQUIRED
    # Set hostname verification during TLS handshake
    ssl_context.check_hostname = consts.CSDL_SSL_CHECK_HOSTNAME
    # Set CSDL approved ciphers in OpenSSL cipher list format
    ssl_context.set_ciphers(':'.join(consts.CSDL_CIPHERS))
    if cafile_ == 'default' or default_certs or consts.DEFAULT_CERTS_ENABLED:
        ssl_context.load_default_certs(ssl.Purpose.SERVER_AUTH)
    if trustpool:
        trustpool_path = get_certificates_location('trustpool')
        if os.path.isfile(os.path.join(trustpool_path, 'trustpool.pem')):
            ssl_context.load_verify_locations(trustpool_path)
    try:
        if cafile_:
            ssl_context.load_verify_locations(cafile=cafile_)
    except (ssl.SSLError, IOError, TypeError):
        if cafile_ != 'default':
            getLogger(__name__).exception("Failed to load configured cafile")
    return ssl_context

def _match_hostname_ip(cert, hostname):
    """This function is copied from ssl.match_hostname available in Python 2.7.9
    and has been extended to support server identity verification through
    IP Address in the subjectAltName field of cert. Verify that *cert* (in
    decoded format as returned by SSLSocket.getpeercert()) matches the
    *hostname*.  RFC 2818 and RFC 6125 rules are followed, but IP addresses are
    not accepted for *hostname*.

    CertificateError is raised on failure. On success, the function returns
    nothing.
    """
    if not cert:
        raise ValueError("empty or no certificate, match_hostname needs a "
                         "SSL socket or SSL context with either "
                         "CERT_OPTIONAL or CERT_REQUIRED")
    dnsnames = []
    san = cert.get('subjectAltName', ())
    for key, value in san:
        if key == 'DNS':
            if ssl._dnsname_match(value, hostname):  # pylint: disable=protected-access
                return
            elif hostname.endswith('.'):  # don't enforce trailing-dot in FQDN
                if ssl._dnsname_match(value, hostname[:-1]):  # pylint: disable=protected-access
                    return
            dnsnames.append(value)
        # PNP SPECIFIC CODE STARTS HERE
        elif key == 'IP Address':
            # Enhancement: if _ip_address_match(value, hostname): return
            if _ssl_ip_address_match(value, hostname):
                return
            dnsnames.append(value)
        # PNP SPECIFIC CODE ENDS HERE
    if not dnsnames:
        # The subject is only checked when there is no dNSName entry
        # in subjectAltName
        for sub in cert.get('subject', ()):
            for key, value in sub:
                if key == 'commonName':
                    if ssl._dnsname_match(value, hostname):  # pylint: disable=protected-access
                        return
                    dnsnames.append(value)
    if len(dnsnames) > 1:
        raise ssl.CertificateError("hostname %r doesn't match either of %s" %
                                   (hostname, ', '.join(map(repr, dnsnames))))  # pylint: disable=bad-builtin
    elif len(dnsnames) == 1:
        raise ssl.CertificateError("hostname %r doesn't match %r" %
                                   (hostname, dnsnames[0]))
    else:
        raise ssl.CertificateError("no appropriate commonName or subjectAltName"
                                   " fields were found")

def _ssl_ip_address_match(value, ip_address):
    """Retruns True if ip_address matches value"""
    # Remove any extra whitespace characters from cert value
    value = value.strip()
    # Determine ip_address protocol version and convert to packed, binary format
    address_family = socket.AF_INET
    try:
        binary_ip_address = socket.inet_aton(ip_address)
    except socket.error:
        address_family = socket.AF_INET6
        try:
            binary_ip_address = socket.inet_pton(address_family, ip_address)
        except socket.error:
            # ip_address is neither IPv6 nor IPv4
            return False
    # Convert value to packed, binary format and enforce address family
    try:
        binary_value = socket.inet_pton(address_family, value)
    except socket.error:
        return False
    # Compare the two addresses after converting each to packed, binary format
    return binary_value == binary_ip_address


#!/usr/bin/env python3
from __future__ import print_function

try:
    from ruamel import yaml
except ImportError:
    import yaml

# Map a Docker Compose "service" name to its image.
SERVICES = {
    'client': 'ubuntu:14.04',
    '8u121-jre-alpine': 'openjdk:8u121-jre-alpine',
    '8u121-jre': 'openjdk:8u121-jre',
}

# docker-compose.yml skeleton to fill out using "service" entries above.
COMPOSITION = {'services': {}, 'version': '3'}

def servicize(name, image):
    '''Create a docker-compose.yml service-section entry.

    Params
    ======
    name : str
        Name of the service.
    image : str
        Docker image to use (what your FROM line would contain).

    Returns
    =======
        dict describing the service. A name starting with 'client' is special.
    '''
    entry = {'command': ['/entrypoint.sh'],
             'image': image,
             'volumes': ['./entrypoint.sh:/entrypoint.sh:ro']}
    if not name.startswith('client'):
        entry['ports'] = ['9000', '9443']
        entry['healthcheck'] = {
            'test': 'curl --fail --insecure https://localhost:9443/'}
        entry['volumes'].append('./svc:/svc:ro')
    return entry

if __name__ == '__main__':
    for name, image in SERVICES.items():
        COMPOSITION['services'][name] = servicize(name, image)
    print(yaml.dump(COMPOSITION, default_flow_style=False, indent=4), end='')


#!/usr/bin/python
from flask import Flask
import os
from config import devConfig,prodConfig

app = Flask(__name__)

port = int(os.getenv('PORT', 5500))

import os
if os.environ['ENV'] == 'prod':
    prodConfig()
elif os.environ['ENV'] == 'dev':
    devConfig()

@app.route('/')
def root():
    return "Hello Intel"

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=port, debug=True)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import os, logging, torch, json
import os.path as osp
from collections import OrderedDict

from network.base_network import *
from utils import utils
from utils import config as cfg

class Network(BaseNetwork):
    root_logger = logging.getLogger("network %s"%__file__.split('/')[-1])

    def __init__(self, dataloader, args, feat_dim=None):
        super(Network, self).__init__(dataloader, args, feat_dim)

        self.dset = self.dataloader.dataset
        self.attr_embedder = utils.Embedder(args.wordvec, self.dset.attrs, args.data)
        self.emb_dim = self.attr_embedder.emb_dim      # dim of wordvec (attr or obj)

        self.pos_attr_id    = tf.placeholder(tf.int32, shape=[None])
        self.pos_obj_id     = tf.placeholder(tf.int32, shape=[None])
        self.pos_image_feat = tf.placeholder(tf.float32, shape=[None, self.feat_dim])

        self.neg_attr_id    = tf.placeholder(tf.int32, shape=[None])
        self.neg_obj_id     = tf.placeholder(tf.int32, shape=[None])
        self.neg_image_feat = tf.placeholder(tf.float32, shape=[None, self.feat_dim])

        self.test_attr_id   = tf.placeholder(tf.int32, shape=[None], name='test_attr_id')
        self.test_obj_id    = tf.placeholder(tf.int32, shape=[None], name='test_obj_id')

        self.pos_obj_prediction = tf.placeholder(tf.float32, shape=[None, self.num_obj])

        self.lr = tf.placeholder(tf.float32)

    def rep_embedder(self, img, is_training, name):
        """transform ResNet feature to image representation"""
        img = self.MLP(img, self.args.rep_dim, is_training, name, hidden_layers=[])
        return img

    def transformer(self, rep, v_attr, is_training, name):
        """CoN or DecoN in our paper"""

        with tf.variable_scope(name) as scope:
            in_dim = out_dim = self.args.rep_dim

            if not self.args.no_attention:
                attention = self.MLP(v_attr, in_dim, is_training,
                    name='fc_attention', hidden_layers=self.args.fc_att)
                attention = tf.sigmoid(attention)

                rep = attention*rep + rep      # short connection

            hidden = tf.concat([rep, v_attr], axis=1)  # 512+300

            output = self.MLP(hidden, out_dim, is_training,
                name='fc_out', hidden_layers=self.args.fc_compress)

        return output

    def attr_classification(self, emb, is_training, name='classifier'):
        with tf.variable_scope(name) as scope:
            score_A = self.MLP(emb, self.num_attr, is_training, "attr",
                hidden_layers=self.args.fc_cls)
            prob_A = tf.nn.softmax(score_A, 1)
            return score_A, prob_A

    def obj_classification(self, emb, is_training, name='classifier'):
        with tf.variable_scope(name) as scope:
            score_O = self.MLP(emb, self.num_obj, is_training, "obj",
                hidden_layers=self.args.fc_cls)
            prob_O = tf.nn.softmax(score_O, 1)
            return score_O, prob_O

    def RMD_prob(self, feat_plus, feat_minus, repeat_img_feat, is_training, metric):
        """return attribute classification probability with our RMD"""
        # feat_plus, feat_minus:  shape=(bz, #attr, dim_emb)
        # d_plus: distance between feature before&after CoN
        # d_minus: distance between feature before&after DecoN

        d_plus = self.distance_metric(feat_plus, repeat_img_feat)
        d_minus = self.distance_metric(feat_minus, repeat_img_feat)
        d_plus = tf.reshape(d_plus, [-1, self.num_attr]) # bz, #attr
        d_minus = tf.reshape(d_minus, [-1, self.num_attr]) # bz, #attr

        if metric == 'softmax':
            p_plus  = tf.nn.softmax(-d_plus, 1) # (bz, #attr), smaller = better
            p_minus = tf.nn.softmax(d_minus, 1) # (bz, #attr), larger = better
            return p_plus, p_minus

        elif metric == 'rmd':
            d_plus_comp  = self.dset.comp_gamma['b']*d_plus
            d_minus_comp = self.dset.comp_gamma['a']*d_minus
            d_plus_attr  = self.dset.attr_gamma['b']*d_plus
            d_minus_attr = self.dset.attr_gamma['a']*d_minus

            p_comp = tf.nn.softmax(d_minus_comp - d_plus_comp, axis=1)
            p_attr = tf.nn.softmax(d_minus_attr - d_plus_attr, axis=1)

            return p_comp, p_attr

    def build_network(self, test_only=False):
        logger = self.logger('create_train_arch')

        total_losses = []

        batchsize = tf.shape(self.pos_image_feat)[0]

        pos_attr_emb = self.attr_embedder.get_embedding(self.pos_attr_id)
        neg_attr_emb = self.attr_embedder.get_embedding(self.neg_attr_id)

        pos_img = self.rep_embedder(self.pos_image_feat,
            is_training=True, name='embedder') # (bz,dim)
        neg_img = self.rep_embedder(self.neg_image_feat,
            is_training=True, name='embedder') # (bz,dim)

        # rA = remove positive attribute A
        # aA = add positive attribute A
        # rB = remove negative attribute B
        # aB = add negative attribute B
        pos_aA = self.transformer(pos_img, pos_attr_emb, True, name='CoN')
        pos_aB = self.transformer(pos_img, neg_attr_emb, True, name='CoN')
        pos_rA = self.transformer(pos_img, pos_attr_emb, True, name='DeCoN')
        pos_rB = self.transformer(pos_img, neg_attr_emb, True, name='DeCoN')

        attr_emb = self.attr_embedder.get_embedding(np.arange(self.num_attr))
        # (#attr, dim_emb), wordvec of all attributes
        tile_attr_emb = utils.tile_tensor(attr_emb, 0, batchsize)
        # (bz*#attr, dim_emb)

        ########################## classification losses ######################
        # unnecessary to compute cls loss for neg_img

        if self.args.lambda_cls_attr > 0:

            # original image
            _, prob_pos_A = self.attr_classification(pos_img, is_training=True)
            loss_cls_pos_a = self.cross_entropy(
                prob_pos_A, self.pos_attr_id, self.num_attr,
                weight=self.attr_weight)

            # after removing pos_attr
            _, prob_pos_rA_A = self.attr_classification(pos_rA, is_training=True)
            loss_cls_pos_rA_a = self.cross_entropy(
                prob_pos_rA_A, self.pos_attr_id, self.num_attr,
                target=0, weight=self.attr_weight)

            # rmd
            repeat_img_feat = utils.repeat_tensor(pos_img, 0, self.num_attr)
            # (bz*#attr, dim_rep)
            feat_plus  = self.transformer(repeat_img_feat, tile_attr_emb,
                is_training=True, name='CoN')
            feat_minus = self.transformer(repeat_img_feat, tile_attr_emb,
                is_training=True, name='DeCoN')

            prob_RMD_plus, prob_RMD_minus = self.RMD_prob(feat_plus, feat_minus,
                repeat_img_feat, is_training=True, metric=self.args.rmd_metric)

            loss_cls_rmd_plus  = self.cross_entropy(
                prob_RMD_plus, self.pos_attr_id, self.num_attr,
                weight=self.attr_weight)
            loss_cls_rmd_minus = self.cross_entropy(
                prob_RMD_minus, self.pos_attr_id, self.num_attr,
                weight=self.attr_weight)

            loss_cls_attr = self.args.lambda_cls_attr * sum([
                loss_cls_pos_a, loss_cls_pos_rA_a,
                loss_cls_rmd_plus, loss_cls_rmd_minus
            ])

            total_losses.append(loss_cls_attr)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_cls_attr', loss_cls_attr)

        if self.args.lambda_cls_obj > 0:
            # original image
            _, prob_pos_O = self.obj_classification(pos_img, is_training=True)
            loss_cls_pos_o = self.cross_entropy(
                prob_pos_O, self.pos_obj_id, self.num_obj,
                weight=self.obj_weight)

            # after removing pos_attr
            _, prob_pos_rA_O = self.obj_classification(pos_rA, is_training=True)
            loss_cls_pos_rA_o = self.cross_entropy(
                prob_pos_rA_O, self.pos_obj_id, self.num_obj,
                weight=self.obj_weight)

            # after adding neg_attr
            _, prob_pos_aB_O = self.obj_classification(pos_aB, is_training=True)
            loss_cls_pos_aB_o = self.cross_entropy(
                prob_pos_aB_O, self.pos_obj_id, self.num_obj,
                weight=self.obj_weight)

            loss_cls_obj = self.args.lambda_cls_obj * sum([
                loss_cls_pos_o,
                loss_cls_pos_rA_o,
                loss_cls_pos_aB_o
            ])

            total_losses.append(loss_cls_obj)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_cls_obj', loss_cls_obj)

        ############################# symmetry loss ###########################

        if self.args.lambda_sym > 0:

            loss_sym_pos = self.MSELoss(pos_aA, pos_img)
            loss_sym_neg = self.MSELoss(pos_rB, pos_img)

            loss_sym = self.args.lambda_sym * (loss_sym_pos + loss_sym_neg)
            total_losses.append(loss_sym)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_sym', loss_sym)

        ############################## axiom losses ###########################
        if self.args.lambda_axiom > 0:
            loss_clo = loss_inv = loss_com = 0

            # closure
            if not self.args.remove_clo:
                pos_aA_rA = self.transformer(pos_aA, pos_attr_emb,
                    is_training=True, name='DeCoN')
                pos_rB_aB = self.transformer(pos_rB, neg_attr_emb,
                    is_training=True, name='CoN')
                loss_clo = self.MSELoss(pos_aA_rA, pos_rA) + \
                            self.MSELoss(pos_rB_aB, pos_aB)

            # invertibility
            if not self.args.remove_inv:
                pos_rA_aA = self.transformer(pos_rA, pos_attr_emb,
                    is_training=True, name='CoN')
                pos_aB_rB = self.transformer(pos_aB, neg_attr_emb,
                    is_training=True, name='DeCoN')
                loss_inv = self.MSELoss(pos_rA_aA, pos_img) + \
                    self.MSELoss(pos_aB_rB, pos_img)

            # Commutativity
            if not self.args.remove_com:
                pos_aA_rB = self.transformer(pos_aA, neg_attr_emb,
                    is_training=True, name='DeCoN')
                pos_rB_aA = self.transformer(pos_rB, pos_attr_emb,
                    is_training=True, name='CoN')
                loss_com = self.MSELoss(pos_aA_rB, pos_rB_aA)

            loss_axiom = self.args.lambda_axiom * (
                loss_clo + loss_inv + loss_com)
            total_losses.append(loss_axiom)

            with tf.device("/cpu:0"):
                with tf.name_scope('loss'):
                    tf.summary.scalar('loss_axiom', loss_axiom)
                    tf.summary.scalar('loss_clo', loss_clo)
                    tf.summary.scalar('loss_inv', loss_inv)
                    tf.summary.scalar('loss_com', loss_com)

        ############################# triplet loss ############################

        if self.args.lambda_trip > 0:
            pos_triplet = tf.reduce_mean(
                self.triplet_margin_loss(pos_img, pos_aA, pos_rA))
            neg_triplet = tf.reduce_mean(
                self.triplet_margin_loss(pos_img, pos_rB, pos_aB))

            loss_triplet = self.args.lambda_trip * (pos_triplet + neg_triplet)
            total_losses.append(loss_triplet)

            with tf.device("/cpu:0"):
                tf.summary.scalar('loss/loss_triplet', loss_triplet)

        ################################ testing ##############################
        pos_img = self.rep_embedder(self.pos_image_feat, False, "embedder")
        repeat_img_feat = utils.repeat_tensor(pos_img, 0, self.num_attr)
        # (bz*#attr, dim_rep)

        feat_plus = self.transformer(repeat_img_feat, tile_attr_emb,
            is_training=False, name='CoN')
        feat_minus = self.transformer(repeat_img_feat, tile_attr_emb,
            is_training=False, name='DeCoN')

        prob_A_rmd, prob_A_attr = self.RMD_prob(feat_plus, feat_minus,
            repeat_img_feat, is_training=False, metric='rmd')

        _, prob_A_fc = self.attr_classification(pos_img, is_training=False)
        _, prob_O_fc = self.obj_classification(pos_img, is_training=False)

        if self.args.obj_pred is None:
            prob_O = prob_O_fc
        else:
            prob_O = self.pos_obj_prediction

        test_a_onehot = tf.one_hot(self.test_attr_id, depth=self.num_attr, axis=1)
        test_o_onehot = tf.one_hot(self.test_obj_id, depth=self.num_obj, axis=1)
        # (n_p, n_o)
        prob_P_rmd = tf.multiply(
            tf.matmul(prob_A_rmd, tf.transpose(test_a_onehot)),
            tf.matmul(prob_O, tf.transpose(test_o_onehot))
        )

        score_res = dict([
            ("score_rmd",   [prob_P_rmd, prob_A_attr, prob_O]),
        ])

        ############################### summary ###############################

        loss = sum(total_losses)

        with tf.device("/cpu:0"):
            tf.summary.scalar('loss_total', loss)
            tf.summary.scalar('lr', self.lr)

        train_summary_op = tf.summary.merge_all()

        return loss, score_res, train_summary_op

    def train_step(self, sess, blobs, lr, train_op, train_summary_op):

        summary, _ = sess.run(
            [train_summary_op, train_op],
            feed_dict={
                self.pos_attr_id     : blobs[1],
                self.pos_obj_id      : blobs[2],
                self.pos_image_feat  : blobs[4],
                self.neg_attr_id     : blobs[6],
                self.neg_image_feat  : blobs[9],
                self.lr: lr,
            })

        return summary


#!/usr/bin/env python

import os
import pathlib

PIPELINE_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
SRC_DIR = os.path.join(PIPELINE_DIR, "src")
UNIT_DIR = os.path.join(PIPELINE_DIR, "unit")

def convert_to_unittest(root, relative_path):
    # put _ut suffix in relative path subdirs
    p = pathlib.Path(os.path.dirname(relative_path))
    subdirs = [f"{d}_ut" for d in p.parts]
    subdirs = os.path.join(*subdirs) if len(subdirs) > 0 else ""

    # put _ut__define in the right place
    basename = os.path.basename(relative_path)
    basename_root, extension = os.path.splitext(basename)
    class_suffix = "__define"
    if basename_root.endswith(class_suffix):
        basename_root = basename_root[0:-len(class_suffix)]
    basename = f"{basename_root}_ut__define{extension}"

    return(os.path.join(os.path.join(root, subdirs), basename))

def find_unit_path(path, src_root, unit_root):
    """Remove prefix or fail.
    """
    if path.startswith(src_root):
        relative_path = path[len(src_root):]
        if relative_path.startswith("/"):
            relative_path = relative_path[1:]
        return(convert_to_unittest(unit_root, relative_path))
    raise FileNotFoundError from None

def main():
    print("#!/bin/sh")
    print()

    for dir, subdirs, files in os.walk(SRC_DIR):
        pro_files = [f for f in files if os.path.splitext(f)[1] == ".pro"]
        for f in files:
            if os.path.splitext(f)[1] != ".pro":
                continue

            src_filename = os.path.join(dir, f)
            unit_filename = find_unit_path(src_filename, SRC_DIR, UNIT_DIR)
            if not os.path.exists(unit_filename):
                unit_dir = os.path.dirname(unit_filename)
                if not os.path.exists(unit_dir):
                    os.makedirs(unit_dir)
                print(f"mgunit_template {src_filename} > {unit_filename}")

if __name__ == "__main__":
    main()


import sys
import time
from unittest import mock

import arrow
import pytest
import staticconf.testing
from colorama import Fore
from colorama import Style

from clusterman.util import any_of
from clusterman.util import ask_for_choice
from clusterman.util import ask_for_confirmation
from clusterman.util import color_conditions
from clusterman.util import FunctionTimeoutError
from clusterman.util import get_cluster_name_list
from clusterman.util import get_pool_name_list
from clusterman.util import limit_function_runtime
from clusterman.util import parse_time_interval_seconds
from clusterman.util import parse_time_string
from clusterman.util import sensu_checkin
from clusterman.util import splay_event_time
from clusterman.util import Status

@pytest.fixture(autouse=True)
def mock_clog():
    sys.modules["clog"] = mock.Mock()  # clog is imported in the main function so this is how we mock it
    yield

@pytest.mark.parametrize("inp,response", [("\n", True), ("\n", False), ("yE", True), ("n", False)])
def test_ask_for_confirmation(inp, response):
    with mock.patch("builtins.input", side_effect=inp):
        assert ask_for_confirmation(default=response) == response

def test_ask_for_confirmation_invalid_input():
    with mock.patch("builtins.input", side_effect=["asdf", "yes"]) as mock_input:
        assert ask_for_confirmation() is True
        assert mock_input.call_count == 2

def test_ask_for_choice():
    with mock.patch("builtins.input", side_effect=["asdf", "-1", "20", "2"]) as mock_input:
        assert ask_for_choice("Pick something", ["a", "b", "c"]) == "c"
        assert mock_input.call_count == 4

@pytest.mark.parametrize(
    "input_str,color",
    [("foo", Fore.GREEN), ("baz", Fore.BLUE), ("hjkl", Fore.RED), ("qwerty", "")],
)
def test_color_conditions(input_str, color):
    assert (
        color_conditions(
            input_str,
            green=any_of("foo", "bar"),
            blue=any_of("baz", "asdf"),
            red=any_of(
                "hjkl",
            ),
        )
        == color + input_str + Style.RESET_ALL
    )

def test_parse_time_string_without_tz():
    t = parse_time_string("2017-08-01 00:00", tz="US/Eastern")
    assert t.timestamp == 1501560000

def test_parse_time_string_with_tz():
    # Ignore the tz argument here and use the '+04:00' in the string
    t = parse_time_string("2017-08-01T00:00:00+04:00", tz="US/Eastern")
    assert t.timestamp == 1501531200

def test_parse_time_string_non_arrow():
    t = parse_time_string("one hour ago", tz="US/Eastern")

    # This has potential to be a little flaky so there's a little wiggle room here
    actual_timestamp = arrow.now().replace(tzinfo="US/Eastern").shift(hours=-1).timestamp
    assert abs(actual_timestamp - t.timestamp) <= 1

def test_parse_time_interval_seconds():
    assert parse_time_interval_seconds("5m") == 60 * 5

def test_parse_time_interval_seconds_invalid():
    with pytest.raises(ValueError):
        parse_time_interval_seconds("asdf")

@mock.patch("clusterman.util._get_sensu")
class TestSensu:
    def _sensu_output(self, output, source, pool=None, app=None, scheduler=None):
        return "".join(
            [
                f"{output}\n\n",
                "This check came from:\n",
                f"- Cluster/region: {source}\n",
                f"- Pool: {pool}.{scheduler}\n" if pool else "",
                f"- App: {app}\n" if app else "",
            ]
        )

    @pytest.mark.parametrize("noop", [True, False])
    def test_sensu_checkin(self, mock_sensu, noop):
        sensu_checkin(
            check_name="my_check",
            output="output",
            source="my_source",
            noop=noop,
        )

        if noop:
            assert mock_sensu.return_value.send_event.call_count == 0
        else:
            assert mock_sensu.return_value.send_event.call_args == mock.call(
                name="my_check",
                output=self._sensu_output("output", "my_source"),
                source="my_source",
                status=Status.OK.value,
                runbook="y/my-runbook",
                team="my_team",
                page=True,
            )

    @pytest.mark.parametrize("app,pool,scheduler", [(None, None, None), ("bar", "foo", "mesos")])
    def test_args_overrides_config(self, mock_sensu, app, pool, scheduler):
        sensu_checkin(
            check_name="my_check",
            output="output",
            source="my_source",
            team="a_different_team",
            app=app,
            pool=pool,
            scheduler=scheduler,
        )

        assert mock_sensu.return_value.send_event.call_args == mock.call(
            name="my_check",
            source="my_source",
            output=self._sensu_output("output", "my_source", pool, app, scheduler),
            status=Status.OK.value,
            runbook="y/my-runbook" if not app else "y/their-runbook",
            team="a_different_team",
            page=True,
        )

    def test_fallback(self, mock_sensu):
        sensu_checkin(
            check_name="my_check",
            output="output",
            source="my_source",
            app="non-existent",
        )
        assert mock_sensu.return_value.send_event.call_args == mock.call(
            name="my_check",
            output=self._sensu_output("output", "my_source", app="non-existent"),
            source="my_source",
            status=Status.OK.value,
            runbook="y/my-runbook",
            team="my_team",
            page=True,
        )

def test_get_cluster_name_list():
    with staticconf.testing.MockConfiguration(
        {
            "clusters": {
                "cluster-A": {
                    "mesos_api_url": "service.leader",
                },
                "cluster-B": {
                    "mesos_api_url": "service.leader",
                },
            },
        },
        namespace=staticconf.config.DEFAULT,
    ):
        assert set(get_cluster_name_list()) == {"cluster-A", "cluster-B"}

@mock.patch("clusterman.util.get_cluster_config_directory")
@mock.patch("os.listdir")
def test_get_pool_name_list(mock_listdir, mock_get_cluster_config_directory):
    mock_get_cluster_config_directory.return_value = "/tmp/somedir/cluster-A"
    mock_listdir.return_value = [
        "pool-A.mesos",
        "pool-B.xml",
        "pool-C.mesos",
        "pool-D",
        "pool-F.kubernetes",
    ]
    assert set(get_pool_name_list("cluster-A", "mesos")) == {"pool-A", "pool-C"}
    assert set(get_pool_name_list("cluster-A", "kubernetes")) == {"pool-F"}
    assert mock_get_cluster_config_directory.call_args == mock.call("cluster-A")
    assert mock_listdir.call_args == mock.call("/tmp/somedir/cluster-A")

@mock.patch("builtins.hash")
def test_splay_event_time(mock_hash):
    frequency = 10
    mock_hash.return_value = frequency - 1  # make `hash(key) % frequency` returns its max value
    for timestamp in range(20):
        assert 0 <= splay_event_time(frequency, "key", timestamp) < frequency

def test_limit_function_runtime():
    assert limit_function_runtime(lambda: 123, 1) == 123
    with pytest.raises(FunctionTimeoutError):
        limit_function_runtime(lambda: time.sleep(10), 1)


from helper import display_task_name, display_example

'''
Excersise: Pangram
URL: https://exercism.org/tracks/python/exercises/pangram
Instructions

Determine if a sentence is a pangram. A pangram (Greek: παν γράμμα, pan gramma, "every letter") is a sentence using every letter of the alphabet at least once. The best known English pangram is:

    The quick brown fox jumps over the lazy dog.

The alphabet used consists of letters a to z, inclusive, and is case insensitive.
'''

def is_pangram(sentence: str) -> bool:
    alphabets = [
        "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M",
        "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"
    ]

    lower_sentence = sentence.lower()

    for character in alphabets:

        if character.lower() not in lower_sentence:
            return False

    return True

# Test 1
display_task_name("I", "Pangram > empty sentence")
display_example(
    'is_pangram("")',
    'False'
)
print(is_pangram(""), "\n")

# Test 2
display_task_name("II", "Pangram > perfect lower case")
display_example(
    'is_pangram("abcdefghijklmnopqrstuvwxyz")',
    'True'
)
print(is_pangram("abcdefghijklmnopqrstuvwxyz"), "\n")

# Test 3
display_task_name("III", "Pangram > only lower case")
display_example(
    'is_pangram("the quick brown fox jumps over the lazy dog")',
    'True'
)
print(is_pangram("the quick brown fox jumps over the lazy dog"), "\n")

# Test 4
display_task_name("IV", "Pangram > missing the letter x")
display_example(
    'is_pangram("a quick movement of the enemy will jeopardize five gunboats")',
    'False'
)
print(is_pangram("a quick movement of the enemy will jeopardize five gunboats"), "\n")

# Test 5
display_task_name("V", "Pangram > missing the letter h")
display_example(
    'is_pangram("five boxing wizards jump quickly at it")',
    'False'
)
print(is_pangram("five boxing wizards jump quickly at it"), "\n")

# Test 6
display_task_name("VI", "Pangram > with underscores")
display_example(
    'is_pangram("the quick brown fox jumps over the lazy dog")',
    'True'
)
print(is_pangram("the quick brown fox jumps over the lazy dog"), "\n")

# Test 7
display_task_name("VII", "Pangram > with numbers")
display_example(
    'is_pangram("the 1 quick brown fox jumps over the 2 lazy dogs")',
    'True'
)
print(is_pangram("the 1 quick brown fox jumps over the 2 lazy dogs"), "\n")

# Test 8
display_task_name("VIII", "Pangram > missing letters replaced by numbers")
display_example(
    'is_pangram("7h3 qu1ck brown fox jumps ov3r 7h3 lazy dog")',
    'False'
)
print(is_pangram("7h3 qu1ck brown fox jumps ov3r 7h3 lazy dog"), "\n")

# Test 9
display_task_name("IX", "Pangram > mixed case and punctuation")
display_example(
    'is_pangram(\'"Five quacking Zephyrs jolt my wax bed."\')',
    'True'
)
print(is_pangram('"Five quacking Zephyrs jolt my wax bed."'), "\n")

# Test 10
display_task_name("X", "Pangram > a m and a m are 26 different characters but not a pangram")
display_example(
    'is_pangram("abcdefghijklm ABCDEFGHIJKLM")',
    'False'
)
print(is_pangram("abcdefghijklm ABCDEFGHIJKLM"), "\n")

# Test 11
display_task_name("XI", "Pangram > sentence without lower bound")
display_example(
    'is_pangram("bcdefghijklmnopqrstuvwxyz")',
    'False'
)
print(is_pangram("bcdefghijklmnopqrstuvwxyz"), "\n")

# Test 12
display_task_name("XII", "Pangram > sentence without upper bound")
display_example(
    'is_pangram("abcdefghijklmnopqrstuvwxy")',
    'False'
)
print(is_pangram("abcdefghijklmnopqrstuvwxy"), "\n")


import itertools as it
import re
from collections import namedtuple
from copy import deepcopy
from inspect import cleandoc

import numpy as np

from getml.data.roles import (
    join_key,
    numerical,
    target,
    text,
    time_stamp,
    unused_float,
    unused_string,
)

from ..templates import environment
from .formatter import (
    MAX_CELL_WIDTH,
    _clip_cell,
    _FormatColumn,
    _Formatter,
    _IndexColumn,
    _infer_precision,
)
from .helpers import _get_data_frame_content

UNITS = ["D", "h", "m", "s", "ms"]
"""
Numpy datetime units.
"""

UNITS_FORMATS = dict(
    D="{:%Y-%m-%d}",
    h="{:%Y-%m-%d %H}",
    m="{:%Y-%m-%d %H:%M}",
    s="{:%Y-%m-%d %H:%M:%S}",
    ms="{:%Y-%m-%d %H:%M:%S.%f}",
)
"""
A mapping of numpy datetime units to strftime format specifiers.
"""

# --------------------------------------------------------------------

def _get_subroles(col):
    subroles = {}

    subroles["exclude"] = [
        _parse_subrole(subrole)
        for subrole in getattr(col, "subroles", [])
        if subrole.startswith("exclude ")
    ]
    subroles["include"] = [
        _parse_subrole(subrole)
        for subrole in getattr(col, "subroles", [])
        if subrole.startswith("include ")
    ]
    subroles["only"] = [
        _parse_subrole(subrole)
        for subrole in getattr(col, "subroles", [])
        if subrole.startswith("only ")
    ]

    return subroles

# --------------------------------------------------------------------

def _parse_subrole(subrole):
    prefix = re.compile(r"^(exclude |include |only )")
    return re.sub(prefix, "", subrole)

# --------------------------------------------------------------------

def _structure_subroles(df):
    subroles = {"exclude": [], "include": [], "only": []}
    df_subroles = [_get_subroles(df[col]) for col in df.colnames]

    for col_subroles in df_subroles:
        for cat, values in subroles.items():
            values.append(col_subroles[cat])

    return subroles

# --------------------------------------------------------------------

def _infer_resolution(cells):
    cells = np.array(cells, dtype="datetime64")
    deltas = np.unique(cells[1:] - cells[:-1])

    for unit in UNITS:
        normalized_deltas = np.array(
            [np.timedelta64(delta, "ms") for delta in deltas]
        ) / np.timedelta64(1, unit)
        if all(np.mod(normalized_deltas, 1) == 0):
            return unit

# --------------------------------------------------------------------

def _max_digits(cells):
    cells = [cell for cell in cells if "." in cell]
    if len(cells) > 0:
        return max(len(cell.split(".")[1]) for cell in cells)
    return 0

# --------------------------------------------------------------------

class _DataFrameFormatter(_Formatter):
    """
    A Formatter for DataFrames.
    """

    max_rows = 10

    template = environment.get_template("data_frame.jinja2")

    # ------------------------------------------------------------

    def __init__(self, df, num_head=max_rows // 2, num_tail=max_rows // 2):
        self.colnames = df.colnames
        self.n_rows = len(df)
        self.name = df.name
        self.memory_usage = df.memory_usage

        num_head = min(num_head, self.n_rows // 2 + 1)
        num_tail = min(num_tail, self.n_rows // 2)

        roles = [df[colname].role for colname in df.colnames]

        units = [df[colname].unit for colname in df.colnames]

        self.units = None

        self.subroles = {
            cat: subroles
            for cat, subroles in _structure_subroles(df).items()
            if any(subrole for subrole in subroles)
        }

        head = _get_data_frame_content(name=df.name, start=0, length=num_head)["data"]
        tail = _get_data_frame_content(
            name=df.name, start=int(self.n_rows - num_tail), length=num_tail
        )["data"]

        rows = head + tail

        columns = [list(col) for col in zip(*rows)]

        headers = [self.colnames] + [roles]

        if any(unit != "" for unit in units):
            self.units = units
            headers += [units]

        if self.subroles:
            headers += [[""] * len(columns)] + [
                subroles
                for subroles in self.subroles.values()
                if any(subrole for subrole in subroles)
            ]

        headers = list(zip(*headers))

        self.data = [
            _DataFrameFormatColumn(
                headers=list(header),
                cells=cells,
                max_cells=self.max_rows,
                n_cells=self.n_rows,
                role=role,
            )
            for header, role, cells in zip(headers, roles, columns)
        ]

        index_headers = ["name"]
        if self.roles:
            index_headers += ["role"]
        if self.units:
            index_headers += ["unit"]
        if self.subroles:
            index_headers += ["subroles:"] + [f"- {cat:7}" for cat in self.subroles]

        self._add_index(index_headers)

    # ------------------------------------------------------------

    def __len__(self):
        return self.n_rows

    # ------------------------------------------------------------

    def _render_footer_lines(self, footer):
        if footer.url is None or footer.name is None:
            return cleandoc(
                f"""
                {footer.n_rows} rows x {footer.n_cols} columns
                memory usage: {footer.memory_usage:.2f} MB
                type: {footer.type}
                """
            )
        return cleandoc(
            f"""
            {footer.n_rows} rows x {footer.n_cols} columns
            memory usage: {footer.memory_usage:.2f} MB
            name: {footer.name}
            type: {footer.type}
            url: {footer.url}
            """
        )

    # ------------------------------------------------------------

    def _add_index(self, headers):
        index_column = _IndexColumn(
            headers=headers,
            n_cells=self.n_rows or self.max_rows + 1,
            length=self.max_rows,
        )

        self.data = [index_column] + self.data

    # ------------------------------------------------------------

    def _render_html(self, footer=None):
        headers = self._render_head()
        rows = self._render_body(as_html=True)

        precisions = [getattr(column, "precision", None) for column in self.data]

        cell = namedtuple("cell", ["role", "value"])
        headers = [
            [cell(role, header) for role, header in zip(self.roles, headers)]
            for headers in headers
        ]
        rows = [
            [cell(role, value) for role, value in zip(self.roles, row)] for row in rows
        ]

        return self.template.render(
            headers=headers, rows=rows, precisions=precisions, footer=footer
        )

    # ------------------------------------------------------------

    def _render_body(self, as_html=False):
        for col in self.data:
            if col._instances_are(str):
                col.data = [_clip_cell(cell, MAX_CELL_WIDTH) for cell in col.data]

        columns_formatted = [column._format_cells() for column in self.data]

        columns_clipped = [column._clip() for column in columns_formatted]

        if as_html:
            cells = [column._unnest_and_strip_html() for column in columns_clipped]
        else:
            cells = [column._expand_cells() for column in columns_clipped]

        rows = [list(row) for row in zip(*cells)]

        return rows

    # ------------------------------------------------------------

    def _render_string(self, footer=None):

        lines = super()._render_string()

        footer_lines = self._render_footer_lines(footer) if footer else ""

        return lines + "\n\n\n" + footer_lines

    @property
    def roles(self):
        return [getattr(column, "role", None) for column in self.data]

# --------------------------------------------------------------------

class _DataFrameFormatColumn(_FormatColumn):
    """
    A Formatter for columns of a DataFrame.

    The cell format is set based on the column's role.
    """

    # ------------------------------------------------------------

    def __init__(self, *args, n_cells, role=None, **kwargs):
        super().__init__(*args, **kwargs)
        self.n_cells = n_cells
        self.resolution = None
        self.role = role

        # self.parse_cells()
        if self.role in [numerical, target, time_stamp, join_key, unused_float]:
            self.precision = _max_digits(self.data)

        self.cell_template = self.default_cell_template
        self.header_template = self.default_header_template

    # ------------------------------------------------------------

    def __len__(self):
        return len(self.data)

    # ------------------------------------------------------------

    def _format_cells(self):
        """
        A patch to justify time stamps. This is necessary because afaik we are not able
        to specify justification within strftime format strings.
        """
        cells_formatted = super()._format_cells()

        if self.role == "time_stamp":
            cells_formatted.data = [
                cell.ljust(self.width) for cell in cells_formatted.data
            ]

        return cells_formatted

    # ------------------------------------------------------------

    def _clip(self):
        ellipses = "..." + getattr(self, "precision", 0) * " "
        column_clipped = deepcopy(self)

        if self.role is None:
            ellipses = [self.header_template.format(ellipses)]
            if self.nested:
                ellipses = [ellipses]
            column_clipped.data += ellipses
            return column_clipped

        return super()._clip(ellipses=ellipses)

    # ------------------------------------------------------------

    def _parse_cells(self):
        # some fun parsing
        if self.role == time_stamp:
            if all(cell.isdigit() for cell in self.data):
                self.data = np.array(self.data, dtype="float")
                self.data = np.array(self.data, dtype="datetime64[s]").tolist()
            else:
                self.data = np.array(self.data, dtype="datetime64[ms]").tolist()

            resolutions = list(
                set(
                    [
                        _infer_resolution(self.data[: len(self.data) // 2]),
                        _infer_resolution(self.data[len(self.data) // 2 :]),
                    ]
                )
            )

            # sort resolutions in ascending order and pick the smallest one
            self.resolution = sorted(
                resolutions, key=lambda res: UNITS.index(res), reverse=True
            )[0]

        if self.role in [numerical, target, join_key, unused_float]:
            # this logic might be too simplistic, but data comes from the engine
            # in a standardized format anyway, so...¯\_(ツ)_/¯
            if any("." in cell for cell in self.data):
                self.data = [float(cell) for cell in self.data]
                self.precision = _infer_precision(self.data)
            elif all(cell.isdigit() for cell in self.data):
                self.data = [int(cell) for cell in self.data]

    # ------------------------------------------------------------

    @property
    def default_cell_template(self):

        if getattr(self, "role", None) in [
            join_key,
            numerical,
            target,
            time_stamp,
            unused_float,
        ]:
            if all(str(cell).isdigit() for cell in self.data):
                return "{:" f">{self.width}" "}"
            return "{:" f" {self.width}.{self.precision}d" "}"

        return super().default_cell_template

    # ------------------------------------------------------------

    @property
    def default_header_template(self):

        if getattr(self, "role", None) in [
            join_key,
            numerical,
            target,
            time_stamp,
            unused_float,
        ]:
            return "{:" f">{self.width}" "}"

        return super().default_header_template


# Ejercicio 1 Proyecto
# Enrique Lopez Serrano
# Importar y definir la fecha y hora.
from datetime import datetime
from datetime import date
import os
import sys

# Definir variables de fecha y hora.
fecha = datetime.now().strftime('%d-%m-%Y')
hora = datetime.now().strftime('%H:%M')

# Bloque principal
temperaturas = []
while True:
    temperatura = int(input("Introduce la temperatura: "))
    if temperatura != 100:
        temperaturas.append(temperatura)
    if temperatura == 100:
        break;

# Bucle
print("Informe de temperaturas del Parque Natural de Doñana: ")
print("-----------------------------------------------------")
print("Fecha:" ,fecha)
print("Hora:" ,hora)
print("Numero de muestras: %d" % len(temperaturas))
print("Temperaturas tomadas:" ,temperaturas)
print("Temperatura maxima:" ,max(temperaturas))
print("Temperatura minima:" ,min(temperaturas))
print("Temperatura media:" ,round(sum(temperaturas)/len(temperaturas),2))
temp_tupla = (fecha,hora,temperaturas)
lista = list(temp_tupla)
file = open("temperaturasELS.txt", "a")
file.write('temperaturas = %s' % lista + '\n')
file.close()


import cv2

url = 'http://devimages.apple.com.edgekey.net/streaming/examples/bipbop_4x3/gear2/prog_index.m3u8'
cap = cv2.VideoCapture(url)
while cap.isOpened():
    success, frame = cap.read()
    if not success:
        break
    cv2.namedWindow('a', 0)
    cv2.resizeWindow('a', 640, 360)
    cv2.imshow('a', frame)
    if cv2.waitKey(1) in [ord('q'), 27]:
        break


from network import *
from autograd import Tensor
from ops import cross_entropy
from datetime import datetime
import random

# Example of building a LeNet

# ref: https://d2l.ai/chapter_convolutional-neural-networks/lenet.html

# pip install python-mnist

def one_hot(y, n_classes):
    return np.eye(n_classes)[y]

lenet = Sequential(
    LazyConv2d(6, kernel_size=5, padding=2), Sigmoid(),
    AvgPool2d(kernel_size=2, stride=2),
    LazyConv2d(16, kernel_size=5), Sigmoid(),
    AvgPool2d(kernel_size=2, stride=2),
    Flatten(),
    LazyLinear(120), Sigmoid(),
    LazyLinear(84), Sigmoid(),
    LazyLinear(10))

from mnist import MNIST

# download MNIST and unzip it in the data directory
# http://yann.lecun.com/exdb/mnist/
mndata = MNIST('data')
train_images, train_labels = mndata.load_training()

def gradient_descent(x, grad):
    learning_rate = 0.1
    return x - learning_rate * grad

mini_batch = 128
epochs = 5

for epoch in range(epochs):
    print(f'epoch: {epoch}, {datetime.now()}')
    for i in range(0, 59904, mini_batch):
        x = Tensor(np.array(train_images[i:i+mini_batch]).reshape(mini_batch, 1, 28, 28) / 255.0)
        y = Tensor(one_hot(train_labels[i:i+mini_batch], 10))
        y_pred = lenet(x)
        loss = cross_entropy(y_pred, y)
        loss.backward()
        lenet.update_gradient(gradient_descent)
        print(f'{datetime.now()} mini_batch: {i:05}: {loss.ndarray}')

def predict(image):
    print(mndata.display(image))
    x = Tensor(np.array(image).reshape(1, 1, 28, 28) / 255.0)
    y_pred_probabilities = lenet(x)
    result = np.argmax(y_pred_probabilities.ndarray, axis=1)[0]
    print(f'prediction: {result}')
    return result

test_images, test_labels = mndata.load_testing()

print('Examples of prediction')
for i in range(10):
    choice = random.randrange(0, 10000)
    predict(test_images[choice])

print('Validating...(takes around 2 minutes)')
# validate on test set
x = Tensor(np.array(test_images).reshape(10000, 1, 28, 28) / 255.0)
y_pred_probabilities = lenet(x)
predictions = np.argmax(y_pred_probabilities.ndarray, axis=1)

true_values = np.array(test_labels)
N = true_values.shape[0]
accuracy = (true_values == predictions).sum() / N
print(f'validation accuracy: {accuracy}')

print('Examples of wrong prediction')
wrong_predictions = np.where(true_values != predictions)[0]
for i in range(10):
    choice = random.randrange(0, len(wrong_predictions))
    image_index = wrong_predictions[choice]
    predict(test_images[image_index])
    print(f'Actual: {test_labels[image_index]}')