text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Apr  1 11:44:46 2022

@author: nikolaibeckjensen
"""

import pandas as pd
import numpy as np
import geopandas as gpd
from geopy import distance
from shapely.geometry import Point, LineString
from pyproj import Geod

data_path = r'/Users/nikolaibeckjensen/Desktop/Danmarks Tekniske Universitet/LiRA Project - Dokumenter/data/interim/map_matched_data_GM_7448.pickle'
df = pd.read_pickle(data_path)
df = df.loc[df.street_name=='Tagensvej']

def avg_dist(points, centroid):
    """Measures the average distance from a set of points to a given centroid
    Points: list of coordinate tuples
    Centroid: tuple with lat long coordinates
    """
    dist = [distance.geodesic(point,centroid).km for point in points]

    return np.mean(dist)

def get_speed_limit(df):
    """
    Parameters
    ----------
    df : DataFrame
        Dataframe containing at least 'lon_map', 'lat_map' and 'street_name' where street_name is limited to a single section
    Returns
    -------
    The speed limit for the given street_name in df
    """

    # Load external data,
    path = r'/Users/nikolaibeckjensen/Desktop/Danmarks Tekniske Universitet/LiRA Project - Dokumenter/data/external/speed_limits/trafikhastigheder.shp'
    speed_data = gpd.read_file(path)
    speed_data = speed_data.loc[speed_data.geometry != None]
    speed_data = speed_data.loc[speed_data.hastigheds != 1000]

    speed_data['centroid'] = speed_data.apply(lambda x: x['geometry'].centroid.coords[0], axis=1)

    # save street name of the given df
    assert len(df.street_name.unique()) == 1
    street_name = df.street_name.unique()[0]
    print(street_name)

    # Connect coordinate set for each row in df to a route
    df['points'] = df.apply(lambda x: (x['lon_map'], x['lat_map']), axis=1)
    if len(df) == 1:
        print("Not enough coordinates")
        return 0
    gps_route = LineString([Point(point) for point in df.points])

    # Check for direct match on street segment
    matches = speed_data.loc[speed_data.vejnavn == street_name]
    if len(matches) == 1:
        return matches.hastigheds.item()
    # if multiple select closest
    elif len(matches) > 1:
        matches['dist'] = matches.apply(lambda x: avg_dist(df['points'].to_list(), x['centroid']), axis=1)
        closest_match = matches.loc[matches.dist == matches.dist.min()]
        return closest_match.hastigheds.item()
    # else check if any of the linestrings intersect with current gpsroute
    else:
        speed_data['intersect'] = speed_data.apply(lambda x: x['geometry'].intersects(gps_route),axis=1)
        intersections = speed_data.loc[speed_data.intersect == True]

        if len(intersections) > 1:
            result = intersections.hastigheds.mean()
        elif len(intersections) == 1:
            result = intersections.hastigheds.item()
        else:
            result = 0
        return result

speed_dict = {}
for streetname in df.street_name.unique():
    data = df.loc[df.street_name==streetname]
    speed_dict[streetname] = get_speed_limit(data)

def pedestrian_helper_centroid(lat, lon, pedestrian, margin=10):
    pedestrian['count'] = pedestrian.apply(lambda x: x['id'] if distance.geodesic(x['centroid'], (lat, lon)).m <= margin else '', axis=1)
    return pedestrian['count'].unique()[-1]

def count_pedestrian_crossings(df):
    path = r'/Users/nikolaibeckjensen/Desktop/Danmarks Tekniske Universitet/LiRA Project - Dokumenter/data/external/pedestrian'
    pedestrian = pd.read_pickle(path)
    df['crosswalk'] = df.apply(lambda x: pedestrian_helper_centroid(x['lat_map'], x['lon_map'], pedestrian), axis=1)
    return df.crosswalk.nunique() - 1

def bump_helper_centroid(lat, lon, bumps, margin=10):
    bumps['count'] = bumps.apply(lambda x: x['id'] if distance.geodesic(x['centroid'], (lat, lon)).m <= margin else '', axis=1)
    return bumps['count'].unique()[-1]

def count_speed_bumps(df):
    path = r'/Users/nikolaibeckjensen/Desktop/Danmarks Tekniske Universitet/LiRA Project - Dokumenter/data/external/speedbumps'
    bumps = pd.read_pickle(path)
    df['bumps'] = df.apply(lambda x: bump_helper_centroid(x['lat_map'], x['lon_map'], bumps), axis=1)
    return df.bumps.nunique() - 1

    # ---- Approach 2: Search closest ------ #

ext: py

reason: import numpy


text:
from __future__ import division
import matplotlib.pyplot as plt
from sqlalchemy import *
import numpy as np
from sklearn.linear_model import LogisticRegression
from sqlalchemy.orm import sessionmaker
from sqlalchemy import func
from sklearn.metrics import confusion_matrix
from sklearn.cross_validation import train_test_split
from churndata import *
from sklearn.preprocessing import StandardScaler
from pandas import DataFrame
from pandas.core.groupby import GroupBy
from util import query_to_df
import pandas as pd

from util import campaign_to_num,event_to_num,transform_column,hist_and_show,vectorize,to_percentage,num_rows,vectorize_label,meal_to_num,to_milliseconds
db = create_engine('sqlite:///forjar.db')

metadata = MetaData(db)

Session = sessionmaker(bind=db)

session = Session()

q = session.query(Event).join(Meal,Event.Meal_Id == Meal.id).join(Users).add_entity(Meal).add_entity(Users).filter(Event.Type == 'bought')

df = query_to_df(session,q)

def group_agg(group):
    return group.tolist()

"""
Most of life time value is here.

We need to calculate the number of months each user stays
"""

df_user = df[['Event_date']].groupby([df.Users_date.map(lambda x: (x.year,x.month)),df.Users_id]).aggregate(np.count_nonzero)

"""
Resets the data frame and unpacks groupings
"""

df = df_user.reset_index()
df = df.groupby(df.Users_date).aggregate(group_agg)
print df


text:
from django.shortcuts import render
from django.http.response import JsonResponse
from django.http import HttpResponse
from django.views.decorators.csrf import ensure_csrf_cookie

import json, base64
import sys

import numpy as np
import cv2

sys.path.append("Swin-Transformer-TF")
sys.path.append("task/view.utils")
from utils import *

def Index(request):
    return render(request, 'task/index.html')

def Res(request):
    data = request.body.decode('utf-8')
    try:
        jsondata = json.loads(data)
        task = jsondata['task']

        if task == "taskB":
            # taskB predict
            image_base64 = jsondata['image']
            encoded_data = image_base64.split(',')[1]
            nparr = np.fromstring(base64.b64decode(encoded_data), np.uint8)
            img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)

            model = load_model("task/ml_models/swintransformerB.pkl")
            pred = model.model.predict(cv2.resize(img/255., (224, 224)).reshape(1, 224, 224, 3))[0][0]

            img = cv2.cvtColor(img, cv2.COLOR_RGB2RGBA)
            result, dst_data = cv2.imencode('.png', img)
            dst_base64 = base64.b64encode(dst_data).decode()
            pred = str(pred)

            response = JsonResponse({
                "image": "data:image/png;base64,"+dst_base64,
                "predict": pred,
                "task" : "taskB"
                })

        else:
            # taskA predict
            address = jsondata['address']
            token_id = jsondata['token_id']
            model = load_model('task/ml_models/swintransformerA.pkl')
            img = get_img(address, token_id)
            pred = model.model.predict(cv2.resize(img/255., (224, 224)).reshape(1, 224, 224, 3))[0][0]

            result, dst_data = cv2.imencode('.png', img)
            dst_base64 = base64.b64encode(dst_data).decode()
            pred = str(pred)

            response = JsonResponse({
                "image": "data:image/png;base64,"+dst_base64,
                "predict": pred,
                "task": "taskA"
                })

        return response

    except json.JSONDecodeError:
        response = JsonResponse({
                "image": None,
                "predict": "Please enter correctly",
                "task": "taskA"
                })
        return response

max_stars_repo_name: nft-appraiser/nft-appraiser-api

max_issues_repo_name: nft-appraiser/nft-appraiser-api

max_forks_repo_name: nft-appraiser/nft-appraiser-api


text:
import flow
from flow.module import Module, Linear
from flow.optim import SGD
from flow import function as F
import numpy as np

class Net(Module):
    def __init__(self):
        super().__init__()
        self.fc1 = Linear(2, 10)
        self.fc2 = Linear(10, 1)

    def forward(self, a):
        x = self.fc1(a)
        y = F.relu(x)
        z = self.fc2(y)
        return z

# y = 3x_1^2 + 2x_2
model = Net()
optim = SGD(model.parameters(), lr = 0.005)
for i in range(100):
    input = flow.randn((1, 2))
    output = model(input)
    target = 3 * input[0, 0] * input[0, 0] + 2 * input[0, 1]
    loss = F.square_loss(output, target)
    loss.backward()
    optim.step()
    optim.zero_grad()
    print("loss", loss)

max_stars_repo_name: Cjkkkk/Pyflow

max_issues_repo_name: Cjkkkk/Pyflow

max_forks_repo_name: Cjkkkk/Pyflow


text:
# local imports
from csv_io import *

# standard imports
import numpy
import os
from datetime import datetime, timedelta
from calendar import monthrange, isleap
import matplotlib.pyplot as plt

__author__ = ["Jeffry Ely, Jeff.ely.08@gmail.com"]

class time_series:
    """
    A subsetable and discretizable time series object

    The primary motivation for creating this object was to allow
    a time series to be discretized into any number of small chunks
    but retain the ability to process and interrogate the time series
    at any level with the exact same external syntax.

    A time series object is comprised of a matrix of data, and may contain
    an object list of subset time_series objects. Potentially unlimited
    nesting of time series datasets is possible, for example: a years worth
    of hourly data may be discretized into 1-month time series, while each    of those is in turn discretized into days. The highest level time series
    will still allow opperations to be performed upon it.

    All internal methods are built to handle this flexible definition of a
    time series, where the steps of the method depend on weather the time
    series is at its smallest subset or not.

    (it isn't quite finished yet)

    MEMORY WARNING. The entirety of the dataset is represented in every layer
    of discretization, so watch out for exploding memory consumption by
    excessive subsetting.

    PLANNED IMPROVEMENTS:
    this needs to be expanded to allow the "data" attribute to be lists of
    filepaths. this will allow advanced time series operations to be performed
    on raster datasets.

    this should allow far more elegant replacement of the follwing funtions
    within the dnppy module:

    "files_in_window"
    "rolling_window"
    "rolling_stats"
    "group_stats"
    """

    def __init__(self, name = "name", units = None, discretized = False, disc_level = 0, parent = None):
        """ initializes the time series """

        self.name           = name          # the name of this time series
        self.units          = units         # unit of time represented by subset if discretized
        self.discretized    = discretized   # does this time series have subsets?
        self.disc_level     = disc_level    # the subset level of this time_series

        self.rasterpaths    = False         # becomes true if time series is of raster data

        self.fmt            = False         # for interpreting timestrings to time objs
        self.headers        = []            # one header for each col in dataset

        self.time_col       = 0             # index of data column with time info
        self.time           = []            # separate copy of data[time_col]
        self.time_dom       = False         # self.time converted to list of datetime objs
        self.time_dec_days  = []            # self.time converted to mono rising decimal days
        self.time_seconds   = []            # self.time converted to mono rising seconds
        self.center_time    = []            # time around which data in a subset it centered
        self.start_dto      = []            # datetime_object that mono rising times start from

        self.subsets        = []            # object list containing constituent time_seires

        self.row_data       = []            # row wise dataset
        self.col_data       = []            # column wise dataset, built as dict

        self.bad_rows       = []            # subset from data attribute with "bad rows"

        self.infilepath     = []            # tracks filepath of input CSV. used to DISALLOW overwriting
                                            # source CSV with output CSV.

        # run some methods to build subset attributes
        if parent:
            self._get_atts_from(parent)

        return

    def __getitem__(self, arg):
        """
        allows subsets of the time series to be accessed as follows, for example:

        Time series whos first row is:
        ['20010101', '045316', '1', '20010101045316', 366]

        With no discretezation layers
        command       : result
        ts[0]         : ['20010101', '045316', '1', '20010101045316', 366]
        ts[0][0]      : 20010101

        with one discretezation layer
        ts[0]         : <__main__.time_series instance at 0x00000000023CEB48>
        ts[0][0]      : ['20010101', '045316', '1', '20010101045316', 366]
        ts[0][0][0]   : 20010101

        with two discretezation layers
        ts[0]         : <__main__.time_series instance at 0x00000000023CEB48>
        ts[0][0]      : <__main__.time_series instance at 0x00000000023CCA48>
        ts[0][0][0]   : ['20010101', '045316', '1', '20010101045316', 366]
        ts[0][0][0][0]: 20010101
        """

        # allows finding a slice of subsets or rows from a subset.
        if isinstance(arg, slice):
            a = arg.start
            b = arg.stop

            if self.discretized:
                if b > len(self.subsets): b = len(self.subsets)
                return [self.subsets[x] for x in range(a,b)]

            else:
                if b > len(self.row_data): b = len(self.row_data)
                return [self.row_data[x] for x in range(a,b)]

        # allows finding subsets or rows by index.
        elif isinstance(arg, int):
            if self.discretized:
                return self.subsets[arg]

            else:
                return list(self.row_data[arg])

        # allows finding subsets by name.
        elif isinstance(arg, str):
            if self.discretized:
                for subset in self.subsets:
                    if subset.name == arg:
                        return subset

                else:   raise Exception("no subset with name {0}".format(arg))
            else:       raise Exception("String input is allowed for discretized time_series only!")
        else:           raise Exception("Unrecognized argument type! use int, slice, or string!")

    def _get_atts_from(self, parent_time_series):
        """
        Allows bulk setting of attributes. usefull for allowing a subset to inherit
        information from its parent time_series

        internal use only
        """

        self.fmt            = parent_time_series.fmt
        self.headers        = parent_time_series.headers
        self.time_col       = parent_time_series.time_col
        self.time_header    = parent_time_series.time_header
        self.disc_level     = parent_time_series.disc_level + 1
        return

    def _fmt_to_units(self, fmt):
        """ converts fmt strings to unit names of associated datetime attributes """

        fmtlist =  ["%Y", "%m", "%b", "%d", "%j", "%H", "%M", "%S"]
        unitlist = ["year","month","month","day","day","hour","minute","second"]

        if fmt in fmtlist:
            unit = unitlist[fmtlist.index(fmt)]
            return unit

        if fmt in unitlist:
            return fmt
        else:
            raise Exception("'{0}' is an invalid unit or format!".format(fmt))

    def _units_to_fmt(self, units):
        """ converts unit names to fmt strings used by datetime.stftime. internal use"""

        fmtlist =  ["%Y", "%b", "%m", "%j", "%d", "%H", "%M", "%S"]
        unitlist = ["year","month","month","day","day","hour","minute","second"]

        if units in unitlist:
            fmt = fmtlist[unitlist.index(units)]
            return fmt

        if units == "%b":
            return "%m"

        if units in fmtlist:
            return units
        else:
            raise Exception("'{0}' is an invalid unit or format!".format(fmt))

    def _extract_time(self, time_header):
        """  special case of "extract_column" method for time domain. internal use """

        self.time_header = time_header
        self.build_col_data()

        if time_header in self.headers:
                self.time_col   = self.headers.index(time_header)
                self.time       = self.col_data[time_header]
        else:
            raise LookupError("Time header not in dataset!")

        if self.discretized:
            for subset in self.subsets:
                subset._extract_time(time_header)

        return self.time

    def _center_datetime(self, datetime_obj, units):
        """ returns datetime obj that is centered on the "unit" of the input datetime obj"""

        dto = datetime_obj

        if units == "day":
            return datetime(dto.year, dto.month, dto.day, 12)

        if units == "month":
            # we need to find the middle day of the month which could be 14 or 15
            middle = (monthrange(dto.year, dto.month)[1]) / 2

            return datetime(dto.year, dto.month, middle, 0)

        if units == "year":
            return datetime(dto.year, 7, 2, 0)

    def _units_to_seconds(self, units, dto = None):
        """ converts other time units to seconds. internal use only"""

        # ensure proper unit formatting
        units = self._fmt_to_units(units)

        if units == "second":   return 1.0
        if units == "minute":   return 60.0
        if units == "hour":     return 60.0 * 60.0
        if units == "day":      return 60.0 * 60.0 * 24.0

        if units == "month":
            if dto:
                return 60.0 * 60.0 * 24.0 * monthrange(dto.year, dto.month)[1]
            else:
                return 60.0 * 60.0 * 24.0 * (365.25/12)

        if units == "year":
            if dto:
                return 60.0 * 60.0 * 24.0 * (365 + isleap(dto.year) *1)
            else:
                return 60.0 * 60.0 * 24.0 * 365.25

    def _seconds_to_units(self, seconds, units):
        """ converts seconds to other time units. internal use only"""

        if units == "second":   return 1.0
        if units == "minute":   return seconds / (60.0)
        if units == "hour":     return seconds / (60.0 * 60.0)
        if units == "day":      return seconds / (60.0 * 60.0 * 24.0)
        if units == "month":    return seconds / (60.0 * 60.0 * 24.0 * (365.25 / 12.0))
        if units == "year":     return seconds / (60.0 * 60.0 * 24.0 * 365.25)

    def _name_as_subset(self, binned = False):
        """ uses time series object to descriptively name itself. internal use only"""

        subset_units = self.units

        if isinstance(self.center_time, datetime):
            datetime_obj = self.center_time
        else:
            datetime_obj = self.time_dom[0]

        if binned:
            self.name = datetime_obj.strftime(self._units_to_fmt(self.units))

        else:
            if subset_units == "year" or subset_units == "%Y":
                self.name = datetime_obj.strftime("%Y")

            if subset_units == "month" or subset_units == "%m":
                self.name = datetime_obj.strftime("%Y-%b")

            if subset_units == "day" or subset_units == "%d":
                self.name = datetime_obj.strftime("%Y-%m-%d")

            if subset_units == "hour" or subset_units == "%H":
                self.name = datetime_obj.strftime("%Y-%m-%d-%H")

            if subset_units == "minute" or subset_units == "%M":
                self.name = datetime_obj.strftime("%Y-%m-%d-%H:%M")

            if subset_units == "second" or subset_units == "%S":
                self.name = datetime_obj.strftime("%Y-%m-%d-%H:%M:%S")

        return

    def from_csv(self, csv_path, has_headers = True, delim = ',', spec_format = False):
        """
        creates the time_series data from a csv.

        This simply wraps the read_csv_rows function in csv_io module.
        """

        self.infilepath = os.path.abspath(csv_path)

        if not self.discretized:
            self.row_data, self.headers = read_csv_rows(csv_path, has_headers, delim, spec_format)
            self.build_col_data()

        else:
            raise Warning("You may not import new data into \
                          an already discretized time series")
        return

    def to_csv(self, csv_path):
        """ simply write the data to a csv file"""

        # disallow overwriting the csv used as input. Added by request
        if os.path.abspath(self.infilepath) == os.path.abspath(csv_path):
            csv_path = csv_path.replace(".csv", "_out.csv")

        print("Saved time series '{0}' with {1} rows and {2} columns".format(
                                    self.name, len(self.row_data), len(self.col_data)))

        write_csv_rows(self.row_data, self.headers, csv_path)
        return

    def from_list(self, data, headers, time_header, fmt):
        """ creates the time series data from a list"""

        self.row_data = data
        self.headers  = headers

        # populates self.time with time series
        self.define_time(self.time_header, self.fmt)
        self.build_col_data()
        return

    def build_col_data(self):
        """ builds columnwise data matrix with an actual dict """

        temp_col = zip(*self.row_data)

        self.col_data = {}
        for i,col in enumerate(temp_col):
            self.col_data[self.headers[i]] = list(col)
        return

    def clean(self, col_header):
        """ Removes rows where the specified column has an invalid number"""

        # loop cleaning for multiple column header inputs
        if isinstance(col_header, list):
            for col_head in col_header:
                self.clean(col_head)

        # clean for just one input column header
        else:
            if not col_header in self.headers:
                raise LookupError("{0} header not in dataset!".format(col_header))

            col_index = self.headers.index(col_header)
            temp_data = self.row_data
            self.row_data = []

            bad_count = 0
            for row in temp_data:

                try:
                    test = float(row[col_index])
                    self.row_data.append(row)
                except:
                    bad_count += 1
                    self.bad_rows.append(row)
                    continue

            if bad_count >0:
                print("Removed {0} rows with invalid '{1}'".format(bad_count,col_header))
                print("Dataset now has {0} rows".format(len(self.row_data)))

            # since rows have been removed, we must redefine the time domain. (sloppy, but concise)
            self.define_time(self.time_header, self.fmt, self.start_dto)

            if self.discretized:
                for subset in self.subsets:
                    subset.clean(col_header)
        return

    def add_row(self, row):
        """ simply adds a row to the time_series data"""

        self.row_data.append(list(row))
        return

    def rebuild(self, destroy_subsets = False):
        """ reconstructs the time series from its constituent subsets"""

        # handles time series with multiple levels of discretezation
        while self.subsets[0].discretized:
            for subset in self.subsets:
                subset.rebuild(destroy_subsets)

        else:
            if self.discretized:
                self.row_data = []
                for subset in self.subsets:
                    for row in subset.row_data:
                        self.row_data.append(row)

                self.define_time(self.time_header, self.fmt)

        if destroy_subsets:
            self.subsets = []
            self.discretized = False

        return

    def merge_cols(self, header1, header2):
        """merges two columns together (string concatenation)"""

        new_header  = "_".join([header1, header2])

        for i, entry in enumerate(self.row_data):
                new_field   = "".join([self.col_data[header1][i], self.col_data[header2][i]])

                self.row_data[i].append(new_field)

        # updates column and row data
        self.headers.append(new_header)
        self.build_col_data()

        print("merged '{0}' and '{1}' columns into new column '{2}'".format(header1, header2, new_header))

        if self.discretized:
            for subset in self.subsets:
                subset.merge_cols(header1, header2)

        return

    def define_time(self, time_header, fmt, start_date = False):
        """
        Converts time strings into time objects for standardized processing

        for tips on how to use 'fmt' variable, see url below:
        https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior

        time header variable can be either the header string, or a column index num

        Creates two things:
            a converted list of time objects
            a new list of monotonically increasing decimal days
        """

        # allows column index integers as time_header inputs
        if isinstance(time_header, int):
            time_header = self.headers[time_header]

        # populates self.time with time series
        self._extract_time(time_header)

        # use manual start date (str or dto) or set to begining of first day on record
        if isinstance(start_date, str):
            start = datetime.strptime(start_date, fmt)

        elif isinstance(start_date, datetime):
            start = start_date

        else:
            earliest = datetime.strptime(self.time[0], fmt)
            start = datetime(earliest.year, earliest.month, earliest.day, 0,0,0,0)

        self.start_dto = start

        datestamp_list      = self.time
        self.fmt            = fmt

        self.time_dom       = []
        self.time_dec_days  = []
        self.time_seconds   = []

        for i,datestamp in enumerate(datestamp_list):

            # If error, give user information about the line on which the error occurs
            try:
                t = datetime.strptime(datestamp, fmt)
            except:
                raise Exception("Input '{0}' in line {1} is not of format {2}".format(
                                datestamp, i+2 , fmt))

            self.time_dom.append(t)

            delta = t - start
            delta = float(delta.total_seconds())
            self.time_seconds.append(float(delta))
            self.time_dec_days.append(float(delta / 86400))

        # If the data was not already in ascending time order, fix it here #unfinished

        if self.discretized:
            for subset in self.subsets:
                subset.define_time(time_header, fmt)

        return

    def sort_ascending(self):
        """ reorders the data in this time_series to be in ascending order by time"""

        sorted_cols = sorted(zip(*self.col_data))

        for i,header in enumerate(self.headers): pass

        return

    def discretize(self, subset_units, overlap_width = 0, cust_center_time = False):
        """
        splits the time series into individual time chunks

        used for taking advanced statistics, usually periodic
        in nature. Also usefull for exploiting temporal relationships
        in a dataset for a variety of purposes including data
        sanitation, periodic curve fitting, etc.

        fmt_units follows convention of fmt. For example:
        %Y  groups files by year
        %m  groups files by month
        %j  groups file by julian day of year

        the "overlap_width" variable can be set to greater than 0
        to allow "window" type statistics, so each subset may contain
        data points from adjacent subsets.

        overlap_width = 1 is like a window size of 3
        overlap_width = 2 is like a window size of 5

        WARNING: this function is imperfect for discretizing by months.
        the possible lengths of a month are extremely variant, so sometimes
        data points at the ends of a month will get placed in the adjacent
        month. If you absolutely require accurate month discretization,
        you should use this function to discretize by year, then use the
        "group_bins" function to bin each year by month. so,

        instead of
            ts.discretize("%Y")
            ts.discretize("%b")

        use
            ts.discretize("%Y")
            ts.group_bins("%b")

        Allows a custom center time to be used! This was added so that
        days could be centered around a specific daily aquisition time.
        for example, its often usefull to define a day as
        satellite data aquisition time +/- 12 hours.
        if used, "cust_center_time" must be a datetime object!
        """

        if self.discretized:

            for subset in self.subsets:
                subset.discretize(subset_units, overlap_width)

        else:

            self.discretized = True

            if overlap_width < 0:
                print("overlap_width must be 0 or more, setting it to 0!")
                overlap_width = 0

            # convert units into subset units and into terms of seconds
            # timedelta objecs can only have units of days or seconds, so we use seconds
            subset_units = self._fmt_to_units(subset_units)

            # initial step width
            step_width = self._units_to_seconds(subset_units)

            if subset_units not in ['year','month','day']:
               raise Exception("Data is too high resolution! try the 'discretize()' fn")

            print("Discretizing data by {0}".format(subset_units))

            if self.time_dom == False:
                raise Exception("must call 'define_time' method before discretization!")

            # determine subset lists starting, end points and incriment
            time_s = self.time_dom[0]
            time_f = self.time_dom[-1]

            # set up starttime with custom center times.
            if cust_center_time:
                if subset_units == "month":
                    ustart  = datetime(time_s.year, time_s.month,
                                       cust_center_time.day, cust_center_time.hour,
                                       cust_center_time.minute, cust_center_time.second,
                                       cust_center_time.microsecond)
                elif subset_units == "hour":
                    ustart  = datetime(time_s.year, time_s.month, time_s.day, time_s.hour,
                                       cust_center_time.minute, cust_center_time.second,
                                       cust_center_time.microsecond)
                elif subset_units == "minute":
                    ustart  = datetime(time_s.year, time_s.month, time_s.day, time_s.hour, time_s.minute,
                                       cust_center_time.second, cust_center_time.microsecond)
                else: #subset_units == "day":
                    ustart  = datetime(time_s.year, time_s.month, time_s.day,
                                       cust_center_time.hour, cust_center_time.minute,
                                       cust_center_time.second, cust_center_time.microsecond)

                td      = time_f - time_s
                uend    = cust_center_time + timedelta(seconds = td.total_seconds())

            # otherwise, set the centers with no offest.
            else:
                ustart  = self._center_datetime(time_s, subset_units)
                uend    = self._center_datetime(time_f, subset_units) + timedelta(seconds = step_width)

            delta = uend - ustart

            # Iterate through entire dataset one time step unit at a time.
            center_time = ustart
            while center_time < uend:

                step_width   = self._units_to_seconds(subset_units, center_time)
                wind_seconds = step_width * (overlap_width + 0.5)

                temp_data = []
                for j, current_time in enumerate(self.time_dom):
                    dt = abs(center_time - current_time)

                    if dt.total_seconds() < wind_seconds:
                        temp_data.append(self.row_data[j])

                # create the subset only if some data was found to populate it
                if len(temp_data) > 0:
                    new_subset = time_series(units = subset_units, parent = self)
                    new_subset.center_time = center_time
                    new_subset.from_list(temp_data, self.headers, self.time_header, self.fmt)
                    new_subset.define_time(self.time_header, self.fmt)
                    new_subset._name_as_subset()
                    self.subsets.append(new_subset)

                center_time += timedelta(seconds = step_width)

        return

    def group_bins(self, fmt_units, overlap_width = 0, cyclical = True):
        """
        sorts the time series into time chunks by common bin_unit

        used for grouping data rows together. For example, if one used
        this function on a 5 year dataset with a bin_unit of month,
        then the time_series would be subseted into 12 sets (1 for each
        month), which each set containing all entries for that month,
        regardless of what year they occurred in.

        fmt_units follows convention of fmt. For example:
        %Y  groups files by year
        %m  groups files by month
        %j  groups file by julian day of year

        similarly to "discretize" the "overlap_width" variable can be
        set to greater than 1 to allow "window" type statistics, so
        each subset may contain data points from adjacent subsets.
        However, for group_bins, overlap_width must be an integer.

        "cyclical" of "True" will allow end points to be considered adjacent.
        So, for example, January will be considered adjacent to December,
        day 1 will be considered adjacent to day 365.
        """

        ow = int(overlap_width)

        if self.discretized:
            for subset in self.subsets:
                subset.group_bins(fmt_units, overlap_width, cyclical)

        else:

            self.discretized = True

            # ensure proper unit format is present
            fmt     = self._units_to_fmt(fmt_units)
            units   = self._fmt_to_units(fmt_units)

            # set up cyclical parameters
            if fmt == "%j": cylen = 365
            if fmt == "%d": cylen = 365
            if fmt == "%m": cylen = 12
            if fmt == "%b": cylen = 12

            # initialize a grouping array to idenfity row indices for each subset
            grouping  = [int(obj.strftime(fmt)) for obj in self.time_dom]

            for i in xrange(min(grouping),max(grouping) + 1):

                subset_units    = self._fmt_to_units(fmt)
                new_subset      = time_series( units = subset_units, parent = self)

                # only take rows whos grouping is within ow of i
                subset_rows = [j for j,g in enumerate(grouping) if g <= i+ow and g >=i-ow]

                # fix endpoints
                if cyclical:
                    if i <= ow:
                        subset_rows = subset_rows + [j for j,g in enumerate(grouping) if g-cylen <= i+ow and g-cylen >=i-ow]
                    elif i >= cylen - ow:
                        subset_rows = subset_rows + [j for j,g in enumerate(grouping) if g+cylen <= i+ow and g+cylen >=i-ow]

                # grab row indeces from parent matrix to put in the subset
                subset_data = [self.row_data[row] for row in subset_rows]

                # run naming methods and definitions on the new subset
                if not len(subset_data) == 0:
                    new_subset.from_list(subset_data, self.headers, self.time_header, self.fmt)
                    new_subset.define_time(self.time_header, self.fmt)

                    new_subset.center_time = self.time_dom[grouping.index(i)]
                    new_subset._name_as_subset(binned = True)

                    self.subsets.append(new_subset)
        return

    def column_stats(self, col_header):
        """
        takes statistics on a specific column of data

        creates object attributes according to the column name
        """

        print("calculated stats for time_series '{0}', col '{1}'".format(self.name,col_header))

        # pull column data and find some stats
        import numpy as np

        self.clean(col_header)
        col_data = map(float, self.col_data[col_header])

        # build array of stats
        stats = [max(col_data),
                 min(col_data),
                 col_data.index(max(col_data)),
                 col_data.index(min(col_data)),
                 np.mean(col_data),
                 np.std(col_data)]

        # build array of names
        names = ["{0}_max_v".format(col_header),
                 "{0}_min_v".format(col_header),
                 "{0}_max_i".format(col_header),
                 "{0}_min_i".format(col_header),
                 "{0}_avg".format(col_header),
                 "{0}_std".format(col_header)]

        # set attributes for names and stats, also make dict for immediate return
        statistics = {}
        for i,stat in enumerate(stats):
            setattr(self, names[i], stats[i])
            statistics[names[i]] = stats[i]

        if self.discretized:
            for subset in self.subsets:
                subset.column_stats(col_header)

        return statistics

    def column_plot(self, col_header):
        """ plots a specific column """

        # clean columnar data of non-numeric elements
        self.clean(col_header)

        return

    def normalize(self, col_header):
        """ used to normalize specific columns in the time series"""

        # make sure data is cleaned for numerical formatting
        self.clean(col_header)
        temp_col = self.col_data[col_header]
        temp_col = map(float, temp_col)

        # perform normalization
        minval   = min(temp_col)
        maxval   = max(temp_col)

        c = self.headers.index(col_header)
        for i,row in enumerate(self.row_data):
            self.row_data[i][c] = (float(row[c]) - minval) / (maxval - minval)

        print("data in column '{0}' has been normalized!".format(col_header))

        if self.discretized:
            for subset in self.subsets:
                subset.normalize(col_header)
        return

    def add_mono_time(self):
        """Adds a monotonically increasing time column with units of decimal days"""

        # add an entry to every row item in row_data, then rebuild column data
        if "decimal_days" not in self.headers:
            self.headers.append("decimal_days")

            for i, row in enumerate(self.row_data):
                self.row_data[i].append(self.time_dec_days[i])

        self.build_col_data()

        if self.discretized:
            for subset in self.subsets:
                subset.add_mono_time()

        return

    def interp_col(self, time_obj, col_header):
        """
        for input column, interpolate values to estimate value at input time_obj.
        input time_obj may also be of datestring matching decared fmt
        """

        # start by cleaning data by input column
        self.clean(col_header)

        # x and y data for interpolation
        y = self.col_data[col_header]
        x = self.time_seconds

        if not isinstance(time_obj, datetime):
            time_obj = datetime.strptime(time_obj, self.fmt)

        delta = time_obj - self.start_dto

        interp_x = delta.total_seconds()
        interp_y = numpy.interp(interp_x, x, y)

        print("Val in '{0}' at time '{1}' is '{2}'".format(col_header, time_obj, interp_y))

        return interp_y

    def interogate(self):
        """ prints a heads up stats table of all subsets in this time_series """

        if self.disc_level == 0:
            print("")
            print("="*84)
            print("time_series name \t\t len \t start \t\t\t end")
            print("="*84)

        # use leading and trailing spaces for visualizing discretization depth
        padded_name = self.disc_level * "   " + self.name
        print("{0} \t {1} \t {2} \t {3}".format(
            padded_name.ljust(28, " "),
            str(len(self.time)).ljust(5," "),
            self.time_dom[0],
            self.time_dom[-1]))

        if self.discretized:
            for subset in self.subsets:
                subset.interrogate()
        return

    def from_rastlist(self, filepaths, fmt):
        """ loads up a list of filepaths as a time series """

        self.headers = ['filepaths','filenames']
        self.row_data = []

        for filepath in filepaths:
            head, filename = os.path.split(filepath)
            self.row_data.append([filepath, filename])

        self.build_col_data()
        self.define_time('filenames', fmt)

        # flags this time series as being comprised of rasterpaths
        self.rasterpaths  = True
        return

    def rast_stats(self, outdir, saves = ["AVG","STD","NUM"],
                                        low_thresh = False, high_thresh = False):
        """
        wraper for the "many_stats" function in the raster module

        performs the many_stats function on each of the lowest level
        subsets of this time_series object
        """

        self.outdir = outdir
        self.saves  = saves

        if not self.rasterpaths:
            raise Exception("this method is only for rasterlist time_series!")

        # only at the lowest discretezation level should stats be taken.
        if self.discretized:
            for subset in self.subsets:
                subset.rast_statistics(outdir, saves)

        else:
            raster.many_stats(self.col_data['filepaths'],
                              outdir, self.name, saves, low_thresh, high_thresh)
        return

# testing code
if __name__ == "__main__":

    filepath = r"test_data\two_years_daily_hourly_variation.csv"
    fmt = "%d/%m/%Y%H%M"
    start = "01/01/20000000"

##    filepath = r"test_data\separate_date_time.csv"
##    fmt = "%Y%m%d%H%M%S"
##    start = "20000101000000"
    # testing csv manipulations
    print(" \n\n testing csv manipulations \n\n")

    ts = time_series('Master_TS')
    ts.from_csv(filepath)
    ts.merge_cols("date", "time")
    ts.define_time("date_time", fmt, start)
    ts.add_mono_time()

    #ts.discretize('%Y')
    ts.discretize('%b', overlap_width = 2)
    ts.interogate()

    ts.to_csv(r"test_data\separate_date_time_disc.csv")

    # testing bining
    print(" \n\n testing bin function \n\n")

    ts = time_series('Master_TS')
    ts.from_csv(filepath)
    ts.merge_cols("date", "time")
    ts.define_time("date_time", fmt, start)
    ts.add_mono_time()

    #ts.discretize("%Y")
    ts.group_bins("%b", overlap_width = 2)
    ts.interogate()

    ts.to_csv(r"test_data\separate_date_time_bin.csv")

max_stars_repo_name: NASA-DEVELOP/dnppy

max_issues_repo_name: snowzm/dnppy

max_forks_repo_name: snowzm/dnppy


text:
import RLAgents
import gym
import numpy
import cv2
import torch

from models import model_ae

class EpisodicMemory:
    def __init__(self, size, initial_count = 16):
        self.size               = size
        self.initial_count      = initial_count
        self.episodic_memory    = None

    def reset(self, state_t):
        self.episodic_memory = torch.zeros((self.size , ) + state_t.shape).to(state_t.device)
        for i in range(self.size):
            self.episodic_memory[i] = state_t
        self.count = 0

    def add(self, state_t):
        if self.episodic_memory is None:
            self.reset(state_t)
        else:
            if self.count < self.initial_count:
                n = self.size//self.initial_count
                for i in range(n):
                    idx = numpy.random.randint(self.size)
                    self.episodic_memory[idx] = state_t

                self.count+= 1
            else:
                idx = numpy.random.randint(self.size)
                self.episodic_memory[idx] = state_t

    def entropy(self):
        mean = self.episodic_memory.mean(axis=0)
        diff = (self.episodic_memory - mean)**2
        max_ = diff.max(axis=0)[0]

        result = max_.mean().detach().to("cpu").numpy()

        if self.count < self.initial_count:
            return 0.0
        else:
            return result

def train_batch(model, state_t):
    state_predicted_t, _ = model(state_t)

    loss = (state_t.detach() - state_predicted_t)**2
    loss = loss.mean()

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

def eval_features(model, state_np):
    states_t = torch.from_numpy(state_diff).unsqueeze(0).to(model.device)
    state_predicted_t, z_t = model(states_t, False)

    state_predicted_np = state_predicted_t.squeeze(0).detach().to("cpu").numpy()

    return state_predicted_np, z_t

def show(model, state_np):
    state_diff = state_np - state_mean
    state_predicted_np, z_t = eval_features(model, state_diff)

    episodic_memory.add(z_t)

    print("entropy = ", episodic_memory.entropy())

    height  = state.shape[1]
    width   = state.shape[2]
    height_ = height
    width_  = 3*width
    im      = numpy.zeros((height_, width_))

    im[0:height, 0*width:1*width]  = state_np[0]
    im[0:height, 1*width:2*width]  = state_diff[0]
    im[0:height, 2*width:3*width]  = state_predicted_np[0]

    im = cv2.resize(im, (2*width_, 2*height_), interpolation = cv2.INTER_AREA)

    cv2.imshow("state", im)
    cv2.waitKey(1)

env = gym.make("MsPacmanNoFrameskip-v4")
#env = gym.make("MontezumaRevengeNoFrameskip-v4")
env = RLAgents.WrapperAtari(env)

state = env.reset()

for i in range(10):
    state, _, _, _ = env.step(0)

state_mean = state.copy()

state_shape = env.observation_space.shape
actions_count = env.action_space.n
batch_size  = 16

model = model_ae.Model(state_shape)
model.load("./models/")

episodic_memory = EpisodicMemory(256, 16)

idx = 0
k = 0.0001
while True:
    action = numpy.random.randint(actions_count)

    state, reward, done, info = env.step(action)

    state_mean = (1.0 - k)*state_mean + k*state
    state_diff = state - state_mean

    if done:
        state = env.reset()
        _, z_t = eval_features(model, state_diff)
        episodic_memory.reset(z_t)

    show(model, state)


text:
import numpy as np
from core import Object3D, Uniform, UniformList
from mathutils import MatrixFactory

class Camera(Object3D):

    def __init__(self):
        super().__init__()
        self.projectionMatrix = MatrixFactory.makeIdentity()
        self.viewMatrix = MatrixFactory.makeIdentity()

        self.uniformList = UniformList()
        self.uniformList.addUniform( Uniform("mat4", "projectionMatrix", self.projectionMatrix) )
        self.uniformList.addUniform( Uniform("mat4", "viewMatrix", self.viewMatrix) )

    def getProjectionMatrix(self):
        return self.projectionMatrix

    def updateViewMatrix(self):
        self.viewMatrix = np.linalg.inv(self.transform.matrix)

    def getViewMatrix(self):
        return self.viewMatrix


text:
# -*- coding:utf-8 -*-
import math
import os

import cv2
import keras
import numpy as np

from keras.utils import to_categorical

from utils import preprocess_input_vgg

class DataGenerator(keras.utils.Sequence):

    def __init__(self, x, param, y=None, batch_size=1, shuffle=True, preprocess=preprocess_input_vgg, postfix=None):
        self.batch_size = batch_size
        self.x = x
        self.y = y
        self.indexes = np.arange(len(self.x))
        self.shuffle = shuffle
        self.preprocess = preprocess
        self.batch_size = batch_size
        self.param = param
        if postfix is None:
            self.data_path = self.param.get_conf('data_path')
        else:
            self.data_path = os.path.join(self.param.get_conf('data_path'), postfix)
        self.train_size = param.get_conf('train_image_size')
        self.nb_class = self.param.get_conf('classes_num')

    def __len__(self):
        return math.ceil(len(self.x) / float(self.batch_size))

    def __getitem__(self, index):
        batch_indexs = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]
        batch_datas = [self.x[k] for k in batch_indexs]
        y=None
        if self.y is not None:
            y = [self.y[k] for k in batch_indexs]
        return self.data_generation(batch_datas,y)

    def on_epoch_end(self):
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def data_generation(self, batch_datas, y=None):
        images = []
        labels = []
        for i, data in enumerate(batch_datas):

            img = cv2.imread(os.path.join(self.data_path, data))[:, :, ::-1]
            img = cv2.resize(img, (self.train_size, self.train_size))

            images.append(img)
        if self.y is not None:
            for label in y:
                labels.append(to_categorical(label, self.nb_class))

        images = np.array(images)

        if self.preprocess is not None:
            images = self.preprocess(images)
        # print(images.shape)
        if self.y is None:
            return np.array(images)

        return np.array(images), np.array(labels)

max_stars_repo_name: sturggleQ/AdvDoor

max_issues_repo_name: sturggleQ/AdvDoor

max_forks_repo_name: sturggleQ/AdvDoor


text:
# import the necessary packages
#from skimage.measure import structural_similarity as ssim
from skimage import measure
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import cv2

def mse(imageA, imageB):
	# the 'Mean Squared Error' between the two images is the
	# sum of the squared difference between the two images;
	# NOTE: the two images must have the same dimension
	err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2)
	err /= float(imageA.shape[0] * imageA.shape[1])

	# return the MSE, the lower the error, the more "similar"
	# the two images are
	return err

def compare_images(imageA, imageB, title):
	# compute the mean squared error and structural similarity
	# index for the images
	m = mse(imageA, imageB)
	s = measure.compare_ssim(imageA, imageB)
	print("mse:{}".format(m))
	print("ssim{}".format(s))
	# setup the figure
	fig = plt.figure(title)
	plt.suptitle("MSE: %.2f, SSIM: %.2f" % (m, s))

	# show first image
	ax = fig.add_subplot(1, 2, 1)
	plt.imshow(imageA, cmap = plt.cm.gray)
	plt.axis("off")

	# show the second image
	ax = fig.add_subplot(1, 2, 2)
	plt.imshow(imageB, cmap = plt.cm.gray)
	plt.axis("off")

	# show the images
	plt.show()

# load the images -- the original, the original + contrast,
# and the original + photoshop
original = cv2.imread("Test_result_bottleneck_HistoVAE_V3_withoutflatten_10x_V3_300epoch_MSESSIMMAEloss/Orig_file_16.png")
recons_MSEloss = cv2.imread("Test_result_bottleneck_HistoVAE_V3_withoutflatten_10x_V3_200epoch/Recons_file_16.png")
recons_MSESSIMMAEloss = cv2.imread("Test_result_bottleneck_HistoVAE_V3_withoutflatten_10x_V3_300epoch_MSESSIMMAEloss/Recons_file_16.png")
recons_SSIMloss = cv2.imread("Test_result_bottleneck_HistoVAE_V3_withoutflatten_10x_V3_300epoch_SSIMloss/Recons_file_16.png")

# convert the images to grayscale
original = cv2.cvtColor(original, cv2.COLOR_BGR2GRAY)
recons_MSEloss = cv2.cvtColor(recons_MSEloss, cv2.COLOR_BGR2GRAY)
recons_MSESSIMMAEloss = cv2.cvtColor(recons_MSESSIMMAEloss, cv2.COLOR_BGR2GRAY)
recons_SSIMloss = cv2.cvtColor(recons_SSIMloss, cv2.COLOR_BGR2GRAY)

# initialize the figure
fig = plt.figure("Images")
images = ("Original", original), ("recons_MSEloss", recons_MSEloss), ("recons_MSESSIMMAEloss", recons_MSESSIMMAEloss), ("recons_SSIMloss", recons_SSIMloss)

# loop over the images
for (i, (name, image)) in enumerate(images):
	# show the image
	ax = fig.add_subplot(1, 4, i + 1)
	ax.set_title(name)
	plt.imshow(image, cmap = plt.cm.gray)
	plt.axis("off")

# show the figure
plt.show()
fig.savefig('Compare_reconstruct_Images.png')

# compare the images
compare_images(original, recons_MSEloss, "Original vs. recons_MSEloss")
compare_images(original, recons_MSESSIMMAEloss, "Original vs. recons_MSESSIMMAEloss")
compare_images(original, recons_SSIMloss, "Original vs. recons_SSIMloss")


text:
# This prediction code is simulated in this kernel:
# https://www.kaggle.com/vbookshelf/e-33-wheat-flask-app-inference-code

# Handling base64 images is simulated in this kernel:
# https://www.kaggle.com/vbookshelf/tb-my-flask-python-app-workflow

# *** NOTE: This entire file will get imported into __init__.py ***
# ------------------------------------------------------------------

#from the app folder import the app object
from app import app

from flask import request
from flask import jsonify
from flask import Flask
import base64
from PIL import Image
import io

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

import torchvision
import torchvision.transforms as transforms

from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
from torchvision import models # reg_model comes from here

import numpy as np
import cv2

# Special packages

# seg_model comes from here
import segmentation_models_pytorch as smp
from segmentation_models_pytorch.encoders import get_preprocessing_fn

# this is for the pre-processing
import albumentations as albu
from albumentations import Compose

# Are we using and special packages?
# ------------------------------------

# Check if these are pre-intalled on PythonAnywhere.
# If not, these packages have to be pip installed onto the server:

# (1) Segmentation Models Pytorch
# https://github.com/qubvel/segmentation_models.pytorch
# $ pip install segmentation-models-pytorch

# (2) Albumentations (used to pre-process the image)
# https://github.com/albumentations-team/albumentations
# $ pip install albumentations

# ----end

# -------------------------------
# Define the Model Architectures
# -------------------------------

# This app uses two models - segmentation and regression.

# (1) seg_model
# --------------

BACKBONE = 'resnet34'
ENCODER_WEIGHTS = 'imagenet'
ACTIVATION = 'sigmoid' # could be None for logits or 'softmax2d' for multicalss segmentation

seg_model = smp.Unet(
    encoder_name=BACKBONE,
    encoder_weights=ENCODER_WEIGHTS,
    classes=1,
    activation=ACTIVATION,
)

# Initialize the pre-processing function.
# Both models are resnet34 so the same image pre-processing method applies to both.
preprocessing_fn = smp.encoders.get_preprocessing_fn(BACKBONE, ENCODER_WEIGHTS)

# (2) reg_model
# --------------

reg_model = models.resnet34(pretrained=True)
in_features = reg_model.fc.in_features

reg_model.fc = nn.Linear(in_features, 1)

# -------------------------------
# Define the helper functions
# -------------------------------

def get_model():

    global seg_model
    global reg_model

	# Load the saved weights into the architecture.
	# Note that this file (views.py) gets imported into the app.ini file therefore,
	# place the model in the same folder as the app.ini file.

    seg_model_path = 'seg_model.pt'
    reg_model_path = 'reg_model.pt'
    seg_model.load_state_dict(torch.load(seg_model_path, map_location=torch.device('cpu')))
    reg_model.load_state_dict(torch.load(reg_model_path, map_location=torch.device('cpu')))

    # send the models to the device
    seg_model.to(device)
    reg_model.to(device)

    # Set the Mode
    seg_model.eval()
    reg_model.eval()

    # Turn gradient calculations off.
    torch.set_grad_enabled(False)

    print(" * Models loaded!")

def get_preprocessing(preprocessing_fn):
    """Construct preprocessing transform

    Args:
        preprocessing_fn (callbale): data normalization function
            (can be specific for each pretrained neural network)
    Return:
        transform: albumentations.Compose

    """

    _transform = [
        albu.Lambda(image=preprocessing_fn),
    ]
    return albu.Compose(_transform)

def preprocess_image(base64Image):

	"""
	Input: base64 image (dataURL with prefix removed)
	Output: image as numpy array --> 512x512x3
	"""

	# Decode the base64 image.
	# The result is a PIL image not a numpy array.
	decoded = base64.b64decode(base64Image)
	PIL_image = Image.open(io.BytesIO(decoded))

	# convert the PIL image into a numpy array
	# This method uses Keras img_to_array.
	#np_image = img_to_array(PIL_image)

	# This method uses numpy.
	## Double check the output shape.
	np_image = np.array(PIL_image)

	# resize the image to 512x512x3
	np_image = cv2.resize(np_image, (512, 512))

	# Pre-process the image

	# This is a dummy mask so that we can use the same
	# setup as that in the Kaggle notebook.
	mask = np.zeros((512, 512, 1))

	# initialize the pre-processing function.
	# This step was done when the model was initialized.
	# source: from segmentation_models_pytorch.encoders import get_preprocessing_fn
	# preprocessing_fn = get_preprocessing_fn(BACKBONE, pretrained='imagenet')

	# call the function that we defined above
	preprocessing = get_preprocessing(preprocessing_fn)

	# get the pre-processed image
	sample = preprocessing(image=np_image, mask=mask)
	np_image, dummy_mask = sample['image'], sample['mask']

	# convert to a channels first format to suit Pytorch
	np_image = np_image.transpose((2, 0, 1))

	# convert to a torch tensor
	torch_image = torch.tensor(np_image, dtype=torch.float)

	# add a batch dimension
	torch_image = torch_image.unsqueeze(0)

	return torch_image

def multiply_masks_and_images(torch_images, torch_thresh_masks):

	"""
	Trying to do this multiplication with Pytorch tensors
	did not produce the result that I wanted. Therefore, here I am
	converting the tensors to numy, doing the multiplication, and
	then converting back to pytorch.

	"""

	# convert from torch tensors to numpy
	np_images = torch_images.cpu().numpy()
	np_thresh_masks = torch_thresh_masks.cpu().numpy()

	# reshape
	np_images = np_images.reshape((-1, 512, 512, 3))
	np_thresh_masks = np_thresh_masks.reshape((-1, 512, 512, 1))

	# multiply the mask by the image
	modified_images = np_thresh_masks * np_images

	# change shape to channels first to suit pytorch
	#modified_images = modified_images.transpose((2, 0, 1))
	modified_images = modified_images.reshape((-1, 3, 512, 512))

	# convert to torch tensor
	modified_images = torch.tensor(modified_images, dtype=torch.float)

	return modified_images

# -------------------------------
# RUN THE CODE
# -------------------------------

# Define the device
# ------------------
device = "cpu"

# Load the models
# ----------------

print(" * Loading Pytorch models...")
get_model()

# Define the endpoints
# ---------------------

@app.route('/')
def index():
	return 'Hello world. I am a flask app.'

@app.route('/test')
def test():
	return 'Testing testing...'

# To access this endpoint navigate to:
# server_ip_address/static/predict.html
# Update the server ip address in the static/predict.html file.
# This endpoint has an html page in the static flask folder.

@app.route("/predict", methods=["POST"])
def predict():
	message = request.get_json(force=True)
	base64Image = message['image']
	#decoded = base64.b64decode(encoded)
	#image = Image.open(io.BytesIO(decoded))

	processed_image = preprocess_image(base64Image)
	print('Image pre-processing complete.')

	# send the masks to the device
	processed_image = processed_image.to(device, dtype=torch.float)

	# use seg_model to make a prediction
	seg_pred = seg_model(processed_image)
	print('Seg prediction complete.')

	# Threshold the predicted segmentation masks.
	# Remember that these are torch tensors and not numpy matrices. We are using
	# the torch tensor method of changing the datatype from bool to integer.
	thresh_mask = (seg_pred >= 0.7).int()

	# do the multiplication with numpy
	seg_output_masks = multiply_masks_and_images(processed_image, thresh_mask)

	# send the masks to the device
	seg_output_masks = seg_output_masks.to(device, dtype=torch.float)

	# pass the input through the model
	reg_pred = reg_model(seg_output_masks)

	# convert the prediction to a number
	pred = reg_pred.item()

	# round the prediction
	pred = round(pred, 0)

	print('Reg prediction complete.')

	response = {
	    'prediction': {
	        'wheat_count': pred,
	    }
	}
	return jsonify(response)

max_stars_repo_name: vbookshelf/Automated-Wheat-Counter

max_issues_repo_name: vbookshelf/Automated-Wheat-Counter

max_forks_repo_name: vbookshelf/Automated-Wheat-Counter


text:
from typing import Callable, Generic, Optional, Tuple, TypeVar

import jax.numpy as jnp
from jax.tree_util import tree_map

from ..annotations import ComplexNumeric, PyTree, RealNumeric
from ..dataclasses import dataclass
from .transform import SecondOrderGradientTransformation

__all__ = ['SMDState', 'SMDGradient']

Weights = TypeVar('Weights', bound=PyTree)

@dataclass
class SMDState(Generic[Weights]):
    log_learning_rate: Weights
    v: Weights

@dataclass
class SMDGradient(SecondOrderGradientTransformation[SMDState[Weights], Weights],
                  Generic[Weights]):
    """
    Schraudolph, N. N. (1999). Local gain adaptation in stochastic gradient descent. Artificial
    Neural Networks, 1999. ICANN 99. Ninth International Conference on (Conf. Publ. No. 470), 2,
    569–574. https://doi.org/10.1049/cp:19991170
    """
    meta_learning_rate: RealNumeric = 1e-2

    def init(self, parameters: Weights) -> SMDState[Weights]:
        z = tree_map(jnp.zeros_like, parameters)
        return SMDState[Weights](z, z)

    def second_order_update(self,
                            gradient: Weights,
                            state: SMDState[Weights],
                            parameters: Optional[Weights],
                            hessian_vector_product: Callable[[Weights], Weights]) -> (
                                Tuple[Weights, SMDState[Weights]]):
        negative_gradient = tree_map(jnp.negative, gradient)  # delta

        # Update log-learning rate.
        def g(log_p: RealNumeric, delta: ComplexNumeric, v: ComplexNumeric) -> ComplexNumeric:
            return log_p + self.meta_learning_rate * delta * v

        new_log_learning_rate = tree_map(g, state.log_learning_rate, negative_gradient, state.v)
        learning_rate = tree_map(jnp.exp, new_log_learning_rate)  # p

        # Calculate gradient.
        gradient = tree_map(jnp.multiply, learning_rate, negative_gradient)

        # Update v.
        def f(v: ComplexNumeric,
              p: RealNumeric,
              delta: ComplexNumeric,
              hv: ComplexNumeric) -> ComplexNumeric:
            return v + p * delta - hv

        new_v = tree_map(f, state.v, learning_rate, negative_gradient,
                         hessian_vector_product(state.v))

        return gradient, SMDState[Weights](new_log_learning_rate, new_v)

max_stars_repo_name: NeilGirdhar/tjax

max_issues_repo_name: NeilGirdhar/tjax

max_forks_repo_name: NeilGirdhar/tjax

reason: import jax,from jax


text:
#!/usr/bin/env python

import os, sys, glob

import scipy
from scipy.linalg import pinv

import numpy as np

import matplotlib
from pylab import *

def rangeLog(min, max, n):
    """Return an array of n log-spaces values from min to max.
    NOTE: All values must be greater than 0"""

    logmin, logmax = log(min), log(max)
    return np.exp( np.arange( logmin, logmax, (logmax-logmin)/float(n) ) )

def rangeLin(min, max, n):
    """Return an array of n linear-spaced values from min to max."""

    return  np.arange( min, max, (max-min)/n )

def Xmatrix(k,t,w, standardizeData=True):
    """Return X matrix Xij = [ exp(-k_j*t_i); I*w^(1/2) ] from arrays k and t. """

    K, N = len(k), len(t)
    X = np.zeros( (N+K,K) )
    for i in range(N):
      for j in range(K):
        X[i,j] = np.exp(-1.0*k[j]*t[i])
    for i in range(K):
        X[i+len(t),i] = w**0.5

    Xmean = (X[0:N,:]).mean(axis=0)
    if standardizeData:
      for j in range(K):
        X[0:N,j] = X[0:N,j] - Xmean[j]

    return X, Xmean

def Xsubmatrix(k,t, standardizeData=True):
    """Return X matrix Xij = [ exp(-k_j*t_i)] from arrays k and t. """

    K, N = len(k), len(t)
    X = np.zeros( (N,K) )
    for i in range(N):
      for j in range(K):
        X[i,j] = np.exp(-1.0*k[j]*t[i])

    Xmean = X.mean(axis=0)
    if standardizeData:
      for j in range(K):
        X[:,j] = X[:,j] - Xmean[j]

    return X, Xmean

def SumSpectra(A, Rates, Times, offset=0.0):
    """Return the sum of the exponential relaxations.
    If the data is standardized, an offset constant zero rate must be provided."""

    #print '*** in SumSpectra: ***'
    result = np.zeros( Times.shape )
    for i in range(len(Rates)):
        #print '***', Rates[i]
        result += A[i]*np.exp( -1.0*Rates[i]*Times)
    return result

def testData(nTimes = 1000, taus = [1.0e-6, 1.0e-4, 5.0e-3], amps = [0.3, 0.3, 0.4], sigma = 0.05, linear=False):
    """
    nTimes = 727 # same number as W55F dataset
    taus = [1.0e-6, 1.0e-4, 5.0e-3] # timescales for a test data curve
    amps = [0.3, 0.3, 0.4]  # ampltudes of each relaxation
    sigma = 0.05   # add artifical noise to the data
    """

    if linear:
        Times = rangeLin(1.0e-7, 1.0e-3, nTimes) # in seconds
    else:
        Times = rangeLog(1.0e-9, 1.0, nTimes) # in seconds

    Data = np.zeros( Times.shape )
    for i in range(len(taus)):
        Data += amps[i]*np.exp(-1.*Times/taus[i])
    Data += sigma*np.random.randn( len(Data) )

    return Times, Data

def testStretchedExpData(nTimes = 1000, beta = 0.5, sigma = 0.05, linear=False):
    """
    nTimes = 727 # same number as W55F dataset
    beta = the stretching exponenent (should be between 0 and 1)
    sigma = 0.05   # add artifical noise to the data
    """

    if linear:
        Times = rangeLin(1.0e-3, 1.0e+3, nTimes) # in seconds
    else:
        Times = rangeLog(1.0e-3, 1.0e+3, nTimes) # in seconds

    Data = np.exp(-1.*(Times**beta))
    Data += sigma*np.random.randn( len(Data) )

    return Times, Data

def fitRateSpectrum(Times, Data, Rates, w, Lnorm='ridge', standardizeData=True, CalcNdof=False, rho=0.5):
    """Using pseudo-inverse, with Tikhonov regularization (w parameter) to solve the inverse lapace tranform.
    Returns coefficients A_k, residual sum of squares (rss), and number of degrees of freedom, for each relaxation rate.
    """

    if Lnorm == 'lasso':
        # Use L1-norm Lasso regression
        try:
            from scikits.learn.linear_model import Lasso
        except:
            print 'Error: could NOT import Lasso from scikits.learn.linear_model.  Using L2 norm (ridge).'
            Lnorm = 'ridge'

    if Lnorm == 'enet':
        # Use L1-L2-mixture norm Lasso regression
        try:
            from scikits.learn.linear_model import ElasticNet
        except:
            print 'Error: could NOT import ElasticNet from scikits.learn.linear_model.  Using L2 norm (ridge).'
            Lnorm = 'ridge'

    if Lnorm == 'lasso':

        lasso = Lasso(alpha = w, fit_intercept=False) # assume the data is already "centered" -- i.e. no zero rate
        X, Xmean = Xsubmatrix(Rates, Times, standardizeData=standardizeData)
        #print 'X.shape', X.shape, 'Data.shape', Data.shape
        lasso.fit(X, Data, max_iter=1e6, tol=1e-7)
        A = lasso.coef_

        # Compute "residual sum of squares" (note loss function is different for L1-norm)
        y_pred_lasso = lasso.predict(X)
        diff = y_pred_lasso - Data

    elif Lnorm == 'enet':

        # NOTE: The convention for rho is backwards in scikits.learn, instead of rho we must send (1-rho)
        enet = ElasticNet(alpha = w, rho=(1.-rho), fit_intercept=False) # assume the data is already "centered" -- i.e. no zero rate
        X, Xmean = Xsubmatrix(Rates, Times, standardizeData=standardizeData)
        #print 'X.shape', X.shape, 'Data.shape', Data.shape
        #enet.fit(X, Data, max_iter=1e6, tol=1e-7)
        enet.fit(X, Data, max_iter=1e6, tol=1e-3)  # for testing
        A = enet.coef_

        # Compute "residual sum of squares" (note loss function is different for L1-norm)
        y_pred_enet = enet.predict(X)
        diff = y_pred_enet - Data

    elif Lnorm == 'ridge':
        X, Xmean = Xmatrix(Rates, Times, w, standardizeData=standardizeData )
        Xinv = linalg.pinv(X)

        y = np.array( Data.tolist() + [0. for k in Rates] )
        if standardizeData:
            y - y.mean()
        A = np.dot(Xinv, y)

        # Compute "residual sum of squares" (note loss function is different for L1-norm)
        diff = SumSpectra(A, Rates, Times) - Data

    rss = np.dot(diff,diff)  # Residual sum of squares

    if CalcNdof:
        Xsub, Xmean = Xsubmatrix(Rates, Times, standardizeData=standardizeData)
        XT = np.transpose(Xsub)
        I_XT = np.eye(XT.shape[0])
        I_X = np.eye(Xsub.shape[0])
        Xtemp = np.dot(Xsub, np.linalg.inv(np.dot(XT,Xsub) + w*I_XT))
        ndof = np.trace(I_X - np.dot(Xtemp,XT))
    else:
        ndof = None

    return A, rss, ndof

def scaleValues(values):
    """Scale a numpy array of values so that (min, max) = (0,1)."""

    values = values - values.min()
    return values/values.max()

def scaleValuesWithInfo(values):
    """Scale a numpy array of values so that (min, max) = (0,1).
    Returns:
    scaled values
    Scaleby
    Shiftby   -- i.e. to get back the original values, first multiply by Scaleby, then add Shiftby
    """

    Shiftby = values.min()
    values = values - Shiftby
    Scaleby = values.max()
    return values/Scaleby, Scaleby, Shiftby

max_stars_repo_name: vvoelz/ratespec

max_issues_repo_name: vvoelz/ratespec

max_forks_repo_name: vvoelz/ratespec

reason: import numpy,import scipy,from scipy


text:
# -*- coding: utf-8 -*-
"""Classes for extinction calculation"""
from addict import Dict
from copy import deepcopy
from ELDAmwl.bases.factory import BaseOperation
from ELDAmwl.bases.factory import BaseOperationFactory
from ELDAmwl.component.interface import IExtOp
from ELDAmwl.component.interface import IMonteCarlo
from ELDAmwl.component.registry import registry
from ELDAmwl.extinction.product import Extinctions
from ELDAmwl.extinction.tools.operation import ExtinctionAutosmooth
from ELDAmwl.extinction.tools.operation import SignalSlope
from ELDAmwl.extinction.tools.operation import SlopeToExtinction
from ELDAmwl.utils.constants import ABOVE_MAX_ALT
from ELDAmwl.utils.constants import BELOW_OVL
from ELDAmwl.utils.constants import MC
from ELDAmwl.utils.constants import NC_FILL_INT
from ELDAmwl.utils.constants import NC_FILL_STR

import numpy as np
import zope

class CalcExtinction(BaseOperationFactory):
    """
    creates a class for the calculation of an extinction coefficient

    Returns an instance of BaseOperation which calculates the particle
    extinction coefficient from a  Raman signal. In this case, it
    will be always an instance of CalcExtinctionDefault().

    Keyword Args:
        ext_params (:class:`ELDAmwl.extinction.params.ExtinctionParams`): \
                retrieval parameter of the extinction product
        slope_routine (:class:`ELDAmwl.bases.factory.BaseOperation`):
            result of :class:`ELDAmwl.extinction.tools.operation.SignalSlope`
        slope_to_ext_routine (:class:`ELDAmwl.bases.factory.BaseOperation`):
            result of :class:`ELDAmwl.extinction.tools.operation.SlopeToExtinction`
        raman_signal (:class:`ELDAmwl.signals.Signals`): Raman signal
        empty_ext (:class:`ELDAmwl.extinction.product.Extinctions`): \
                instance of Extinctions which has all meta data but profile data are empty arrays

    Returns:
        instance of :class:`ELDAmwl.bases.factory.BaseOperation`

    """

    name = 'CalcExtinction'

    def __call__(self, **kwargs):
        assert 'ext_params' in kwargs
        assert 'slope_routine' in kwargs
        assert 'slope_to_ext_routine' in kwargs
        assert 'raman_signal' in kwargs
        assert 'empty_ext' in kwargs

        res = super(CalcExtinction, self).__call__(**kwargs)
        return res

    def get_classname_from_db(self):
        """

        return: always 'CalcExtinctionDefault' .
        """
        return 'CalcExtinctionDefault'

@zope.interface.implementer(IExtOp)
class CalcExtinctionDefault(BaseOperation):
    """
    Calculates particle extinction coefficient from Raman signal.

    The result is a copy of empty_ext, but its dataset (data, err, qf) is filled with the calculated values

    Keyword Args:
        ext_params (:class:`ELDAmwl.extinction.params.ExtinctionParams`): \
                retrieval parameter of the extinction product
        slope_routine (:class:`ELDAmwl.bases.factory.BaseOperation`):
            result of :class:`ELDAmwl.extinction.tools.operation.SignalSlope`
        slope_to_ext_routine (:class:`ELDAmwl.bases.factory.BaseOperation`):
            result of :class:`ELDAmwl.extinction.tools.operation.SlopeToExtinction`
        raman_signal (:class:`ELDAmwl.signals.Signals`): Raman signal
        empty_ext (:class:`ELDAmwl.extinction.product.Extinctions`): \
                instance of Extinctions which has all meta data but profile data are empty arrays

    Returns:
        profiles of particle extinction coefficients(:class:`ELDAmwl.extinction.product.Extinctions`)
    """

    name = 'CalcExtinctionDefault'

    ext_params = None
    signal = None
    slope_routine = None
    slope_to_ext_routine = None
    result = None
    x_data = None
    y_data = None
    yerr_data = None
    qf_data = None

    def __init__(self, **kwargs):
        super(CalcExtinctionDefault, self).__init__(**kwargs)
        self.signal = self.kwargs['raman_signal']
        self.ext_params = self.kwargs['ext_params']
        self.slope_routine = self.kwargs['slope_routine']
        self.slope_to_ext_routine = self.kwargs['slope_to_ext_routine']
        self.result = deepcopy(self.kwargs['empty_ext'])

    def calc_slope(self, t, lev, window, half_win):
        fb = lev - half_win
        lb = lev + half_win
        window_data = Dict({'x_data': self.x_data[t, fb:lb + 1],
                            'y_data': self.y_data[t, fb:lb + 1],
                            'yerr_data': self.yerr_data[t, fb:lb + 1],
                            })

        sig_slope = self.slope_routine.run(signal=window_data)
        qf = np.bitwise_or.reduce(self.qf_data[t, fb:lb + 1])

        self.result.ds['data'][t, lev] = sig_slope.slope
        self.result.ds['err'][t, lev] = sig_slope.slope_err
        self.result.ds['qf'][t, lev] = qf
        self.result.ds['binres'][t, lev] = window

    def prepare_data(self, data):
        self.x_data = np.array(data.range)
        self.y_data = np.array(data.ds.data)
        self.yerr_data = np.array(data.ds.err)
        self.qf_data = np.array(data.ds.qf)

    def calc_single_profile(self, t, data):
        fvb = data.first_valid_bin(t)
        lvb = data.last_valid_bin(t)
        for lev in range(fvb, lvb):
            window = int(data.ds.binres[t, lev])
            half_win = window // 2

            if lev < (fvb + half_win):
                self.result.set_invalid_point(t, lev, BELOW_OVL)

            elif lev >= (lvb - half_win):
                self.result.set_invalid_point(t, lev, ABOVE_MAX_ALT)

            else:
                self.calc_slope(t, lev, window, half_win)

    def run(self, data=None):
        """
        run the extinction calculation

        The the optional keyword arg 'data' allows to feed new raman signals into
        an existing instance of CalcExtinctionDefault and run a new calculation.
        This feature is used e.g., for Monte-Carlo error retrievals

        Keyword Args:
            data (:class:`ELDAmwl.signals.Signals`): Raman signal, default=None

        Returns:
            profiles of particle extinction coefficients(:class:`ELDAmwl.extinction.product.Extinctions`)

        """
        if data is None:
            data = self.signal

        self.prepare_data(data)

        for t in range(data.num_times):
            self.calc_single_profile(t, data)

        # extract relevant parameter for calculation of ext from signal slope
        # from ExtinctionParams into Dict
        param_dct = Dict({
            'detection_wavelength': data.detection_wavelength,
            'emission_wavelength': data.emission_wavelength,
            'angstroem_exponent': self.ext_params.ang_exp_asDataArray,
        })

        # SlopeToExtinction converts the slope into extinction coefficients
        self.slope_to_ext_routine(
            slope=self.result.ds,
            ext_params=param_dct).run()

        return self.result

class ExtinctionFactory(BaseOperationFactory):
    """
    optional argument resolution, can be LOWRES(=0) or HIGHRES(=1)
    """

    name = 'ExtinctionFactory'

    def __call__(self, **kwargs):
        assert 'data_storage' in kwargs
        assert 'ext_param' in kwargs
        assert 'autosmooth' in kwargs
        res = super(ExtinctionFactory, self).__call__(**kwargs)
        return res

    def get_classname_from_db(self):
        """

        return: always 'ExtinctionFactoryDefault' .
        """
        return ExtinctionFactoryDefault.__name__

class ExtinctionFactoryDefault(BaseOperation):
    """
    derives particle extinction coefficient.
    """

    name = 'ExtinctionFactoryDefault'
    param = None
    raman_sig = None
    smooth_res = None
    empty_ext = None
    prod_id = NC_FILL_STR
    resolution = NC_FILL_INT

    def get_smooth_res(self):
        if self.kwargs['autosmooth']:
            result = ExtinctionAutosmooth()(
                signal=self.raman_sig.ds,
                smooth_params=self.param.smooth_params,
            ).run()

        else:
            result = self.data_storage.binres_common_smooth(self.prod_id, self.resolution)

        self.smooth_res = result

    def prepare(self):
        self.param = self.kwargs['ext_param']
        self.prod_id = self.param.prod_id_str
        self.resolution = self.kwargs['resolution']

        # raman_sig is a deepcopy from data_storage
        self.raman_sig = self.data_storage.prepared_signal(
            self.param.prod_id_str,
            self.param.raman_sig_id)

        self.get_smooth_res()

        self.raman_sig.ds['binres'] = self.smooth_res

        self.empty_ext = Extinctions.init(self.raman_sig, self.param)

    def get_non_merge_product(self):

        calc_routine = CalcExtinction()(
            ext_params=self.param,
            slope_routine=SignalSlope()(prod_id=self.prod_id),
            slope_to_ext_routine=SlopeToExtinction(),
            raman_signal=self.raman_sig,
            empty_ext=self.empty_ext,
        )
        ext = calc_routine.run()

        if self.param.error_method == MC:
            adapter = zope.component.getAdapter(calc_routine, IMonteCarlo)
            ext.err[:] = adapter(self.param.mc_params)
        else:
            ext = ext

        del self.raman_sig
        return ext

    def get_product(self):
        self.prepare()

        if not self.param.includes_product_merging():
            ext = self.get_non_merge_product()
        else:
            # todo: result = Extinctions.from_merged_signals()
            ext = None
            pass

        return ext

registry.register_class(ExtinctionFactory,
                        ExtinctionFactoryDefault.__name__,
                        ExtinctionFactoryDefault)

registry.register_class(CalcExtinction,
                        CalcExtinctionDefault.__name__,
                        CalcExtinctionDefault)

max_stars_repo_name: actris-scc/ELDAmwl

max_issues_repo_name: actris-scc/ELDAmwl

max_forks_repo_name: actris-scc/ELDAmwl


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Nov 26 09:50:04 2021

@author: jtm545
"""

import sys
sys.path.insert(0, '../')

import numpy as np
import seaborn as sns
import pandas as pd
from cyipopt import minimize_ipopt

from pysilsub.problem import SilentSubstitutionProblem
from pysilsub.colorfunc import LMS_to_xyY, xyY_to_LMS
from pysilsub.plotting import stim_plot

sns.set_context('notebook')
sns.set_style('whitegrid')

spds = pd.read_csv('../data/S2_corrected_oo_spectra.csv',
                   index_col=['led','intensity'])
spds.index.rename(['Primary', 'Setting'], inplace=True)
spds.columns = pd.Int64Index(spds.columns.astype(int))
spds.columns.name = 'Wavelength'

# list of colors for the primaries
colors = ['blueviolet', 'royalblue', 'darkblue', 'blue', 'cyan',
          'green', 'lime', 'orange', 'red', 'darkred']

ss = SilentSubstitutionProblem(
    resolutions=[4095]*10,
    colors=colors,
    spds=spds,
    spd_binwidth=1,
    isolate=['I'],
    silence=['S', 'L','M'],
    target_contrast=.5
    )

target_xy=[0.31271, 0.32902]  # D65
target_luminance=600.
bg = ss.find_settings_xyY(target_xy, target_luminance)
ss.background = bg.x

# Define constraints and local minimizer
constraints = (
    {
    'type': 'eq',
    'fun': ss.silencing_constraint
    }
    )

# if ss.target_xy is not None:
#     constraints.append({
#         'type': 'eq',
#         'fun': ss.xy_chromaticity_constraint
#     })

# if ss.target_luminance is not None:
#     constraints.append({
#         'type': 'eq',
#         'fun': ss.luminance_constraint
#     })

# from scipy.optimize import basinhopping
# minimizer_kwargs = {
#     'method': 'SLSQP',
#     'bounds': ss.bounds,
#     'constraints': constraints}
# basinhopping(
#     func=ss.objective_function,
#     x0=ss.initial_guess_x0(),
#     minimizer_kwargs=minimizer_kwargs,
#     disp=True)

result = minimize_ipopt(
    fun=ss.objective_function,
    x0=ss.initial_guess_x0(),
    args=(),
    kwargs=None,
    method=None,
    jac=None,
    hess=None,
    hessp=None,
    bounds=ss.bounds,
    constraints=constraints,
    tol=1e-5,
    callback=None,
    options={b'print_level': 5, b'constr_viol_tol': 1e-5},
)

# Plot solution
bg, mod = ss.smlri_calculator(result.x)
ss.plot_solution(bg, mod)
fig = ss.debug_callback_plot(result.x)

#%% Make modulation

# target_contrasts = np.linspace(.35, 0, 20)
# new_bounds = [(min(val), max(val)) for val in zip(ss.background, result.x)]

# results = []
# for tc in target_contrasts[1:-1]:
#     print(f'target contrast: {tc}\n')
#     ss.target_contrast = tc
#     r = minimize_ipopt(
#         fun=ss.objective_function,
#         x0=ss.initial_guess_x0(),
#         args=(),
#         kwargs=None,
#         method=None,
#         jac=None,
#         hess=None,
#         hessp=None,
#         bounds=new_bounds,
#         constraints=constraints,
#         tol=1e-6,
#         callback=None,
#         options={b'print_level': 5, b'constr_viol_tol': 1e-6},
#     )
#     results.append(r)

# #%% plot

# new = []
# for i in results:
#     ss.debug_callback_plot(i.x)
#     c = ss.get_photoreceptor_contrasts(i.x)[2][0]
#     new.append(c)

# #%%
# bg = np.array([.5] *10)
# mod = np.array([-0.3568, -0.2557, 0.1752, 0.4446, 0.3304, -0.1898, -0.1393, 0.1307, 0.1147, 0.0494])
# mod+bg

max_stars_repo_name: PySilentSubstitution/silent-sub

max_issues_repo_name: PySilentSubstitution/pysilsub

max_forks_repo_name: PySilentSubstitution/pysilsub

reason: import numpy,from scipy


text:
from PIL import Image
from numpy import *
import os

from PCV.localdescriptors import sift

def process_image_dsift(imagename,resultname,size=20,steps=10,force_orientation=False,resize=None):
    """ Process an image with densely sampled SIFT descriptors
        and save the results in a file. Optional input: size of features,
        steps between locations, forcing computation of descriptor orientation
        (False means all are oriented upwards), tuple for resizing the image."""

    im = Image.open(imagename).convert('L')
    if resize!=None:
        im = im.resize(resize)
    m,n = im.size

    if imagename[-3:] != 'pgm':
        #create a pgm file
        im.save('tmp.pgm')
        imagename = 'tmp.pgm'

    # create frames and save to temporary file
    scale = size/3.0
    x,y = meshgrid(range(steps,m,steps),range(steps,n,steps))
    xx,yy = x.flatten(),y.flatten()
    frame = array([xx,yy,scale*ones(xx.shape[0]),zeros(xx.shape[0])])
    savetxt('tmp.frame',frame.T,fmt='%03.3f')

    if force_orientation:
        cmmd = str("sift "+imagename+" --output="+resultname+
                    " --read-frames=tmp.frame --orientations")
    else:
        cmmd = str("sift "+imagename+" --output="+resultname+
                    " --read-frames=tmp.frame")
    os.system(cmmd)
    print( 'processed', imagename, 'to', resultname)

reason: from numpy


text:
#Importamos todo lo necesario
from tensorflow import keras
from tensorflow.keras.preprocessing import image
import cv2
import numpy as np
import matplotlib.image as mim
import matplotlib.pyplot as plt
from tensorflow.keras.models import load_model
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
import os
from matplotlib import pyplot as plt
dimension_reescalar = (224, 224) #Las dimensiones (para cada uno de los 3 rgb que queremos en cada foto)
path_del_val = '/srv/image-classification-tf/data/dataset_files/val.txt'
path_del_modelo = '/srv/image-classification-tf/models/2020-02-04_182818/ckpts/final_model.h5'
modelo = load_model(path_del_modelo ,compile=False) #Importas el modelo que quieras .h5
vector_entradas = np.array(([])) #inicializo vector vacío
etiquetas_reales = np.array(([])) #Donde guardaremos las etiquetas reales para la matriz de confusión
contador = 0 #tengo que saber cuántas imágenes voy a hacer ya que numpy te guarda todos los coeficientes en una fila
#y, antes de pasarle a la red las imágenes, te toca hacer un resize a (numero_de_fotos,224,224,3)

#EJEMPLO BIEN HECHO CON LARA PARA UNA SOLA IMAGEN:
path_imagen = '/srv/image-classification-tf/data/images/Conus/INDO-PACIFIC/lohri_IP/006-lohri-IP-Mozambique.jpg' #categoría 31
imagen_sola = image.load_img(path_imagen, target_size=(224,224))
imagen_sola = np.expand_dims(imagen_sola, axis = 0)
prediccion_sola = modelo.predict(imagen_sola)
print(prediccion_sola)

with open(path_del_val, 'r') as path:
    for x in path:
        print(x)
        etiquetas_reales = np.append(etiquetas_reales, x.split()[1])
        #pasamos la imagen al numpy
        imagen = image.load_img(x.split()[0], target_size = dimension_reescalar)
        imagen = image.img_to_array(imagen)
        imagen = np.expand_dims(imagen, axis = 0)
        vector_entradas = np.append(vector_entradas, imagen)
        contador = contador + 1
#Ya tenemos en una sola fila todos los coeficientes de todas las imágenes. Vamos a hacer el reshape para tenerlas como las requiere la red
#Es decir (numero_de_fotos, 224,224,3)
print(vector_entradas.shape)
vector_entradas = np.resize(vector_entradas,(contador,224,224,3))
print("Y después")
print(vector_entradas.shape)
#vector_parcial = vector_entradas[0] PRUEBAS
#vector_parcial = np.resize(vector_parcial, (1,224,224,3)) PRUEBAS
predicciones = modelo.predict(vector_entradas) #le pasamos en un batch todo y nos retorna las predicciones
#cv2.imwrite('foto_comprobar.jpg', vector_entradas[0]) PRUEBAS
print(predicciones[0])
predicciones_sin_densidad = [] #Inicializo LISTA con las predicciones
for i in predicciones:
    clase_mas_probable, = np.where(i == max(i))
    predicciones_sin_densidad.append(clase_mas_probable[0])
print('las predicciones:')
print(predicciones_sin_densidad)
print('la realidad:')
print(etiquetas_reales)

#Como nos retorna densidades de probabilidad, vamos uno por uno viendo para qué clase se lleva la máxima densidad de probabilidad y guardandolo en
#vector

print("hemos terminado")


text:
# pip3 install 'gym[atari,accept-rom-license]==0.22.0'

import matplotlib.pyplot as plt
import gym
from gym import wrappers
import random
import numpy as np

env = gym.make('ALE/MsPacman-v5', render_mode='human')
height, width, channels = env.observation_space.shape
actions = env.action_space.n

episodes = 1

random_model_scores = []

# Saves .mp4 and .json files
env = wrappers.Monitor(env, "./stream_test/gym-results", force=True)

game_arrays = []

for episode in range(1, episodes+1):
    state = env.reset()
    done = False
    score = 0

    while not done:
        # action = random.choice([0,1,2,3,4,5,6,7,8])
        action = env.action_space.sample()
        n_state, reward, done, info = env.step(action)
        game_arrays.append(env.render(mode='rgb_array'))
        # action: int - 0,1,2,3,4,5,6,7,8
        # n_state: numpy array - dimensions (210, 160, 3) --> (height, width, RGB channels)
        # reward: float - 0.0
        # done: boolean - True, False
        # info: dictionary - {'lives': 1, 'episode_frame_number': 1892, 'frame_number': 1892}
        score += reward
    print('Episode:{} Score:{}'.format(episode, score))
    # env.play()

    random_model_scores.append(score)

env.close()

# Get numpy array of game to plot later on
game_array = np.stack(game_arrays, axis=3)
game_array = np.rollaxis(game_array, -1)
print(game_array.shape)

# for i in range(20):
#     plt.imshow(game_array[i,:,:,:])
#     plt.show()

# Create video stream from numpy arrays in matplotlib
# https://ben.bolte.cc/matplotlib-videos

reason: import numpy,from numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2018-02-06 10:28:33
# @Author  : Yan Liu & Zhi Liu (zhiliu.mind@gmail.com)
# @Link    : http://iridescent.ink
# @Version : $1.0$
from __future__ import division, print_function, absolute_import
import numpy as np

def rectangle(SceneArea, x0, y0, h, w, a=None, dx=None, dy=None, verbose=False):
    r"""rectangle area

    generates rectangle area

    Parameters
    ----------
    SceneArea : list or tuple
        Scene Area, unit m, [xmin,xmax,ymin,ymax]
    x0 : float
        x center of rectangle, unit m (x: horizontal, y: vertical)
    y0 : float
        y center of rectangle, unit m (x: horizontal, y: vertical)
    h : float
        height of rectangle, unit m (the default is None, which generates randomly)
    w : float
        width of rectangle, unit m (the default is None, which generates randomly)
    a : float, optional
        amplitude (the default is None, which generates randomly)
    dx : float, optional
        resolution in range (default: {1 / (xmax-xmin)})
    dy : float, optional
        resolution in azimuth (default: {1 / (ymax-ymin)})
    verbose : bool, optional
        show more log info (the default is False, which means does not show)

    Returns
    -------
    rects : tensor
        rectangle area
    """

    if verbose:
        print(SceneArea)
    (Xmin, Xmax, Ymin, Ymax) = SceneArea
    if dx is None:
        dx = 1. / (Xmax - Xmin)
    if dy is None:
        dy = 1. / (Ymax - Ymin)

    if h is None:
        h = np.random.rand() * 100.
    if w is None:
        w = np.random.rand() * 100.

    if a is not None:
        g = a.copy()
    rects = []
    for x in np.linspace(max(x0 - w / 2., Xmin), min(x0 + w / 2., Xmax), int(w)):
        for y in np.linspace(max(y0 - h / 2., Ymin), min(y0 + h / 2., Ymax), int(h)):
            if a is None:
                g = np.random.rand()
            rects.append([x, y, 0, 0, 0, 0, g])

    rects = np.array(rects)

    if verbose:
        print(rects, rects.shape)
    return rects

def disc(SceneArea, x0, y0, r, a=None, dx=None, dy=None, verbose=False):
    r"""disc area

    generates disc area

    Parameters
    ----------
    SceneArea : list or tuple
        Scene Area, [xmin,xmax,ymin,ymax]
    x0 : int
        x center of disc (x: horizontal, y: vertical)
    y0 : int
        y center of disc (x: horizontal, y: vertical)
    r : int
        radius of disc
    a : int, optional
        amplitude of disc (default: {None})
    dx : float, optional
        resolution in range (default: {1 / (xmax-xmin)})
    dy : float, optional
        resolution in azimuth (default: {1 / (ymax-ymin)})
    verbose : bool, optional
        show more log info (the default is False, which means does not show)

    Returns
    -------
    ccas : tensor
        disk area
    """

    if verbose:
        print(SceneArea)
    (Xmin, Xmax, Ymin, Ymax) = SceneArea
    if dx is None:
        dx = 1. / (Xmax - Xmin)
    if dy is None:
        dy = 1. / (Ymax - Ymin)
    # print("disc: x0, y0, r, a, dx, dy --> ", x0, y0, r, a, dx, dy)
    r2 = r * r
    ccas = []
    if a is not None:
        g = a.copy()
    for x in np.linspace(max(x0 - r, Xmin), min(x0 + r, Xmax), int(2 * r + 2)):
        for y in np.linspace(max(y0 - r, Ymin), min(y0 + r, Ymax), int(2 * r + 2)):
            d = (x - x0) ** 2 + (y - y0) ** 2
            if d <= r2:
                if a is None:
                    g = np.random.rand()
                ccas.append([x, y, 0, 0, 0, 0, g])

    ccas = np.array(ccas)
    if verbose:
        print(ccas, ccas.shape, "===++===")
    return ccas

max_stars_repo_name: aisari/torchsar

max_issues_repo_name: aisari/torchsar

max_forks_repo_name: aisari/torchsar


text:
from numpy import abs

from skyfield.api import load
from skyfield.toposlib import Topos

angle = (15, 25, 35, 45)

def ts():
    yield load.timescale()

def test_beneath(ts, angle):
    t = ts.utc(2018, 1, 19, 14, 37, 55)
    # An elevation of 0 is more difficult for the routine's accuracy
    # than a very large elevation.
    top = Topos(latitude_degrees=angle, longitude_degrees=0, elevation_m=0)
    p = top.at(t)
    b = Topos.subpoint(p)
    error_degrees = abs(b.latitude.degrees - angle)
    error_mas = 60.0 * 60.0 * 1000.0 * error_degrees
    #print(b.latitude.degrees, deg, error_mas)
    assert error_mas < 0.1


text:
from __future__ import (absolute_import, unicode_literals, division,
                        print_function)
import sys
import collections

import warnings
import numpy as np
# If numba is installed, import jit. Otherwise, define an empty decorator with
# the same name.

try:
    from numba import jit
except:
    def jit(fun):
        return fun

def simon(message, **kwargs):
    """The Statistical Interpretation MONitor.

    A warning system designed to always remind the user that Simon
    is watching him/her.

    Parameters
    ----------
    message : string
        The message that is thrown

    kwargs : dict
        The rest of the arguments that are passed to warnings.warn
    """

    warnings.warn("SIMON says: {0}".format(message), **kwargs)

def rebin_data(x, y, dx_new, method='sum'):

    """Rebin some data to an arbitrary new data resolution. Either sum
    the data points in the new bins or average them.

    Parameters
    ----------
    x: iterable
        The dependent variable with some resolution dx_old = x[1]-x[0]

    y: iterable
        The independent variable to be binned

    dx_new: float
        The new resolution of the dependent variable x

    method: {"sum" | "average" | "mean"}, optional, default "sum"
        The method to be used in binning. Either sum the samples y in
        each new bin of x, or take the arithmetic mean.

    Returns
    -------
    xbin: numpy.ndarray
        The midpoints of the new bins in x

    ybin: numpy.ndarray
        The binned quantity y
    """

    y = np.asarray(y)

    dx_old = x[1] - x[0]

    if dx_new < dx_old:
        raise ValueError("New frequency resolution must be larger than "
                         "old frequency resolution.")

    step_size = dx_new / dx_old

    output = []
    for i in np.arange(0, y.shape[0], step_size):
        total = 0

        int_i = int(i)
        prev_frac = int_i + 1 - i
        prev_bin = int_i
        total += prev_frac * y[prev_bin]

        if i + step_size < len(x):
            # Fractional part of next bin:
            next_frac = i + step_size - int(i + step_size)
            next_bin = int(i + step_size)
            total += next_frac * y[next_bin]

        total += sum(y[int(i+1):int(i+step_size)])
        output.append(total)

    output = np.asarray(output)

    if method in ['mean', 'avg', 'average', 'arithmetic mean']:
        ybin = output / np.float(step_size)

    elif method == "sum":
        ybin = output

    else:
        raise ValueError("Method for summing or averaging not recognized. "
                         "Please enter either 'sum' or 'mean'.")

    tseg = x[-1] - x[0] + dx_old

    if (tseg / dx_new % 1) > 0:
        ybin = ybin[:-1]

    new_x0 = (x[0] - (0.5*dx_old)) + (0.5*dx_new)
    xbin = np.arange(ybin.shape[0]) * dx_new + new_x0

    return xbin, ybin, step_size

def assign_value_if_none(value, default):
    return default if value is None else value

def look_for_array_in_array(array1, array2):
    return next((i for i in array1 if i in array2), None)

def is_string(s):  # pragma : no cover
    """Portable function to answer this question."""

    PY2 = sys.version_info[0] == 2
    if PY2:
        return isinstance(s, basestring)  # NOQA
    else:
        return isinstance(s, str)  # NOQA

def is_iterable(stuff):
    """Test if stuff is an iterable."""

    return isinstance(stuff, collections.Iterable)

def order_list_of_arrays(data, order):
    if hasattr(data, 'items'):
        data = dict([(key, value[order])
                     for key, value in data.items()])
    elif is_iterable(data):
        data = [i[order] for i in data]
    else:
        data = None
    return data

def optimal_bin_time(fftlen, tbin):
    """Vary slightly the bin time to have a power of two number of bins.

    Given an FFT length and a proposed bin time, return a bin time
    slightly shorter than the original, that will produce a power-of-two number
    of FFT bins.
    """

    return fftlen / (2 ** np.ceil(np.log2(fftlen / tbin)))

def contiguous_regions(condition):
    """Find contiguous True regions of the boolean array "condition".

    Return a 2D array where the first column is the start index of the region
    and the second column is the end index.

    Parameters
    ----------
    condition : boolean array

    Returns
    -------
    idx : [[i0_0, i0_1], [i1_0, i1_1], ...]
        A list of integer couples, with the start and end of each True blocks
        in the original array

    Notes
    -----
    From : http://stackoverflow.com/questions/4494404/find-large-number-of-consecutive-values-
    fulfilling-condition-in-a-numpy-array
    """

    # NOQA
    # Find the indicies of changes in "condition"
    diff = np.diff(condition)
    idx, = diff.nonzero()
    # We need to start things after the change in "condition". Therefore,
    # we'll shift the index by 1 to the right.
    idx += 1
    if condition[0]:
        # If the start of condition is True prepend a 0
        idx = np.r_[0, idx]
    if condition[-1]:
        # If the end of condition is True, append the length of the array
        idx = np.r_[idx, condition.size]
    # Reshape the result into two columns
    idx.shape = (-1, 2)
    return idx

max_stars_repo_name: svenbuder/SummerSchool

max_issues_repo_name: svenbuder/SummerSchool

max_forks_repo_name: svenbuder/SummerSchool

reason: import numpy,from numba