from time import time
from requests import Session as BaseSession
from id3.cache import LRUDictCache

class Session(BaseSession):
    """
    A normal requests.Session object which has GET requests cached for a specified duration.
    """

    def __init__(self, cache_duration: float = 3600) -> None:
        super().__init__()
        self.cache_duration = float(cache_duration)
        self.cache = LRUDictCache()

    def request(self, method: str, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=None, proxies=None, hooks=None, stream=None, verify=None, cert=None, json=None):
        method = method.upper()
        cache_key = f"{url}{params}{headers}{cookies}{files}{auth}"
        try:
            if method == "GET" and cache_key in self.cache:
                cache_item = self.cache[cache_key]
                if time() - cache_item["time"] <= self.cache_duration:
                    return cache_item["response"]
                self.cache.pop(cache_key, None)
        except Exception:
            pass
        r = super().request(method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
        if method == "GET":
            self.cache[cache_key] = {"response": r, "time": time()}
        return r


# this program resize a RGB image to 160*320 in shape

import cv2
import numpy as np
img = cv2.imread('images.jpeg')
res = cv2.resize(img,None,fx=2, fy=2, interpolation = cv2.INTER_CUBIC)
#OR
height, width = img.shape[:2]
res = cv2.resize(img,(320, 160), interpolation = cv2.INTER_CUBIC)
cv2.imwrite("road.jpg", res)


from biscuit.biscuit import runGame as biscuit
from memoryGame.memory_game import runGame as memory
from segmentation.coloriage import runGame as coloriage
from soleil.soleil  import runGame as soleil
from piano.glass import runGame as piano
import utils as animation
import cv2
import pygame
# lancer la sequence intro du jeu
# lancer le jeu sur lequel on cliquer

mousePos = (-1,-1)

def introDuJeu():
    global mousePos
    pygame.mixer.init()
    musicIntro = pygame.mixer.Sound('animation/intro.mp3')
    musicIntro.play()
    musicAmbiance = pygame.mixer.Sound('biscuit/assets/UnfinishedBusiness.mp3')
    action = animation.afficher_intro()
    if action == "Quitter":
        return
    musicIntro.stop()
    musicAmbiance.play()
    play = True
    while play:
        n = get_game()
        mousePos = (-1,-1)
        if n == -1:
            return
        play = lancer_le_jeu(n)
    return

def get_mouse_pos(event,x,y,flags,param):
    global mousePos
    if event == cv2.EVENT_LBUTTONDOWN:
        mousePos = (x,y)

def get_game():
    global mousePos
    mousePos = (-1,-1)
    menu = cv2.imread("animation/intro/introduction539.png")
    cv2.imshow("jeu", menu)
    while True:
        cv2.setMouseCallback('jeu',get_mouse_pos)
        key = cv2.waitKey(1)
        if mousePos[0] != -1:
            if mousePos[1] < 600:
                if mousePos[0] < 640:
                    print(1)
                    return 1
                elif mousePos[0] < 1280:
                    print(2)
                    return 2
                elif mousePos[0] < 1920:
                    print(3)
                    return 3
            elif mousePos[1]<1080:
                if mousePos[0] < 640:
                    print(4)
                    return 4
                elif mousePos[0] < 1280:
                    print(5)
                    return 5
                elif mousePos[0] < 1920:
                    return -1
        if key == ord('&') :
            return 1
        if key == ord('é') :
            return 2
        if key == ord('"') :
            return 3
        if key == ord("'") :
            return 4
        if key == ord('('):
            return 5
        if key == ord('q') or key == ord('§'):
            return -1

def lancer_le_jeu(n):
    if n == 1:
        action = soleil()
    elif n == 2:
        action = biscuit()
    elif n== 3 :
        action = memory()
    elif n== 4:
        action = coloriage()
    elif n== 5 :
        action = piano()
    else :
        return False

    return action

introDuJeu()


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name = 'home_page_landing'),
    path('presale_choice/', views.presale_choice, name = 'presale_choice'),
    path('presale_analysis/', views.presale_analysis, name = 'presale_analysis'),
    path('presale_set_up/', views.presale_set_up, name = 'presale_set_up')

]


from os import environ, listdir
from json import load, dump
from discord import Activity, Embed, Intents, opus
from discord.ext import commands
from rich import print
from rich.console import Console
from rich.table import Table
from ctypes.util import find_library
console = Console()
__import__("dotenv").load_dotenv()

# -------
# prefix area

defaultprefixes = ["World ", "w/", "world "]
ownerprefix = ["w/", "world ", "World "]

async def get_prefix(world, message):
    if await world.is_owner(message.author):
        return commands.when_mentioned_or(*ownerprefix)(world, message)
    prefixes = load(open('prefixes.json', 'r'))
    if not str(message.guild.id) in prefixes:
        return commands.when_mentioned_or(*defaultprefixes)(world, message)
    return commands.when_mentioned_or(prefixes[str(message.guild.id)])(world, message)

# -------
# Main bot area

world = commands.Bot(
    command_prefix=get_prefix,
    description="Discord Bot Made For All",
    case_insensitive=True,
    intents=Intents.all()
)
setattr(world, "color", 0x2F3136)
world.remove_command("help")

# -------
# Command area

@world.command(help="Change the prefix for your discord guild")
@commands.has_permissions(administrator=True)
async def changeprefix(ctx, prefix):
    if len(prefix) > 7:
        return await ctx.send(f"Sorry {ctx.author.mention} a limit of `7` letters please.")
    with open('prefixes.json', 'r') as f:
        prefixes = load(f)

    prefixes[str(ctx.guild.id)] = prefix

    with open('prefixes.json', 'w') as f:
        dump(prefixes, f, indent=4)
    await ctx.send(embed=Embed(
        title="Custom prefix",
        description=f"Prefix for {ctx.guild} is now `{prefix}`.\nTry using the help command: `{prefix}help`"
    ))

@world.command()
async def enablenoprefix(ctx):
    if await world.is_owner(ctx.author):
        ownerprefix.append("")
        await ctx.send(f"Hey {ctx.author.mention} i have enabled `No Prefix mode`.")

@world.command()
async def disablenoprefix(ctx):
    if await world.is_owner(ctx.author):
        ownerprefix.remove("")
        await ctx.send(f"Hey {ctx.author.mention} i have disabled `No Prefix mode`.")

# -------
# Cogs area
for cog in filter(lambda x: x.endswith(".py"), listdir("cogs/")):
    world.load_extension(f"cogs.{cog[:-3]}")
    print(f"[bold cyan]INFO  [/bold cyan][yellow]COG:[/yellow] [purple]{cog}[/purple] [yellow]is loaded.[/yellow]")

# -------
# Blacklist area
with open("blacklisted.json") as f:
    blacklisted_people = load(f)

# -------
# Message received area
@world.event
async def on_message(message):
    """
    Dispatched every time a message is sent.
    Checks if the message wasn't send by a bot, the message wasn't send
    on DMs and the user isn't blacklisted.
    """
    if message.author.bot or (str(message.author.id) in blacklisted_people):
        return
    elif not message.guild:
        return await message.channel.send((
            "You can't use commands on DMs, invite the bot your server: "
            f"<https://discord.com/oauth2/authorize?client_id={world.user.id}&permissions=8&scope=bot>"
        ))

    await world.process_commands(message)

# -------
# Event area
@world.event
async def on_ready():
    """Dispatched when the bot has successfully connected into Discord."""
    await world.change_presence(status="dnd", activity=Activity(type=2, name="w/help"))
    table = Table()
    table.add_column("[bold white]Bot info[/bold white]", justify="left", style="bold blue", no_wrap=True)
    table.add_row(f"[bold cyan]Name:[/bold cyan] {world.user.name}")
    table.add_row(f"[bold cyan]Latency:[/bold cyan] {float(world.latency*1000):.0f}ms")
    table.add_row(f"[bold cyan]Guilds:[/bold cyan] {len(world.guilds)}")
    console.print(table)

@world.event
async def on_guild_remove(guild):
    prefixes = load(open('prefixes.json', 'r'))
    prefixes.pop(str(guild.id))

    with open('prefixes.json', 'w') as f:
        dump(prefixes, f, indent=4)

#if not opus.is_loaded():
 #   opus.load_opus(find_library("opus"))

world.run(environ["TOKEN"])


import requests
import tempfile
import json
from astropy.io import fits

from .utilities import Filter

class ArchiveApi(object):
    def __init__(self, api_key=None, observer_id=None):
        """Bayfordbury Archive API layer. See documentation here: https://observatory.herts.ac.uk/wiki/API

        api_key, str : your API key

        observer_id, int : your observer ID
        """

        assert api_key is not None, "You need an API key to search and download from Bayfordbury servers. Get one from https://observatory.herts.ac.uk/telescopes/myaccount.php"
        assert observer_id is not None, "You need an Observer ID to search and download from Bayfordbury servers. You can find it at https://observatory.herts.ac.uk/telescopes/myaccount.php"

        self.api_key = api_key
        self.observer_id = observer_id

        self._search_url   = "https://observatory.herts.ac.uk/api/imagesearch.php"
        self._download_url = "https://observatory.herts.ac.uk/api/getfit.php"

    def search(self, ra, dec, radius, minjd=None, maxjd=None, minexp=None, maxexp=None, solved=False, mine=False):
        """Search for images in archive around coordinates, within a certain radius.

        ra, float       : right ascention of the search center, decimal degrees, required.

        dec, float      : declination of the search center, decimal degrees, required.

        radius, float   : search radius, decimal degrees, required.

        minjd, float    : earliest Julian date of the observation, optional.

        maxjd, float    : latest Julian date of the observation, optional.

        minexp, float   : shortest exposure time of the observation, seconds, optional.

        maxexp, float   : longest exposure time of the observation, seconds, optional.

        solved, boolean : must be plate solved, optional.

        mine, boolean   : return only your images.
        """
        assert ra is not None, "ra parameter is required."
        assert dec is not None, "dec parameter is required."
        assert radius is not None, "radius parameter is required."

        # construct a query parameters dictionary
        params = { # required parameters and those with a default value
            "key"    : self.api_key,
            "id"     : self.observer_id,
            "ra"     : ra,
            "dec"    : dec,
            "dist"   : radius,
            "solved" : solved,
            "mine"   : mine
        }

        if minjd is not None:
            params["minjd"] = minjd

        if maxjd is not None:
            params["maxjd"] = maxjd

        if minexp is not None:
            params["minexp"] = minexp

        if maxexp is not None:
            params["maxexp"] = maxexp

        # send the request
        response = requests.get(self._search_url, params)
        self._handle_bad_response(response)

        return self._unpack_search_response(response.content)

    def download(self, image_id, saveto=None):
        """
        Download an image from Bayfordbury Archive.

        image_id, int : database ID in the archive.

        saveto, str : path to save the .fits file to. If None, creates a temporary file (in /tmp if you're using Linux)

        Returns an Astropy FITS image.
        """
        assert isinstance(image_id, int), "image_id must be an integer number."

        # construct a query parameters dictionary
        params = { # required parameters and those with a default value
            "key"    : self.api_key,
            "id"     : self.observer_id,
            "dbid"   : image_id
        }

        # send the request
        response = requests.get(self._download_url, params, stream=True)
        self._handle_bad_response(response)

        # create the file handle
        file_handle = tempfile.NamedTemporaryFile(mode="wb") if saveto is None else open(saveto, "wb")
        # write the bytes to the new file
        with file_handle as handle:
            for block in response.iter_content(1024):
                handle.write(block)
            file_path = file_handle.name if saveto is None else saveto
            file_instance = fits.open(file_path, ignore_missing_end=True, cache=False)

        return file_instance

    def _handle_bad_response(self, response):
        """
        Raises:
            HTTPStatus errors if any.
            PermissionError if credentials are wrong.
            FileNotFoundError if image ID is not in the Bayfordbury archive.
        """
        response.raise_for_status()
        # wrap this in try/catch in case the response is not actual JSON (could be file data, for instance)
        try:
            response_dictionary = json.loads(response.content)
        except:
            return

        if "Exception" in response_dictionary and response_dictionary["Exception"] == True:
            if "verification_failed" in response_dictionary["ExceptionReason"]:
                raise PermissionError("Verification failed for the <observer_id> and <api_key> you provided.")
            elif "image_id_not_found" in response_dictionary["ExceptionReason"]:
                raise FileNotFoundError("File with a supplie ID not found in Bayfordbury archive.")

    def _unpack_search_response(self, response):
        """
        Converts json returned by the server's 'search' API to a list of dictionaries.

        response, byte array : server response

        Returns a list of images found.
        """
        column_names = ["id", "ra", "dec", "dist", "exp", "tel", "bin", "jd", "filter"]
        response_dictionary = json.loads(response)

        if "Exception" in response_dictionary and response_dictionary["Exception"] == True:
            if "no_matching_images" in response_dictionary["ExceptionReason"]:
                return []
            else:
                raise NotImplementedError("Server returned an error <{}>, but it was not handled here ¯\_(ツ)_/¯ .".format(response_dictionary["ExceptionReason"]))

        return [value for key, value in response_dictionary["images"].items()]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Oct 18 09:43:34 2018

@author: adameshel
"""

'''
Implementation of GMZ algorithm.
Based on "Rain Rate Estimation Using Measurements From
Commercial Telecommunications Links"
by Oren Goldshtein, Hagit Messer and Artem Zinevich
'''
# from __future__ import print_function
import numpy as np
from collections import Counter

def apply_inverse_power_law(A, L, a, b):
    ''' variance of the attenuation measurement error.
    Formula (1) from paper.
    A - attenuation
    L - cml length in KM
    a, b - ITU power law parameters
    '''
    return (A/(a*L))**(1.0/b)

def apply_power_law(R, L, a, b):
    ''' variance of the attenuation measurement error.
    Formula (1) from paper.
    R - rain
    L - cml length in KM
    a, b - ITU power law parameters
    '''
    return (R**b)*a*L

def calc_rain_from_atten(df):
    '''calculate the average rain rate at each cml.
    df should contain the following columns:
    A - attenuation due to rain
    L - length of the cmls in KM
    a, b - ITU power law parameters
    '''
    df['R'] = df.apply(lambda cml: apply_inverse_power_law(cml['A'],
                                                           cml['L'],
                                                           cml['a'],
                                                           cml['b']),
                       axis=1)
    return df

def calc_atten_from_rain(df):
    '''calculate the attenuation from the avg. rain.
    df should contain the following columns:
    R - rain
    L - length of the cmls in KM
    a, b - ITU power law parameters
    '''
    df['A'] = df.apply(lambda cml: apply_power_law(cml['R'],
                                                   cml['L'],
                                                   cml['a'],
                                                   cml['b']),
                       axis=1)
    return df

def create_virtual_gauges(df,
                          gauge_length=0.5,
                          num_gauges=None):
    ''' split each cml (a single row of df) into several virtual
    rain gauges.
    gauge_length - the distance between two gauges in KM. Not yot verified.
    num_gauges - number of virtual gauges per cml (overrides gauge_length)
    df should contain the following columns:
    xa - longitude of site A of the cml
    ya - latitude of site A of the cml
    xb - longitude of site B of the cml
    yb - latitude of site B of the cml
    L - length of the cml
    '''
    df = df.copy()
    x_gauges = []
    y_gauges = []
    z_gauges = []

    # split each cml into several virtual rain gauges
    for i, cml in df.iterrows():
        L = cml['L']
        if num_gauges is None:
#            import pdb; pdb.set_trace()
            num_gauges_along_cml = int(np.ceil(L / float(gauge_length)))
#            vrgs_per_cml = 'ed' + str(round(gauge_length,2))
            vrgs_per_cml = 'ED' + str(int(gauge_length * 1e3))
        else:
            num_gauges_along_cml = int(num_gauges)
            vrgs_per_cml = num_gauges
        x, y = get_gauges_lon_lat(cml['xa'], cml['ya'],
                                  cml['xb'], cml['yb'],
                                  G=num_gauges_along_cml)

        # initial rain value for each gauge
        z = tuple([cml['R']] * num_gauges_along_cml)

        x_gauges.append(x)
        y_gauges.append(y)
        z_gauges.append(z)

    # add x, y locations of the virtual rain gauges of each cml
    df['x'] = x_gauges
    df['y'] = y_gauges

    # add initial z (rain rate) of each virtual rain gauge of each cml
    df['z'] = z_gauges
    return df, vrgs_per_cml

def get_gauges_lon_lat(a_lon, a_lat, b_lon, b_lat, G=2):
    """ Calculate and return longitude and latitude
    of G gauges along the cml """
    def GaugeCoords(t):
        return a_lon + t*(b_lon - a_lon), a_lat + t*(b_lat - a_lat)

    LonTuple = []
    LatTuple = []
    for i in range(1, G+1):
        GaugeCoordsTup = GaugeCoords(float(i)/(G+1))
        LonTuple.append(GaugeCoordsTup[0])
        LatTuple.append(GaugeCoordsTup[1])

    return tuple([tuple(LonTuple), tuple(LatTuple)])

def error_variance(A, Q, L, a, b):
    ''' variance of the attenuation measurement error.
    Formula (7) from paper.
    A - attenuation
    Q - quantizatin
    L - length in KM
    a, b - ITU power law parameters
    '''
    if A < 0.001:    # no rain induced attenuation
        A = 0.001

    return ((Q**2)/12.0) * (1/(a*L)) * (1.0/b) * (A**(2.0*(1.0-b)/b))

def calc_grid_weights(D, ROI, method, p_par):
    ''' calculate shepard IDW coefficients/weights (for mapping)
    with radius of influence ROI. Formula (22) from paper. '''
    w = np.zeros(D.shape)
    D[D < 0.001] = 0.001
    if method==0:
        w[D < ROI] = (ROI/D[D < ROI] - 1.0)**p_par
        w[w < 0.0000000000001] = 0.0
    else:
        w[D < ROI] = \
            (ROI**p_par - D[D < ROI]**p_par)/(ROI**p_par + D[D < ROI]**p_par)
        w[w < 0.0000000000001] = 0.0
    return w

class IdwIterative():
    def __init__(self, df_for_dist, xgrid, ygrid, ROI=0.0, max_iterations=1,
                 tolerance=0.0, method=0, p_par=2.0, fixed_gmz_roi=None):
        '''ROI- Radius of Influence in meters. A parameter of IDW'''
        self.ROI = ROI
        self.ROI_gmz = ROI
        self.max_iterations = max_iterations
        self.tolerance = tolerance
        self.method = method # weighting method (0-shepard, 1-cressman)
        self.p_par = p_par # power parameter of IDW
        self.df_for_dist = df_for_dist
        self.xgrid = xgrid
        self.ygrid = ygrid
        self.ROI_prev = None
        self.fixed_gmz_roi = fixed_gmz_roi

        self._calc_all_weights()
        print('Calculation of weights finished')

    def __call__(self, df, quantization, interpolate=True,
                 ROI_for_interp=None):
        ''' Calculates the rain rate at every virtual gauge of every cml
        and uses the results to create a rain map with IDW interpolation.
        df is a pandas DataFrame that should include the following columns:
        x, y - (x,y) location of virtual gauges
        z - initial z value at each virtual gauge
        xa, ya, xb, yb - (x,y) location of the cml's sites
        L - length of each cml (in KM)
        a, b - ITU power law parameters
        A - attenuation due to rain only
        '''
        if ROI_for_interp:
            self.ROI = ROI_for_interp

        # Check whether the same links are available for weights calculations
        if df['Link_num'].equals(self.df_for_dist['Link_num']):
            if ROI_for_interp:
                if self.ROI == self.ROI_prev:
                    print('Reusing precalculated weights')
                else:
                    print('Recalculating weights because' +\
                          ' CML IDs in input DataFrame did change')
                    self.df_for_dist = df
                    self._calc_all_weights()
        else:
            print('Recalculating weights because' +\
                  ' CML IDs in input DataFrame did change')
            self.df_for_dist = df
            self._calc_all_weights()

        if ROI_for_interp:
            self.ROI_prev = ROI_for_interp

        # each vector contains data for all the gauges of all the links
        self.df = df
        self.Q = quantization
#        import pdb; pdb.set_trace()
        self.df['num_of_gauges'] = self.df['x'].apply(len)
        # the grid points and Z values at each point (initialized to 0)
        self.Z = np.zeros((self.xgrid.shape[0], self.xgrid.shape[1]))

        # create vectors with x,y,z values of all the virtual gauges
        # self.gauges_z = np.zeros(self.use_gauges.shape)
        self.gauges_z = np.empty(self.use_gauges.shape)
        self.gauges_z[:] = np.nan

        for cml_index, cml in self.df.iterrows():
            gauges = cml['num_of_gauges']
            self.gauges_z[cml_index, :gauges] = cml['z']

        self.gauges_z_prev = self.gauges_z.copy()

        # calculate the measurement error variance for each cml
        # these variance values do not change during each iteration
        self.df['variance'] = df.apply(lambda cml: error_variance(cml['A'],
                                                                  self.Q,
                                                                  cml['L'],
                                                                  cml['a'],
                                                                  cml['b']),
                                       axis=1) # CHECK

        # calculate the radius of influence
        if self.ROI == 0.0:
            self.calc_ROI()

        # run multiple iterations to find the virtual gauge values
        self.dz_vec = []
        dz = np.inf  # change in virtual gauge values

        for i in range(self.max_iterations):
#            print('Running iteration {}'.format(i))

            # perform a single iteration on all the cmls
            self.calc_cmls_from_other_cmls()

            if self.tolerance > 0.0:
                diff_norm = np.linalg.norm(self.gauges_z - self.gauges_z_prev,
                                           axis=None)
                prev_norm = np.linalg.norm(self.gauges_z_prev, axis=None)
                dz = float(diff_norm)/(prev_norm + 1e-10)
                self.dz_vec.append(dz)
#                print('Norm change in Z values: {}'.format(dz))
                if dz <= self.tolerance:
                    break

            # update the previous z values for the next iteration
            self.gauges_z_prev[:, :] = self.gauges_z

        # calculate value at each grid point using the data from the cmls
        if interpolate == True:
            print('Interpolating\n')
            self.calc_grid_from_cmls()

        print('Processing finished.\n')
        return self.Z

    def calc_ROI(self):
        ''' Calculate the radius of influence by measuring the largest
        distance between any two cmls. '''

        # calculate the coordinates of each cml's center point
        cml_x_middle = self.df['x'].apply(lambda x: np.mean(x)).values
        cml_y_middle = self.df['y'].apply(lambda y: np.mean(y)).values

        cml_coords_middle = np.concatenate((cml_x_middle.reshape(-1, 1),
                                            cml_y_middle.reshape(-1, 1)),
                                           axis=1)
        dests = cml_coords_middle

        # calculate the distance between every pair of cmls
        subts = cml_coords_middle[:, None, :] - dests
        cml_distances = np.sqrt(np.einsum('ijk,ijk->ij', subts, subts))

        # use the max distance between any two cmls as the ROI
        self.ROI = cml_distances.max(axis=None)

    def calc_cmls_from_other_cmls(self):
        ''' Isolate one cml at a time and compute the influence of other cmls
        in its radius of influence to calculate the rain rate at each of
        the cml's virtual gauges. '''

        # compute the rain rate at each gauge of a cml (with index cml_i)
        # using the rain rate of virtual gauges from all the OTHER cmls
        cml_prev = self.cmls_vg_with_neighbors_list[0][0]
        cml_num_of_gauges = self.df.iloc[cml_prev]['num_of_gauges']
        cml_new_z = np.zeros((cml_num_of_gauges,))
        last_i = len(self.cmls_vg_with_neighbors_list)-1
#        import pdb; pdb.set_trace()
        for i, cml_vg in enumerate(self.cmls_vg_with_neighbors_list):
            # Apply formula (20) from paper to compute the rain rate vector
            if cml_vg[0] != cml_prev:
                K = cml_num_of_gauges
                R = self.df.iloc[cml_prev]['R']
                b = self.df.iloc[cml_prev]['b']
                theta = cml_new_z**b
                r = (R**b - (1.0/K)*np.sum(theta) + theta)
                # print(np.sum(r[r < 0.0]))
    #            import pdb; pdb.set_trace()
                r[r < 0.0] = 0.0  # set negative rain rates to 0

                # update the new z values at the cml's virtual gauges
                self.gauges_z[cml_prev, :cml_num_of_gauges] = r**(1.0/b)
                # initial new virtual gauge rain vector for current cml
                cml_num_of_gauges = self.df.iloc[cml_vg[0]]['num_of_gauges']
                cml_new_z = np.zeros((cml_num_of_gauges,))
                cml_prev = cml_vg[0]

            # variances and number of gauges of cmls in ROI
            neighbor_cmls = []
            for _, k in enumerate(self.cml_vg_roi_list_all[i]):
                neighbor_cmls.append(k[0])
#                gauges_in_ROI.append(k[1])
            neighbor_cmls = list(set(neighbor_cmls))
            variances = self.df['variance'][neighbor_cmls].values

            # Initialize a covariance matrix to zero
            M = len(self.cml_vg_roi_list_all[i])  # total number of gauges in ROI
            cov = np.zeros((M, M))

            neighbor_cmls_count_temp = [q[0] for q in self.cml_vg_roi_list_all[i]]
            count = Counter(neighbor_cmls_count_temp)
            dictionary_items = count.items()
            d_count = dict(sorted(dictionary_items))

            # add measurement quantization error to the cov matrix
            stop = 0
            for ic, c in enumerate(d_count):
            # for tt, sigma in enumerate(variances):
                start = stop
                stop = start + d_count[c]
                cov[start:stop, start:stop] = variances[ic]
            # add IDW weights to covariance matrix
            z = 1.0
            W = z*np.diagflat(1.0/self.weights_vg_list[i])  # 1/weights on diagonal
            cov = cov + W

            # compute inverse of covariance matrix
            cov_inv = np.linalg.inv(cov)
            cov_inv_sum = cov_inv.sum(axis=None)  # sum of all elements
            cov_inv_col_sum = cov_inv.sum(axis=0)
            prev_theta = np.take(self.gauges_z_prev, \
                                 self.cml_vg_roi_single_i_list_all[i])
            nominator = (cov_inv_col_sum * prev_theta).sum()
#            if i==2:
#                import pdb; pdb.set_trace()
            cml_new_z[cml_vg[1]] = nominator / cov_inv_sum

            # Apply formula (20) from paper to compute the rain rate vector
#            print(cml_i)
            if i == last_i:
                K = cml_num_of_gauges
                R = self.df.iloc[cml_vg[0]]['R']
                b = self.df.iloc[cml_vg[0]]['b']
                theta = cml_new_z**b
                r = (R**b - (1.0/K)*np.sum(theta) + theta)
                r[r < 0.0] = 0.0  # set negative rain rates to 0
                # update the new z values at the cml's virtual gauges
                self.gauges_z[cml_vg[0], :cml_num_of_gauges] = r**(1.0/b)

    def calc_grid_from_cmls(self):
        ''' calculate the z values of each grid point using z values of the
        virtual gauges '''
        Z_flat = self.Z.flatten()
        # use the cmls to caluculate the rain rate at each (x,y) grid point
        for i, p_i in enumerate(self.pixels_with_neighbors_list):
            gauges_i = np.take(self.gauges_z,
                               self.idx_pxl_single_i_list_all[i])
#            import pdb; pdb.set_trace()
            Z_flat[p_i] = (self.weights_pxl_list_all[i] * gauges_i).sum()/\
            self.sum_of_weights[i]
#            else:
        Z_flat[Z_flat <= 0] = np.nan
        self.Z = Z_flat.reshape((self.xgrid.shape[0], self.xgrid.shape[1]))

    def _calc_all_weights(self):
        # gmz calcs:
        self.num_cmls = self.df_for_dist.shape[0]
        # compute the number of virtual gauges for each cml
        self.df_for_dist['num_of_gauges'] = self.df_for_dist['x'].apply(len)
        self.max_num_of_gauges = self.df_for_dist['num_of_gauges'].max()
        self.use_gauges = np.zeros((self.num_cmls, self.max_num_of_gauges),
                                   dtype=bool)
        # self.gauges_x = np.zeros(self.use_gauges.shape)
        # self.gauges_y = np.zeros(self.use_gauges.shape)
        self.gauges_x = np.empty(self.use_gauges.shape)
        self.gauges_x[:] = np.nan
        self.gauges_y = np.empty(self.use_gauges.shape)
        self.gauges_y[:] = np.nan

        for cml_index, cml in self.df_for_dist.iterrows():
            gauges = cml['num_of_gauges']
            self.use_gauges[cml_index, :gauges] = True
            self.gauges_x[cml_index, :gauges] = cml['x']
            self.gauges_y[cml_index, :gauges] = cml['y']

        self.cmls_with_neighbors_list = []
        self.cmls_vg_with_neighbors_list = []
        self.cml_vg_roi_list_all = []
        self.cml_vg_roi_single_i_list_all = []
        self.weights_vg_list = []
        for cml_i, cml in self.df_for_dist.iterrows():    # loop over cmls
            cml_gx = cml['x']   # x position of virtual gauges
            cml_gy = cml['y']   # y position of virtual gauges

            # initial new virtual gauge rain vector for current cml
            cml_num_of_gauges = cml['num_of_gauges']

            # loop over current cml's virtual gauges
            for gauge_i in range(cml_num_of_gauges):
                cml_vg_roi_list = []
                cml_vg_roi_single_i_list = []
                gx = cml_gx[gauge_i]  # x position of current gauge
                gy = cml_gy[gauge_i]  # y position of currnet gauge

                # calculate distance of current gauge from all gauges
                distances = np.sqrt((self.gauges_x - gx)**2.0 +
                                    (self.gauges_y - gy)**2.0)
                ## calculate IDW weights
                if self.fixed_gmz_roi:
                    weights = calc_grid_weights(distances,
                                                self.fixed_gmz_roi,
                                                self.method,
                                                self.p_par)

                else:
                    weights = calc_grid_weights(distances,
                                                self.ROI,
                                                self.method,
                                                self.p_par)
                weights = weights * self.use_gauges
                weights[cml_i, :] = 0.0  # remove weights for current cml
                weights[weights < weights.max()/100.0] = 0.0

                # find the indices of cmls in the current cml's ROI
                cmls_in_ROI = (weights.sum(axis=1) > 0.0)
                gauges_in_ROI = (weights > 0.0)

                # exclude current cml from all further calculations
                cmls_in_ROI[cml_i] = False
                gauges_in_ROI[cml_i, :] = False

                # select the indices of cml gauges (only in the ROI)
                select_gauges = gauges_in_ROI * self.use_gauges
                if select_gauges.sum() != 0:
                    self.cmls_vg_with_neighbors_list.append((cml_i, gauge_i))
                    for row in range(len(select_gauges[:,0])):
                        if select_gauges[row,:].sum() != 0:
                            for col in range(self.max_num_of_gauges):
                                if select_gauges[row,col].sum() != 0: # CAN REMOVE SUM
                                    cml_vg_roi_list.append((row,col))
                                    cml_vg_roi_single_i_list.\
                                    append(row*self.max_num_of_gauges + col)
                # add IDW weights to covariance matrix
#                z = 0.5
                self.cml_vg_roi_single_i_list_all.append(
                    cml_vg_roi_single_i_list
                    )
                self.cml_vg_roi_single_i_list_all = \
                list(filter(None, self.cml_vg_roi_single_i_list_all))
                self.cml_vg_roi_list_all.append(cml_vg_roi_list)
                self.cml_vg_roi_list_all = list(
                    filter(None,
                           self.cml_vg_roi_list_all)
                    )
                weights_vector = weights[select_gauges].flatten()
                if len(weights_vector) != 0:
                    weights_vector /= weights_vector.sum(axis=None)  # Normalize
                    self.weights_vg_list.append(weights_vector)

        # for idw pixels calcs
        self.pixels_with_neighbors_list = []
        self.idx_weights_pxl_list_all = []
        self.weights_pxl_list_all = []
        self.idx_pxl_single_i_list_all = []
        for p_i in range(len(self.xgrid.flatten())):
            px = self.xgrid.flatten()[p_i]
            py = self.ygrid.flatten()[p_i]
#            import pdb; pdb.set_trace()
            # perform basic IDW
            idx_weights_pxl_list = []
            weights_pxl_list = []
            idx_pxl_single_i_list = []
            dist_pxl = np.sqrt((self.gauges_x-px)**2 + (self.gauges_y-py)**2)
            if self.fixed_gmz_roi:
                weights_pxl = calc_grid_weights(dist_pxl,
                                                self.fixed_gmz_roi,
                                                self.method,
                                                self.p_par)
            else:
                weights_pxl = calc_grid_weights(dist_pxl,
                                                self.ROI,
                                                self.method,
                                                self.p_par)
            weights_pxl[np.bitwise_not(self.use_gauges)] = 0.0 # Check!
            if weights_pxl.sum() != 0:
                    self.pixels_with_neighbors_list.append(p_i)
                    for row in range(len(weights_pxl[:,0])):
                        if weights_pxl[row,:].sum() != 0:
                            for col in range(self.max_num_of_gauges):
                                if weights_pxl[row,col].sum() != 0: # CAN REMOVE SUM
                                    idx_weights_pxl_list.append((row,col))
                                    weights_pxl_list.append(
                                        weights_pxl[row,col]
                                        )
                                    idx_pxl_single_i_list.\
                                    append(row*self.max_num_of_gauges + col)
            self.idx_weights_pxl_list_all.append(idx_weights_pxl_list)
            self.idx_weights_pxl_list_all = list(
                filter(None, self.idx_weights_pxl_list_all)
                )
            self.weights_pxl_list_all.append(weights_pxl_list)
            self.weights_pxl_list_all = list(
                filter(None,self.weights_pxl_list_all)
                )
            self.idx_pxl_single_i_list_all.append(idx_pxl_single_i_list)
            self.idx_pxl_single_i_list_all = \
                list(filter(None, self.idx_pxl_single_i_list_all))

        self.sum_of_weights = []
        for i, w_vec_pxl in enumerate(self.weights_pxl_list_all):
            w_vec_pxl = np.array(w_vec_pxl)
            self.sum_of_weights.append(w_vec_pxl.sum(axis=None))


# -*- coding: utf-8 -*-
"""
Created on Mon Mar  2 22:38:31 2020

@author: allen
"""

import argparse
import requests
import json
import math
import operator

def conv4326ToXY3414(Lat,Lng):
    #This function converts google's destLat & destLng to 4326 to 3414
    #More info: https://docs.onemap.sg/#4326-wgs84-to-3857
    urlOnemapHeadTo3414 = "https://developers.onemap.sg/commonapi/convert/4326to3414?"
    latlng = "latitude=" + str(Lat) + "&longitude=" + str(Lng)
    urlOnemapTo3414 = urlOnemapHeadTo3414 + latlng
    fileobj = requests.get(urlOnemapTo3414)
    convToXY3414 = json.loads(fileobj.text)
    p1 = [convToXY3414["X"],convToXY3414["Y"]]
    return p1

def getDistancefromXYcoord(p1,p2):
    dist = math.sqrt(((p1[0]-p2[0])**2)+((p1[1]-p2[1])**2))
    return dist

def parseUserInput():
    parser = argparse.ArgumentParser()
    parser.add_argument("destLat", type=float)
    parser.add_argument("destLng", type=float)
    args = parser.parse_args()

    destLat = args.destLat
    destLng = args.destLng
    destLatLngObj = {
        "destLat" : destLat,
        "destLng" : destLng
    }
    return destLatLngObj

def httpRequestForHDBCarpark():
    #website: https://data.gov.sg/dataset/hdb-carpark-information
    urlCarpark = "https://data.gov.sg/api/action/datastore_search?resource_id=139a3035-e624-4f56-b63f-89ae28d4ae4c&limit=10000"
    fileobj = requests.get(urlCarpark)
    carparkDB = json.loads(fileobj.text) #this function should only run once every period and store in a db
    with open('HDBCarpark_DB.txt', 'w') as my_data_file:
        my_data_file.write(fileobj.text)
    return

def httpRequestForCarparkShoppingMallAndHotel():
    urlCarpark = "https://data.gov.sg/api/action/datastore_search?resource_id=85207289-6ae7-4a56-9066-e6090a3684a5&limit=10000"
    fileobj = requests.get(urlCarpark)
    carparkDB = json.loads(fileobj.text) #this function should only run once every period and store in a db
    with open('CarparkShoppingMallAndHotel(original).txt', 'w') as my_data_file:
        my_data_file.write(fileobj.text)
    return

def httpRequestForURACarpark():
    urlCarpark = "https://data.gov.sg/api/action/datastore_search?resource_id=5a8f7572-0d46-4ce5-be71-a7081d2c42b0&limit=10000"
    fileobj = requests.get(urlCarpark)
    carparkDB = json.loads(fileobj.text) #this function should only run once every period and store in a db
    with open('URACarpark_DB.txt', 'w') as my_data_file:
        my_data_file.write(fileobj.text)
    return

def fromTXTHDBCarpark():
    carparkDB = open('HDBCarpark_DB.txt', 'r')
    carparkDB = json.loads(carparkDB.read())
    return carparkDB

def fromTXTCarparkShoppingMallAndHotel():
    carparkDB = open('CarparkShoppingMallAndHotel_DB.txt', 'r')
    carparkDB = json.loads(carparkDB.read())
    return carparkDB

def fromTXTCarparkURA():
    carparkDB = open('URACarpark_DB(Motor Car).txt', 'r')
    carparkDB = json.loads(carparkDB.read())
    return carparkDB

#depreciated
def insertXYCoordToCarparkDB(carparkDB):
    #Key in your Google API key here
    apiKey = 'YOUR GOOGLE API KEY'
    geocodeRequest = 'https://maps.googleapis.com/maps/api/geocode/json?address='
    keyCode = '&key='
    noresultsIDArray = []
    for num in range(len(carparkDB["result"]["records"])):
        print(num)
        inputAddress = carparkDB["result"]["records"][num]["carpark"]
        inputAddress = inputAddress.replace(" ","+")
        googleHttpRequest = geocodeRequest + inputAddress + keyCode + apiKey
        googleMapResponseObj = requests.post(googleHttpRequest).text
        googleMapResponseObj = json.loads(googleMapResponseObj)
        if googleMapResponseObj["status"]=='ZERO_RESULTS':
            noresultsIDArray.append(carparkDB["result"]["records"][num]["_id"])
            print(noresultsIDArray)
        else:
            Lat = googleMapResponseObj["results"][0]["geometry"]["location"]["lat"]
            Lng = googleMapResponseObj["results"][0]["geometry"]["location"]["lng"]
            p1 = conv4326ToXY3414(Lat,Lng)
            xyCoordObj = {
                "x_coord": p1[0],
                "y_coord": p1[1]
            }
            carparkDB["result"]["records"][num].update(xyCoordObj)

    # print(carparkDB["result"]["records"])
    carparkDBtoStr = json.dumps(carparkDB)
    with open('CarparkShoppingMallAndHotel_DB.txt', 'w') as my_data_file:
        my_data_file.write(carparkDBtoStr)

    # loadedtxt = open('data.txt', 'r')
    # loadedtxt = json.loads(loadedtxt.read())
    # print(type(loadedtxt))

    noresultsIDArray = json.dumps(noresultsIDArray)
    with open('noresultsIDArray.txt', 'w') as my_data_file:
        my_data_file.write(noresultsIDArray)
    return

#this 2 are run together after function httpRequestForCarparkShoppingMallAndHotel() and then insertXYCoordToCarparkDB(carparkDB). CarparkDB for shopping mall is ready to be used for nearest carpark
def deleteNoGoogleResultRoutine(notepadRead,notepadWrite,delArrayID):
    carparkDB = open(notepadRead, 'r')
    carparkDB = json.loads(carparkDB.read())

    for index in range(len(delArrayID)):
        idValue = delArrayID[index]
        for num in range(len(carparkDB["result"]["records"])):
            if carparkDB["result"]["records"][num]["_id"] == idValue:
                print(num)
                print(carparkDB["result"]["records"][num])
                del carparkDB["result"]["records"][num]
                break

    carparkDB = json.dumps(carparkDB)
    with open(notepadWrite, 'w') as my_data_file:
        my_data_file.write(carparkDB)
    return
def fillXYCoordSubRoutine(carparkDB,carparkKeyValue,noresultsIDArray,notepadWrite,notepadWriteNoResult1stArray,notepadWriteNoResult2ndArray):
    apiKey = 'YOUR GOOGLE API KEY'
    geocodeRequest = 'https://maps.googleapis.com/maps/api/geocode/json?address='
    keyCode = '&key='

    noresultsIDArray2ndRound = []
    if len(noresultsIDArray) > 0:
        for index in range(len(noresultsIDArray)):
            print("this should not be running!")
            idValue = noresultsIDArray[index]
            for num in range(len(carparkDB["result"]["records"])):
                if carparkDB["result"]["records"][num]["_id"] == idValue:
                    inputAddress = carparkDB["result"]["records"][num][carparkKeyValue]
                    inputAddress = inputAddress.replace(" ","+")
                    googleHttpRequest = geocodeRequest + inputAddress + keyCode + apiKey
                    googleMapResponseObj = requests.post(googleHttpRequest).text
                    googleMapResponseObj = json.loads(googleMapResponseObj)

                    if googleMapResponseObj["status"]=='ZERO_RESULTS':
                        noresultsIDArray2ndRound.append(carparkDB["result"]["records"][num]["_id"])
                        print(noresultsIDArray2ndRound)
                    else:
                        Lat = googleMapResponseObj["results"][0]["geometry"]["location"]["lat"]
                        Lng = googleMapResponseObj["results"][0]["geometry"]["location"]["lng"]
                        p1 = conv4326ToXY3414(Lat,Lng)
                        xyCoordObj = {
                            "x_coord": p1[0],
                            "y_coord": p1[1]
                        }
                        carparkDB["result"]["records"][num].update(xyCoordObj)
    else:
        for num in range(len(carparkDB["result"]["records"])):
            inputAddress = carparkDB["result"]["records"][num][carparkKeyValue]
            inputAddress = inputAddress.replace(" ","+")
            googleHttpRequest = geocodeRequest + inputAddress + keyCode + apiKey
            googleMapResponseObj = requests.post(googleHttpRequest).text
            googleMapResponseObj = json.loads(googleMapResponseObj)

            if googleMapResponseObj["status"]=='ZERO_RESULTS':
                noresultsIDArray.append(carparkDB["result"]["records"][num]["_id"])
                print(noresultsIDArray)
            else:
                Lat = googleMapResponseObj["results"][0]["geometry"]["location"]["lat"]
                Lng = googleMapResponseObj["results"][0]["geometry"]["location"]["lng"]
                p1 = conv4326ToXY3414(Lat,Lng)
                xyCoordObj = {
                    "x_coord": p1[0],
                    "y_coord": p1[1]
                }
                carparkDB["result"]["records"][num].update(xyCoordObj)
                print(carparkDB["result"]["records"][num])

    carparkDB = json.dumps(carparkDB)
    with open(notepadWrite, 'w') as my_data_file:
        my_data_file.write(carparkDB)

    if len(noresultsIDArray) > 0:
        noresultsIDArray = json.dumps(noresultsIDArray)
        with open(notepadWriteNoResult1stArray, 'w') as my_data_file:
            my_data_file.write(noresultsIDArray)

    if len(noresultsIDArray2ndRound) > 0:
        with open(notepadWriteNoResult2ndArray, 'w') as my_data_file:
            my_data_file.write(noresultsIDArray2ndRound)

    return
def fillXYCoordMain(notepadRead,notepadWrite,notepadWriteNoResult1stArray,carparkKeyValue,notepadWriteNoResult2ndArray):
    carparkDB = open(notepadRead, 'r')
    carparkDB = json.loads(carparkDB.read())
    noresultsID1stArray = []
    #line below should be run at the start and then comment out
    # fillXYCoordSubRoutine(carparkDB,carparkKeyValue,noresultsID1stArray,notepadWrite,notepadWriteNoResult1stArray,notepadWriteNoResult2ndArray)

    #3 lines below should be run after line at the top
    delArrayID = open(notepadWriteNoResult2ndArray, 'r')
    delArrayID = json.loads(delArrayID.read())
    deleteNoGoogleResultRoutine(notepadRead,notepadWrite,delArrayID)

    return
# fillXYCoordMain("URACarpark_DB(Motor Car).txt","URACarpark_DB(Motor Car)(XY).txt","noresultsIDArrayURA1st.txt","ura_carpark_name","noresultsIDArrayURA2nd.txt")
#for running delete code
# fillXYCoordMain("URACarpark_DB(Motor Car)(XY).txt","URACarpark_DB(Motor Car)(XY)- After Delete.txt","noresultsIDArrayURA1st.txt","ura_carpark_name","noresultsIDArrayURA2nd.txt")

#this 3 function below will run together to filter motorcar carparks from original database
def loop(carparkDB,key,value):
    print(len(carparkDB["result"]["records"]))
    for index, carparks in enumerate(carparkDB["result"]["records"]):
        if type(value) == str:
            if value not in carparks[key]:
                # print(carparkDB["result"]["records"][index])
                del carparkDB["result"]["records"][index]
                bDel = 1
            else:
                bDel = 0
        else:
            if  carparks[key] == str(value):
                print(carparkDB["result"]["records"][index])
                del carparkDB["result"]["records"][index]
                bDel = 1
            else:
                bDel = 0

    print(len(carparkDB["result"]["records"]))

    obj = {
        "carparkDB": carparkDB,
        "bDel": bDel
    }
    return obj
def filterURACarpark(notepadRead,key,value,notepadWrite):
    carparkDB = open(notepadRead, 'r')
    carparkDB = json.loads(carparkDB.read())
    bDel = 1
    while bDel != 0:
        obj = loop(carparkDB,key,value)
        carparkDB = obj["carparkDB"]
        bDel = obj["bDel"]

    carparkDB = json.dumps(carparkDB)
    with open(notepadWrite, 'w') as my_data_file:
        my_data_file.write(carparkDB)
    return
def filterURACarparkOverall():
    filterURACarpark('URACarpark_DB.txt',"type_of_lot","Motor Car",'URACarpark_DB(Motor Car).txt')
    filterURACarpark('URACarpark_DB(Motor Car).txt',"number_of_lots",0,'URACarpark_DB(Motor Car).txt')
    return

def combineAllCarparksDB():
    carparkDBURA = open('URACarpark_DB(Motor Car)(XY)- After Delete.txt', 'r')
    carparkDBURA = json.loads(carparkDBURA.read())

    carparkDBHDB = open('HDBCarpark_DB.txt', 'r')
    carparkDBHDB = json.loads(carparkDBHDB.read())

    carparkDBSPM = open('CarparkShoppingMallAndHotel_DB(XY).txt', 'r')
    carparkDBSPM = json.loads(carparkDBSPM.read())

    combinedDB = {
        "result": {"records":[]}
    }

    combinedDB["result"]["records"].extend(carparkDBURA["result"]["records"])
    combinedDB["result"]["records"].extend(carparkDBHDB["result"]["records"])
    combinedDB["result"]["records"].extend(carparkDBSPM["result"]["records"])
    combinedDB["result"]["total_records"] = len(combinedDB["result"]["records"])

    combinedDB = json.dumps(combinedDB)
    with open("combinedCarparkDB.txt", 'w') as my_data_file:
        my_data_file.write(combinedDB)
    return

def returnNearestCarpark(carparkDB):
    #for testing, comment 3 lines below
    destLatLngObj = parseUserInput()
    destLat = destLatLngObj["destLat"]
    destLng = destLatLngObj["destLng"]

    # #for testing
    # destLat = 1.3602298
    # destLng = 103.8595684
    p1 = conv4326ToXY3414(destLat,destLng)

    #Long = X, Lat = Y
    #This function finds the distance between p2 (x,y coord for HDB carpark DB) with p1 (destination's x,y coord)
    distDict = {}
    for num in range(len(carparkDB["result"]["records"])):
        xCoord2 = float(carparkDB["result"]["records"][num]["x_coord"])
        yCoord2 = float(carparkDB["result"]["records"][num]["y_coord"])
        p2 = [xCoord2, yCoord2]
        dist = getDistancefromXYcoord(p1,p2)
        distDict[num] = round(dist)

    #This function sort distance (value) with their carpark (keys) with smallest distance at the top
    distDict = sorted(distDict.items(), key=operator.itemgetter(1))
    distDict = dict((x, y) for x, y in distDict)
    #distKeysArray will store only the keys to be used later to reference and get the particular carpark name
    distKeysArray = list(distDict.keys())

    nearestCarpark = {"carpark_details" : {}}
    for num in range(0,5):
        index = distKeysArray[num]
        #For getting carpark address
        if "address" in carparkDB["result"]["records"][index]:
            carparkAddress = str(carparkDB["result"]["records"][index]["address"])
            #to also additionally get HDB parking rates
            carparkRate = "$0.60 or $1.20 per half-hour* (More info here: https://www.hdb.gov.sg/cs/infoweb/car-parks/short-term-parking/short-term-parking-charges)"
        elif "carpark" in carparkDB["result"]["records"][index]:
            carparkAddress = str(carparkDB["result"]["records"][index]["carpark"])
            #to also additionally get SPM parking rates
            carparkRate1 = carparkDB["result"]["records"][index]["weekdays_rate_1"]
            carparkRate2 = carparkDB["result"]["records"][index]["weekdays_rate_2"]

            if (carparkRate2 == "-" or carparkRate2 == carparkRate1):
                carparkRate = carparkRate1
            else:
                carparkRate = carparkRate1 + "AND" + carparkRate2

        elif "ura_carpark_name" in carparkDB["result"]["records"][index]:
            carparkAddress = str(carparkDB["result"]["records"][index]["ura_carpark_name"])
            #to also additionally get URA parking rates
            carparkRate = "$0.60 or $1.20 per half-hour* (More info here: https://www.ura.gov.sg/Corporate/Car-Parks/Short-Term-Parking/Parking-coupons"

        # For calculating distance
        distanceFromDest = str(distDict.get(index))
        xCoord = float(carparkDB["result"]["records"][index]["x_coord"])
        yCoord = float(carparkDB["result"]["records"][index]["y_coord"])
        urlOnemapTo3857 = "https://developers.onemap.sg/commonapi/convert/3414to3857?"
        XandY = "X=" + str(xCoord) + "&Y=" + str(yCoord)
        urlOnemapTo3857 = urlOnemapTo3857 + XandY
        fileobj = requests.get(urlOnemapTo3857)
        convTo3857 = json.loads(fileobj.text)
        lng = convTo3857["X"]
        lat = convTo3857["Y"]
        nearestCarpark["carpark_details"][num] = {"address": str(carparkAddress), "lat": str(lat), "lng": str(lng), "distance_from_dest": str(distanceFromDest+'m'),"carpark_rates": str(carparkRate)}

    nearestCarpark = json.dumps(nearestCarpark).replace('\'', '"')
    print(nearestCarpark)
    return

carparkDB = open('combinedCarparkDB.txt','r')
carparkDB = json.loads(carparkDB.read())
returnNearestCarpark(carparkDB)

# this 2 function should be run periodically once every month to refresh data from govtech
# httpRequestForHDBCarpark()
# httpRequestForCarparkShoppingMallAndHotel()


from django.contrib.auth.decorators import login_required
from django.views.generic import FormView, ListView
from aircraft.models import NewPlaneMaster,AircraftModel
from logbook.models import FlightTime
from user.views import getuserid
from dal import autocomplete
from aircraft.forms import AirplaneEntry
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect
from django.views.generic.base import TemplateView
from django.views.generic import FormView, DetailView
from django.views.generic.list import ListView
from aircraft.models import AircraftModel, Manufacture, NewPlaneMaster
from airport.views import airporthome
from logbook.models import FlightTime
from user.models import Users
from django.contrib.auth.models import User
from django_currentuser.middleware import (
    get_current_user)
from user.views import getuserid
from airline.views import flightaware
from django.core.paginator import Paginator
from django.db.models import Count,Sum,Avg
from django.db.models.functions import Round
from django.db.models import Q
from django.http import HttpResponse

@login_required(login_url='/')

def aircrafthome(request):
    print('hello')

class CategoryDisplayAll(TemplateView):
    template_name = 'aircraft/aircraftbase.html'
    def get(self, request, *args, **kwargs):
        pagetitle = "Category Report"
        qs = []
        userid = getuserid()
        manufacture = Manufacture.objects.all().values()
        #this is used to get the category totals, you can also select the manufacture totals dropdown
        #gets all the airplanes
        uniquetype = FlightTime.objects.filter(userid=userid).values('aircrafttype').distinct()
        for airtype in uniquetype:
            info = {'aircraftsummary': FlightTime.objects.filter(userid=userid,aircrafttype=airtype['aircrafttype']).aggregate(
                airtotal = Sum('total'),
                avgflight = Avg('total'),
                numflight = Count('total'),
                avgmiles = Round(Avg('distance')),
                totalmiles = Sum('distance'),
                avgpax = Round(Avg('passengercount')),
                totalpax = Sum('passengercount'),
                night = Sum('night'),
                landings = Sum('landings'),
            )}
            lastflown = FlightTime.objects.filter(userid=userid,aircrafttype=airtype['aircrafttype']).exclude(scheduledflight=1).latest('flightdate')

            record = {'aircraftype':airtype['aircrafttype'],'info':{'aircraftsummary':{'airtotal':info['aircraftsummary']['airtotal'],'avgflight':round((info['aircraftsummary']['avgflight']),2),'numflight':info['aircraftsummary']['numflight'],'avgmiles':int(round((info['aircraftsummary']['avgmiles']),0)),'totalmiles':info['aircraftsummary']['totalmiles'],'avgpax':int(round((info['aircraftsummary']['avgpax']),0)),'totalpax':info['aircraftsummary']['totalpax'],'night':info['aircraftsummary']['night'],'landings':info['aircraftsummary']['landings']}},'lastflown':lastflown}

            if record['info']['aircraftsummary']['airtotal'] > 0:
                qs.append(record)

        return self.render_to_response({"manufacture":manufacture,"qs":qs, "title":pagetitle})
class CategoryDisplay(TemplateView):
    template_name = 'aircraft/categorydisplay.html'
    def get(self, request, *args, **kwargs):
        pagetitle = "Category Report"
        qs = []
        userid = getuserid()
        manufacture = Manufacture.objects.all().values()
        #this is used to get the category totals, you can also select the manufacture totals dropdown

        id = request.GET.get('category')
        uniquetype = AircraftModel.objects.filter(mfr=id)
        for airtype in uniquetype:
            checkingfornone = FlightTime.objects.filter(userid=userid,aircrafttype=airtype).aggregate(Sum('total'))

            if checkingfornone['total__sum'] != None:
                info = {'aircraftsummary': FlightTime.objects.filter(userid=userid,aircrafttype=airtype).aggregate(
                    airtotal = Sum('total'),
                    avgflight = Avg('total'),
                    numflight = Count('total'),
                    avgmiles = Avg('distance'),
                    totalmiles = Sum('distance'),
                    avgpax = Avg('passengercount'),
                    totalpax = Sum('passengercount'),
                    night = Sum('night'),
                    landings = Sum('landings'),
                )}
                lastflown = FlightTime.objects.filter(userid=userid,aircrafttype=airtype).exclude(scheduledflight=1).latest('flightdate')

                #making it so that i can round the results the way I want
                #them before appending them to the list to send them to the
                #template in the context
                record = {'aircraftype':airtype,'info':{'aircraftsummary':{'airtotal':info['aircraftsummary']['airtotal'],'avgflight':round((info['aircraftsummary']['avgflight']),2),'numflight':info['aircraftsummary']['numflight'],'avgmiles':int(round((info['aircraftsummary']['avgmiles']),0)),'totalmiles':info['aircraftsummary']['totalmiles'],'avgpax':int(round((info['aircraftsummary']['avgpax']),0)),'totalpax':info['aircraftsummary']['totalpax'],'night':info['aircraftsummary']['night'],'landings':info['aircraftsummary']['landings']}},'lastflown':lastflown}
                if record['info']['aircraftsummary']['airtotal'] != None:
                    qs.append(record)
        tairtotal = 0; tavgflight = 0; tnumflight = 0; tavgmiles = 0
        ttotalmiles = 0; tavgpax = 0; ttotalpax = 0; count = 0; tnight = 0; tlandings = 0;
        #adding the totals for the categories
        for itemtotal in qs:
            tairtotal = tairtotal + (itemtotal['info']['aircraftsummary']['airtotal'])
            tavgflight = tavgflight + (itemtotal['info']['aircraftsummary']['avgflight'])
            tnumflight = tnumflight + (itemtotal['info']['aircraftsummary']['numflight'])
            tavgmiles = tavgmiles + (itemtotal['info']['aircraftsummary']['avgmiles'])
            ttotalmiles = ttotalmiles + (itemtotal['info']['aircraftsummary']['totalmiles'])
            tavgpax = tavgpax + (itemtotal['info']['aircraftsummary']['avgpax'])
            ttotalpax = ttotalpax + (itemtotal['info']['aircraftsummary']['totalpax'])
            tnight = tnight + (itemtotal['info']['aircraftsummary']['night'])
            tlandings = tlandings + (itemtotal['info']['aircraftsummary']['landings'])
            count += 1

        record = {'aircraftype':'Total','info':{'aircraftsummary':{'airtotal':tairtotal,'avgflight':round((tavgflight/count),2),'numflight':tnumflight,'avgmiles':int((tavgmiles/count)),'totalmiles':ttotalmiles,'avgpax':int(tavgpax/count),'totalpax':ttotalpax,'night':tnight,'landings':tlandings}}}
        qs.append(record)
        return self.render_to_response({"manufacture":manufacture,"qs":qs, "title":pagetitle})

class TailBase(TemplateView):
    template_name = 'aircraft/tailbase.html'
    def get(self, request, *args, **kwargs):
        pagetitle = "Tail Number Lookup"
        return self.render_to_response({"title":pagetitle})
class TailLookupDisplay(TemplateView):
    template_name = 'aircraft/taillookup.html'

    def get(self, request, *args, **kwargs):
        userid=getuserid()
        pagetitle = "Tail Number Lookup"
        secondrequest = request.GET.get('lookup[id]')

        info = FlightTime.objects.filter(userid=userid,aircraftId=secondrequest).aggregate(
                    airtotal = Sum('total'),
                    miletotal = Sum('distance'),
                    paxtotal = Sum('passengercount'),
                    flighttotal = Count('total'),
                    avgflight = Avg('total'),
                    avgdistance = Avg('distance'),
                    avgpax = Avg('passengercount')
            )
        flights = FlightTime.objects.filter(userid=userid,aircraftId=secondrequest).exclude(scheduledflight=True).order_by('-flightdate')

        return self.render_to_response({"title":pagetitle,"years":info,"depart":flights})
class TailNumberLookup(autocomplete.Select2QuerySetView):
    def get_queryset(self):
        qs = []
        userid = getuserid()
        disticttails = FlightTime.objects.filter(userid=userid).values('aircraftId').distinct().order_by('aircraftId')

        for aircraft in disticttails:
            data = {'id':aircraft['aircraftId'],'text':aircraft['aircraftId']}
            qs.append(data)

        # return  JsonResponse(correctdata, safe=False)
        # print(correctdata)
        # return self.render_to_response({"result":correctdata})
        qs = NewPlaneMaster.objects.all().order_by('nnumber')
        if self.q:
            qs = qs.filter(nnumber__istartswith=self.q)
        return qs

class NewIDLookup(autocomplete.Select2QuerySetView):
    def get_queryset(self):
        qs = AircraftModel.objects.all().order_by('type')
        if self.q:
            qs = qs.filter(type__istartswith=self.q)
        return qs
class AircraftEntry(FormView):

    template_name = 'aircraft/searchtype.html'
    form_class = AirplaneEntry
    success_url = 'entry'

    def form_valid(self,form):
        form.save()
        return super().form_valid(form)


"""empty message

Revision ID: ed7f15060080
Revises: b938f049e226
Create Date: 2020-10-03 21:48:14.073875

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'ed7f15060080'
down_revision = 'b938f049e226'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('Show',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('start_time', sa.DateTime(), nullable=False),
    sa.Column('artist_id', sa.Integer(), nullable=False),
    sa.Column('venue_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['artist_id'], ['Artist.id'], ),
    sa.ForeignKeyConstraint(['venue_id'], ['Venue.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.drop_column('Venue', 'past_shows')
    op.drop_column('Venue', 'upcoming_shows')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('Venue', sa.Column('upcoming_shows', sa.VARCHAR(length=120), autoincrement=False, nullable=True))
    op.add_column('Venue', sa.Column('past_shows', sa.VARCHAR(length=120), autoincrement=False, nullable=True))
    op.drop_table('Show')
    # ### end Alembic commands ###


# import collections
#
# def solution(numbers, target):
#     answer = 0
#     # [현재까지 합, 합한 성분 개수]
#     q = collections.deque([(0, 0)])
#     # 큐가 다 비워질때까지
#     while q:
#         current_sum, num_idx = q.popleft()
#
#         if num_idx == len(numbers):
#             if current_sum == target:
#                 answer += 1
#         else:
#             # 하나 빼고 두개 생성
#             number = numbers[num_idx]
#             q.append((current_sum+number, num_idx + 1))
#             q.append((current_sum-number, num_idx + 1))
#
#     return answer

# def solution(numbers, target):
#     def dfs(idx, result):
#         if idx == len(numbers):
#             if result == target:
#                 nonlocal answer
#                 answer += 1
#             return
#         dfs(idx + 1, result + numbers[idx])
#         dfs(idx + 1, result - numbers[idx])
#         return
#
#     answer = 0
#     dfs(0, 0)
#     return answer

from collections import deque

def solution(numbers, target):
    answer = 0
    q = deque([(0, 0)])
    while q:
        result, idx = q.popleft()
        if idx == len(numbers):
            if result == target:
                answer += 1
            continue
        q.append((result + numbers[idx], idx + 1))
        q.append((result - numbers[idx], idx + 1))
    return answer

print(solution([1, 1, 1, 1, 1], 3))


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
import random, math, time

def get_power_flow(src_cord, point_cord, n, power, cell_size,
                   lda, abs_coeff):
    '''Считает удельную мощность в точке с координатой point_cord[] от источника с координатами src_cord для длины волны lda'''
    distance = ((src_cord[0] - point_cord[0])**2 +
                (src_cord[1] - point_cord[1])**2 +
                (src_cord[2] - point_cord[2])**2)**0.5 # Находим расстояние от источника до заданной точки
    distance = (distance/n) * cell_size
    try:
        power_flow = power/(4*3.14159*distance**2) * 10**(-abs_coeff * distance) # Вводим коэффициент поглощения среды в расчёт

    except ZeroDivisionError: # На случай вокселя-источника, чтобы избежать деления на нулевое расстояние
        power_flow = 0
        pass
    return power_flow

# Функция для цветного вывода заданной грани
def show_edge(edge):
    fig, dots = plt.subplots()  # Задаём подразделы графика для вывода
    fig.set_figwidth(6.8)
    fig.set_figheight(10)
    dots.pcolormesh(edge, cmap='rainbow', vmin=0, vmax=10)  # Задаём имя объекта для вывода (первое в скобках) и цветовую гамму
    plt.show()

# Функция для цветного вывода всего кубика
def show_cube_3D(array):
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    z, y, x = array.nonzero()
    cube = ax.scatter(x, y, z, zdir='z', c=array[z, y, x], cmap=plt.cm.rainbow)  # Plot the cube
    cbar = fig.colorbar(cube, shrink=0.06, aspect=5)  # Add a color bar which maps values to colors.
    plt.show()

def MonteCarlo(n = 51, n_iter = 10000,
         x_source = 3, y_source = 3, z_source = 3): # Возвращает 3D массив cell с распределением излучения на наружных слоях
    start_time = time.time()
    cell = np.zeros((n, n, n)) # Задаём ячейку (3-мерный массив из нулей)
    h = 0.45
    for _ in range(n_iter):
        x = x_source
        y = y_source
        z = z_source
        theta = math.acos(random.uniform(-1,1))
        phi = random.uniform(0,2*math.pi)
        sin_theta = math.sin(theta)
        cos_theta = math.cos(theta)
        sin_phi = math.sin(phi)
        cos_phi = math.cos(phi)
        while (x < n-1) and (x > 0) and (y < n-1) and (y > 0) and (z < n-1) and (z > 0):
            x += (sin_theta * cos_phi) * h
            y += (sin_theta * sin_phi) * h
            z += (cos_theta) * h
        cell[round(x),round(y),round(z)] += 1

    '''Часть Монте-Карло и углов'''
    r = round(n/2)  # радиус кругового клеточного слоя
    innerdot = 0
    angles = np.zeros((n,n))
    angle_weight = np.zeros((n,n))
    x0 = round(n/2)-1 # координаты центра круглого клеточного слоя
    y0 = round(n/2)-1

    for x in range(0, n): # расчет отношения числа точек в нужной области к их общему числу
        for y in range(0, n):
            if (x - x0) ** 2 + (y - y0) ** 2 <= r ** 2:  # условие попадения в круг на дне ячейки
                innerdot += cell[x, y, 0]
                '''Часть расчета углов падения'''
                alpha = math.atan(math.sqrt((x-x_source)**2+(y-y_source)**2)/z_source)
                angles[x,y] = alpha
                angle_weight[x, y] = alpha * cell[x, y, 0]

    dots_in_area = innerdot/np.sum(cell)
    #print('Часть точек, попавших в нужную область:', dots_in_area)
    np.savetxt('angle_weight.txt', angle_weight)
    np.savetxt('angles.txt', angles)
    np.savetxt('cell_down.txt', cell[:, :, 0])
    #print('Средний угол падения:', (np.sum(angle_weight) / innerdot) / math.pi * 180)
    runtime = time.time() - start_time
    #print("--- %s seconds ---" % (runtime))  # Вывод времени работы программы
    return dots_in_area


from fastapi import FastAPI
from pydantic import BaseModel
import random

# pip3 install uvicorn
# pip3 install fastapi

# uvicorn file_name:app_instance --reload
#

app = FastAPI()
database = []

class Product(BaseModel):
    title: str
    quantity: int
    price: float
    inStock: bool

@app.post("/createItem/")
def create_item_endpoint(product: Product): # if schema/submitted data is validated then it goes through to our logic in the function
    database.append({
        "id": random.randint(0, 999),
        "title": product.title,
        "quantity": product.quantity,
        "price": product.price,
        "inStock": product.inStock
    })

    return {
        "status": "product created", # a custom response field we added for style
        "dataAccepted": product # the data the user submitted to the endpoint
    }

@app.get("/products")
def get_products():
    return database

@app.delete("/delete/product/{product_id}")
def delete_product(product_id):
    for index, obj in enumerate(database):
        if obj['id'] == int(product_id):
            database.pop(index)
            break

    return {
        "status": f"Product with ID {product_id} has been deleted"
    }


#!/usr/local/env python
import math
from pathlib import Path
import operator

each_line_is_bool = False
with (Path(__file__).parent / "input.txt").open() as text:
    lines = [ line.strip() for line in text.readlines() if line]

def solve(init_list: list) -> int:
    length = len(init_list[0])
    gamma_delta = []
    for i in range(length):
        zeros = 0
        ones = 0
        for bin in init_list:
            if bin[i] == "1":
                ones += 1
            if bin[i] == "0":
                zeros += 1

        if zeros >= ones:
            gamma_delta.append(1)
        else:
            gamma_delta.append(0)
        bit = "".join([str(x) for x in gamma_delta])
    inverse_bit = ''.join(['1' if i == '0' else '0'
                     for i in bit])
    gamma = int(bit,2)
    epsilon = int(inverse_bit,2)
    return gamma * epsilon

def inner_solve2(init_list: list, operator, index: int = 0,) -> int:
    length = len(init_list[0])
    for i in range(index, length):
        one_list = []
        zero_list = []
        for bin in init_list:
            zeros = 0
            ones = 0
            for bin in init_list:
                if bin[i] == "1":
                    ones += 1
                    one_list.append(bin)
                if bin[i] == "0":
                    zeros += 1
                    zero_list.append(bin)

            if operator(len(one_list), len(zero_list)):
                if len(one_list) > 1:
                    return inner_solve2(one_list, operator, index + 1)
                else:
                    return one_list[0]
            else:
                if len(zero_list) > 1:
                    return inner_solve2(zero_list, operator, index + 1)
                else:
                    return zero_list[0]

def solve2(init_list: list) -> int:
    o2 = int(inner_solve2(init_list, operator.ge ), 2)
    co2 = int(inner_solve2(init_list, operator.lt ), 2)
    return o2 * co2

example = [e for e in """00100
11110
10110
10111
10101
01111
00111
11100
10000
11001
00010
01010
""".split("\n") if e]
if each_line_is_bool:
    example = [ int(e) for e in example if e ]

part1 = solve(example)
print("Unit A", part1)
assert part1 == 198
print("Part 1:", solve(lines))

part2 = solve2(example)
print("Unit B", part2)
assert part2 == 230
print('Part 2:', solve2(lines))


#!/usr/bin/env python

import argparse
import ParseSlice
import numpy as np

parser = argparse.ArgumentParser()

# Explicit approach
parser.add_argument('--default-to-everything', nargs='*', action=ParseSlice.ParseSlice, default=ParseSlice.Everything)
parser.add_argument('--default-to-nothing',    nargs='*', action=ParseSlice.ParseSlice, default=ParseSlice.Nothing   )
# With sugar
parser.add_argument('--star-star-everything', **ParseSlice.SliceEverything)
parser.add_argument('--star-star-nothing',    **ParseSlice.SliceNothing)
parser.add_argument('--star-star-default',    **ParseSlice.SliceWithDefault([2, 3, 5, 7]))
parser.add_argument('--star-star-evens',      **ParseSlice.SliceWithDefault(slice(0,None,2)))

args = parser.parse_args()

x = np.arange(10)

print(f"       numpy array is: {x}")
print(f"Default to everything: {x[args.default_to_everything]}")
print(f"Default to    nothing: {x[args.default_to_nothing]}")
print(f"Star star  everything: {x[args.star_star_everything]}")
print(f"Star star     nothing: {x[args.star_star_nothing]}")
print(f"Star star     default: {x[args.star_star_default]}")
print(f"Star star       evens: {x[args.star_star_evens]}")


"""Google Sheets Authentication."""

import json
from datetime import datetime
from typing import Optional

import requests
from singer_sdk.authenticators import OAuthAuthenticator, SingletonMeta
from singer_sdk.helpers._util import utc_now
from singer_sdk.streams import Stream as RESTStreamBase

class GoogleSheetsAuthenticator(OAuthAuthenticator, metaclass=SingletonMeta):
    """Authenticator class for Google Sheets."""

    @property
    def oauth_request_body(self) -> dict:
        """Define the OAuth request body for the GoogleSheets API."""
        oauth_credentials = self.config.get("oauth_credentials", {})
        return {
            "grant_type": "refresh_token",
            "client_id": oauth_credentials.get("client_id"),
            "client_secret": oauth_credentials.get("client_secret"),
            "refresh_token": oauth_credentials.get("refresh_token"),
        }

class ProxyGoogleSheetsAuthenticator(OAuthAuthenticator, metaclass=SingletonMeta):
    """API Authenticator for Proxy OAuth 2.0 flows."""

    def __init__(
        self,
        stream: RESTStreamBase,
        auth_endpoint=None,
        oauth_scopes=None,
        auth_headers=None,
        auth_body=None,
    ) -> None:
        """Create a new authenticator."""
        super().__init__(stream=stream)
        self._auth_endpoint = auth_endpoint
        self._oauth_scopes = oauth_scopes
        self._auth_headers = auth_headers
        self._auth_body = auth_body

        # Initialize internal tracking attributes
        self.access_token: Optional[str] = None
        self.refresh_token: Optional[str] = None
        self.last_refreshed: Optional[datetime] = None
        self.expires_in: Optional[int] = None

    def is_token_valid(self) -> bool:
        """Check if token is valid."""
        if self.last_refreshed is None:
            return False
        if not self.expires_in:
            return True

        now = datetime.now(self.last_refreshed.tzinfo)

        if self.expires_in > (now - self.last_refreshed).total_seconds():
            return True
        return False

    # Authentication and refresh
    def update_access_token(self) -> None:
        """Update `access_token` along with: `last_refreshed` and `expires_in`."""
        request_time = utc_now()

        token_response = requests.post(
            self.auth_endpoint,
            headers=self._auth_headers,
            data=json.dumps(self._auth_body),
        )
        try:
            token_response.raise_for_status()
            self.logger.info("OAuth authorization attempt was successful.")
        except Exception as ex:
            raise RuntimeError(
                f"Failed OAuth login, response was '{token_response.json()}'. {ex}"
            )
        token_json = token_response.json()
        self.access_token = token_json["access_token"]
        self.expires_in = token_json["expires_in"]
        self.last_refreshed = request_time

    @property
    def oauth_request_body(self) -> dict:
        """Define the OAuth request body."""
        return {}


#!/usr/bin/env python

"""
Tomography image stack preprocessor service. Processes HDF5 files from tomography fly scans,
performs median filtering on repeated scans to remove zingers, and averages the frames to
improve SNR. If there is only one repetition of data, zinger removal using TomoPy will be performed.
If there are two repetitions, zingers will be replaced with the minimum value at that pixel location.
The code also calculates required chunksize assuming that the user wants to keep 1/3 of the
system's available RAM free. This is set by the variable "threshold", and can be changed

This file belongs in the top-level directory as it takes its location as the current working directory.
The script walks through all of the folders inside this top-level directory, searches for HDF5 files, and checks
if a prefiltered file already exists. If it doesn't, then it will create one.

Alan Kastengren, XSD, APS
Katie Matusik, XSD, APS
"""

import time
import os
from multiprocessing import Process, Queue, JoinableQueue, cpu_count
import glob
import h5py
import numpy as np
import tomopy
import psutil

'''Function definitions'''

# The consumer function takes data off of the Queue
def consumer(inqueue,output):
    # Run indefinitely
    while True:
        # If the queue is empty, queue.get() will block until the queue has data
        all_data = inqueue.get()
        if all_data:
            #n is the index corresponding to the projection location
            n, image_data = all_data
            #if there is only one image set, do zinger removal with TomoPy
            if image_data.shape[0]==1:
                out = tomopy.remove_outlier(image_data,zinger_level,ncore = cpu_count()-1)
            #if there are two datasets, replace zingers by min value at each pixel location
            elif image_data.shape[0]==2:
                # If the difference in pixel value between two frames is too high,
                #  assume the higher value is a zinger, and hence invalid
                minval = np.fmin(image_data[0,...],image_data[1,...])
                data_range = np.abs(image_data[0,...],image_data[1,...])
                mask = data_range > zinger_level
                image_data[:,mask] = minval[mask]
                out = np.mean(image_data,axis=0,dtype=np.float32).astype(np.uint16)
            #otherwise, replace zingers with median and average stack
            elif image_data.shape[0]>2:
                #Find the median for each pixel of the prefiltered image
                med = np.median(image_data,axis=0)
                #Loop through the image set
                for j in range(image_data.shape[0]):
                    replicate = image_data[j,...]
                    mask = replicate - med > zinger_level
                    replicate[mask] = med[mask] # Substitute with median
                    image_data[j,...] = replicate # Put data back in place
                out = np.mean(image_data,axis=0,dtype=np.float32).astype(np.uint16)
            output.put((n,out))
        else:
            break

#function for writing out HDF5 file
def write_hdf(output,output_filename):
    #create output HDF5 file
    while True:
        args = output.get()
        if args:
            i,data = args
            with h5py.File(output_filename,'a') as fout:
                fout['Prefiltered_images'][i,...] = data
        else:
            break

def fprocess_hdf_stack(hdf_filenames,output_filename,chunksize):
    #take care of bright and dark fields first
    file_list = []
    bright_list = []
    dark_list = []
    print("Loading in bright and dark field data...")
    for fname in hdf_filenames:
        file_list.append(h5py.File(fname,'r'))
        #save all of the bright and dark fields as separate lists
        bright_list.extend(file_list[-1]['exchange']['data_white'][...])
        dark_list.extend(file_list[-1]['exchange']['data_dark'][...])
    bright_list = np.asarray(bright_list)
    dark_list = np.asarray(dark_list)
    #run the bright and dark images through median filtering to get a single image out
    bright_field = fprocess_hdf_bright_dark(bright_list)
    dark_field = fprocess_hdf_bright_dark(dark_list)

    #put everything in output HDF5 file
    fout = h5py.File(output_filename,'a')
    fout.create_dataset('Bright_field',data = bright_field,dtype=np.uint16)
    fout.create_dataset('Dark_field',data = dark_field,dtype=np.uint16)
    print("Done writing bright and dark fields to HDF5 file.")

    #copy things over from one of the data HDF5 files
    with h5py.File(fname,'r') as fin:
        fin.copy('measurement',fout)
        #fix the filter names and other attributes that should be strings
        filt1 = fout['measurement']['instrument']['filters']['Filter_1_Material'][...].ravel()
        filt2 = fout['measurement']['instrument']['filters']['a givenFilter_2_Material'][...].ravel()
        del fout['measurement']['instrument/filters/Filter_1_Material']
        del fout['measurement']['instrument/filters/Filter_2_Material']
        fout['measurement']['instrument']['filters']['Filter_1_Material'] = ''.join(chr(i) for i in [d for d in filt1 if d])
        fout['measurement']['instrument']['filters']['Filter_2_Material'] = ''.join(chr(i) for i in [d for d in filt2 if d])
        strings = ['firmware_version','manufacturer','model','software_version']
        for keys in strings:
            temp = fout['measurement']['instrument']['detector'][keys][...].ravel()
            del fout['measurement']['instrument']['detector'][keys]
            fout['measurement']['instrument']['detector'][keys] = ''.join(chr(i) for i in [d for d in temp if d])
        if 'table' in fout['measurement'].keys():
            del fout['measurement']['table']
    print("Done copying attributes over to HDF5 file.")

    #process chunks of data so that we don't run out of memory
    totsize = h5py.File(hdf_filenames[0],'r')['exchange']['data'].shape[0]
    data_shape =  h5py.File(hdf_filenames[0],'r')['exchange']['data'].shape
    fout.create_dataset('Prefiltered_images',data_shape,dtype=np.uint16)
    fout.close()
    ints = range(totsize)
    chunkSize= chunksize

    #initialize how many consumers we would like working
    num_consumers = cpu_count()*2

    #Create the Queue objects
    inqueue = JoinableQueue()
    output = Queue()

    #start process for writing HDF5 file
    proc = Process(target=write_hdf, args=(output,output_filename))
    proc.start()

    print("Loading %i images into memory..."%chunkSize)
    for i in range(0,totsize,chunkSize):
        time0 = time.time()
        chunk = ints[i:i+chunkSize]
        data_list = []
        #Make a list of the HDF5 datasets we are reading in
        for files in file_list:
            #shape is (angles, rows, columns)
            data_list.append(files['exchange']['data'][chunk,...])
        data_list = np.asarray(data_list)
        print("Elapsed time to load images %i-%i is %0.2f minutes." %(chunk[0],chunk[-1],(time.time() - time0)/60))

        consumers = []

        #Create consumer processes
        for i in range(num_consumers):
            p = Process(target=consumer, args=(inqueue,output))
            consumers.append(p)
            p.start()

        for n in range(data_list.shape[1]):
            #Feed data into the queue
            inqueue.put((chunk[n],data_list[:,n,...]))

        #Kill all of the processes when everything is finished
        for i in range(num_consumers):
            inqueue.put(None)

        for c in consumers:
            c.join()
        print("Elapsed time to process images %i-%i is %0.2f minutes." %(chunk[0],chunk[-1],(time.time() - time0)/60))

    time.sleep(1)
    output.put(None)
    proc.join()

    #Close the input HDF5 files.
    for hdf_file in file_list:
        hdf_file.close()
    print("Input HDF5 files closed.")
    return

def fprocess_hdf_bright_dark(data):
    '''Processes an HDF5 file that contains a stack of images to be prefiltered.
    Useful for bright/dark files.
    '''
    med = np.median(data,axis=0)
    for j in range(data.shape[0]):
            replicate = data[j,...]
            mask = replicate - med > zinger_level
            replicate[mask] = med[mask] # Substitute with median
            data[j,...] = replicate # Put data back in place
    # Mean filter the stack now that the zingers are gone.
    # Keep as int16 to save space
    img = np.mean(data,axis=0,dtype=np.float32).astype(np.uint16)
    return img

if __name__ == '__main__':
    start_time = time.time()
    main_folder = os.getcwd()
    zinger_level=3000
    all_folders = next(os.walk('.'))[1]
    #all_folders = glob.glob(main_folder+'/22_5E_6_BC_CC*')
    for item in all_folders:
        raw_images_filenames = glob.glob(main_folder+'/'+item + '/*.hdf5')
        #raw_images_filenames = glob.glob(item+'/*.hdf5')
        #check for prefiltered hdf5 file in folder
        if raw_images_filenames:
            here = [j for j in raw_images_filenames if 'Prefiltered.hdf5' in j]
            if here:
                print("Prefiltering has already been completed. Prefiltered images are saved in %s."%os.path.basename(here[0]))
                pass
            else:
                output_dir = item+'/'
                if output_dir[-1] != '/':
                    output_dir=output_dir + '/'
                if raw_images_filenames:
                    print('Raw HDF5 files are in directory %s' %item)
                    if len(raw_images_filenames)>1:
                        print('There are %i raw image files. These will be averaged together.' %(len(raw_images_filenames)))
                    elif len(raw_images_filenames)==1:
                        print('There is %i raw image file.' %(len(raw_images_filenames)))
                    else:
                        print('There are %i raw image files.' %(len(raw_images_filenames)))
                    tempname = os.path.basename(raw_images_filenames[0]).split('.')[0]
                    tempname_split = tempname.split('_')[:-1]
                    output_filename = output_dir+'_'.join(tempname_split) + '_Prefiltered.hdf5'
                    print("Data will be written to %s"%output_filename)
                    #get size of images assuming 16 bit to figure out chunksize
                    with h5py.File(raw_images_filenames[0],'r') as f:
                        #m = projections, n = rows, o = columns
                        (m,n,o) = f['exchange']['data'].shape
                        size = (2*m*n*o)/(1e9)
                        totsize = size*(len(raw_images_filenames)+1) # +1 accounts for one prefiltered set
                        mem = psutil.virtual_memory()
                        avmem = mem.available/(1e9) #system memory that is available and not swap
                        #assume that we want 1/3 of the system memory to be free for other things and can therefore use 2/3 of it
                        threshold = avmem*(2/3.)
                        if totsize>threshold:
                            #back out how many images can be processed with threshold memory
                            chunksize = int((threshold*1e9)/(len(raw_images_filenames)+1)/(2*n*o))
                        else:
                            chunksize = int(m)
                    fprocess_hdf_stack(raw_images_filenames,output_filename,chunksize)
                else:
                    print('Raw image files could not be found. Please check file location and names.')
                print("Elapsed time is %0.2f minutes" %((time.time() - start_time)/60))
        else:
            print("No 'hdf5' files found in %s. Skipping directory." %item)
            pass


""" dgbpy contains generic deep learning python tools for seismic interpretation. """

from setuptools import setup, find_packages
import re

with open('dgbpy/__init__.py', 'r') as f:
    version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', f.read(), re.MULTILINE).group(1)

setup(
    name='dgbpy',
    packages=find_packages(exclude=[]),
    version=version,
    url='https://github.com/OpendTect/dgbpy',
    license='Apache 2.0',
    author='dGB Earth Sciences',
    author_email='info@dgbes.com',
    description='Deep Learning tools for seismic interpretation using OpendTect',
    long_description='',
    zip_safe=False,
    platforms='any',
    install_requires=[
        'bokeh>=2.1.1',
        'h5py>=2.10.0',
        'joblib>=1.0.1',
        'keras>=2.3.1',
        'numpy>=1.19.2',
        'odpy>=1.0.0',
        'scikit-learn>=0.24.2',
        'psutil>=5.7.0',
        'tensorflow>=2.1.4',
        'torch>=1.9.0',
        'fastprogress>=1.0.0',
        'onnxruntime-gpu>=1.0.0',
    ],
    extras_require={
        'onnxmltools': ['onnxmltools>=1.9.0'],
        'pydot': ['pydot>=1.4.1'],
        'scikit-learn-intelex': ['scikit-learn-intelex>=1.0.0'],
        'skl2onnx': ['skl2onnx>=1.0.0'],
        'xgboost': ['xgboost>=1.1.1'],
    },
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Science/Research',
        'Intended Audience :: Information Technology',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Topic :: Scientific/Engineering',
    ],
)


from pyrogram import Client, filters

from plugins import r, admin

import asyncio

@Client.on_message(filters.private &~ filters.user([admin, "SpamBot"]), group=3)
async def PrivateUserChat(c, m):

    chat_id = m.chat.id

    if r.exists("Messages"):
        data = sorted(r.smembers("Messages"), key=lambda x: x.split(":")[0])

        for message in data:
            await c.send_chat_action(chat_id=chat_id, action='typing')
            await asyncio.sleep(0.17)

            await c.send_message(chat_id=m.chat.id, text=message.split(":", 1)[1])

            await asyncio.sleep(3)

    else:
        await c.send_chat_action(chat_id=chat_id, action='typing')
        await asyncio.sleep(0.15)

        message=f"سلام {m.from_user.first_name} ✋"

        await c.send_message(chat_id=m.chat.id, text=message)

    current_user = await c.get_me()
    r.sadd(f"users:{current_user.id}", m.from_user.id)


from metrics import StreamSegMetrics
from .trainer import Trainer

class SourceOnlyTrainer(Trainer):

    def __init__(self, args, writer, device, rank, world_size):
        super().__init__(args, writer, device, rank, world_size)

    @staticmethod
    def set_metrics(writer, num_classes):
        writer.write("Setting up metrics...")
        metrics = {
            'train': StreamSegMetrics(num_classes, 'train'),
            'test_source': StreamSegMetrics(num_classes, 'test_source'),
            'test_target': StreamSegMetrics(num_classes, 'test_target'),
        }
        writer.write("Done.")
        return metrics

    def handle_ckpt_step(self):
        return None, self.checkpoint_step, None, None

    def max_iter(self):
        return self.args.num_source_epochs * self.source_train_clients[0].len_loader

    def perform_centr_sourceonly_training(self, e_type, e, e_ref, train_client, train_metric):
        self.writer.write(f"{e_type}: {e + 1}/{e_ref}")
        self.model.train()
        _ = train_client.run_epoch(e, self.optimizer, train_metric, self.scheduler, e_name=e_type)
        # train_metric.synch(self.device)
        self.writer.plot_metric(e, train_metric, str(train_client), self.ret_score)
        train_metric.reset()
        self.save_model(e + 1, self.optimizer, self.scheduler)

    def train(self, train_metric=None, max_scores=None):

        if train_metric is None:
            train_metric = self.metrics['train']

        max_scores = [0] * len(self.target_test_clients)

        for e in range(self.ckpt_source_epoch, self.args.num_source_epochs):
            self.perform_centr_sourceonly_training('SOURCE EPOCH', e, self.args.num_source_epochs,
                                               self.source_train_clients[0], train_metric)
            if (e + 1) % self.args.test_interval == 0 or (e + 1) == self.args.num_source_epochs:
                max_scores, improvement = self.test(self.target_test_clients, self.metrics['test_target'], e,
                                                    'SOURCE EPOCH', max_scores, cl_type='target')
                self.test(self.source_test_clients, self.metrics['test_source'], e, 'SOURCE EPOCH',
                          self.get_fake_max_scores(improvement, len(self.source_test_clients)), cl_type='source')

        return max_scores


import pytest

from ops.model import ActiveStatus, BlockedStatus, WaitingStatus
from ops.testing import Harness
import yaml

from charm import ArgoControllerCharm

@pytest.fixture
def harness():
    return Harness(ArgoControllerCharm)

def test_not_leader(harness):
    harness.begin_with_initial_hooks()
    assert isinstance(harness.charm.model.unit.status, WaitingStatus)

def test_missing_image(harness):
    harness.set_leader(True)
    harness.begin_with_initial_hooks()
    assert isinstance(harness.charm.model.unit.status, BlockedStatus)

def test_main_no_relation(harness):
    harness.set_leader(True)
    harness.add_oci_resource(
        "oci-image",
        {
            "registrypath": "argoproj/workflow-controller:v3.0.1",
            "username": "",
            "password": "",
        },
    )
    harness.begin_with_initial_hooks()
    pod_spec = harness.get_pod_spec()

    # confirm that we can serialize the pod spec
    yaml.safe_dump(pod_spec)

    assert isinstance(harness.charm.model.unit.status, BlockedStatus)

def test_main_with_relation(harness):
    harness.set_leader(True)
    harness.add_oci_resource(
        "oci-image",
        {
            "registrypath": "argoproj/workflow-controller:v3.0.1",
            "username": "",
            "password": "",
        },
    )
    rel_id = harness.add_relation("object-storage", "minio")
    harness.add_relation_unit(rel_id, "minio/0")
    data = {
        "service": "my-service",
        "port": 4242,
        "access-key": "my-access-key",
        "secret-key": "my-secret-key",
        "secure": True,
        "namespace": "my-namespace",
    }
    harness.update_relation_data(
        rel_id,
        "minio",
        {
            "data": yaml.dump(data),
            "_supported_versions": yaml.dump(["v1"]),
        },
    )
    harness.begin_with_initial_hooks()
    assert isinstance(harness.charm.model.unit.status, ActiveStatus)


from loader import dp, bot
from handlers.utils.db_api import quick_commands as commands

import asyncio
import aioschedule

async def notice():
    users = await commands.select_all_users()
    for user in users:
        text = f'{user.first_name}, а ты давно проходил тесты?'
        await bot.send_message(chat_id=user.user_id, text=text)

async def scheduler():
    aioschedule.every().day.at("13:00").do(notice)
    while True:
        await aioschedule.run_pending()
        await asyncio.sleep(1)


import bs4 as bs
from urllib.request import Request, urlopen
import json
import csv

#Converting INPUT JSON to DICT
with open('scraper-config.json', 'rb') as f:
  input_config = json.load(f)

#Function to read the input config file and scrap data as per the css selectors mentioned
def scraped(input_config2, soup):
    if(input_config2["entity_type"]!="entity"):
        if(input_config2["config"]=="img.img_delete_bg_noshow::attr(src)"): ##Handling one exception case of attr::(src)
            input_config2["entity"] = soup.select("img.img_delete_bg_noshow")[0]["src"]
        else:
            if not (soup.select(input_config2["config"][:-6])):
                input_config2["entity"] = "Not Found"
            else:
                input_config2["entity"] = soup.select(input_config2["config"][:-6])[0].text

    else:
        for nested_entity in input_config2["entity_scrapers"]:
            scraped(nested_entity, soup) ## To handle nested entities

#Empty list of dict which will be exported as JSON
final_output = []

##Readind the INPUT CSV File
content = []
with open('urls-100.csv', 'r') as file:
    reader = csv.reader(file)
    for url in reader:
        content.append(url)
content.pop(0) ##Removing the first row which is not a url

for url in content:
    req = Request(url[0]+'/', headers={'User-Agent': 'Mozilla/5.0'})
    sauce = urlopen(req).read()
    soup = bs.BeautifulSoup(sauce,'lxml')
    print(soup.title.text)
    output = input_config.copy()
    scraped(output,soup) ## Passing the url to custom function
    print(output) ## The output consists of scraped data for every different url
    final_output.append(output.copy()) ## Appending the dict to final list
    ## --------PROBLEM(pending) - The respective output with iteration over urls in console is correct, but when we look at final_output, it contains 100 dicts of last output only

## Exporting to JSON Object
final_output_json = json.dumps(final_output)
with open("output.json", "w") as outfile:
    outfile.write(final_output_json)