from django.conf.urls import url
from video.view.votes import *
from django.contrib.auth.decorators import login_required

urlpatterns = [
	url(r'^user_like/(?P<uuid>[0-9a-f-]+)/(?P<pk>\d+)/$',login_required(VideoUserLikeCreate.as_view())),
    url(r'^user_dislike/(?P<uuid>[0-9a-f-]+)/(?P<pk>\d+)/$',login_required(VideoUserDislikeCreate.as_view())),
    url(r'^user_comment/(?P<comment_pk>\d+)/(?P<pk>\d+)/like/$',login_required(VideoCommentUserLikeCreate.as_view())),
    url(r'^user_comment/(?P<comment_pk>\d+)/(?P<pk>\d+)/dislike/$',login_required(VideoCommentUserDislikeCreate.as_view())),

	url(r'^community_like/(?P<uuid>[0-9a-f-]+)/(?P<pk>\d+)/$',login_required(VideoCommunityLikeCreate.as_view())),
    url(r'^community_dislike/(?P<uuid>[0-9a-f-]+)/(?P<pk>\d+)/$',login_required(VideoCommunityDislikeCreate.as_view())),
    url(r'^community_comment/(?P<comment_pk>\d+)/(?P<pk>\d+)/like/$',login_required(VideoCommentCommunityLikeCreate.as_view())),
    url(r'^community_comment/(?P<comment_pk>\d+)/(?P<pk>\d+)/dislike/$',login_required(VideoCommentCommunityDislikeCreate.as_view())),
]


#!/usr/bin/env python

import os
import argparse
import re
import json
import logging

ORG_SOURCE_DIR = os.path.expanduser("~/org/database")
SOURCE_CODE_BLOCK_REGEX = (
    r"#\+NAME: *(?P<name>.*?) *\n"
    r"#\+begin_src *(?P<source_type>\w*) *.*?\n"
    r"(?P<source_code>.*?)\n"
    r"#\+end_src"
)
TARGET_ORG_FILE_REGEX = r"[a-zA-z_-]*\.org"

SOURCE_TYPES = ["source-code", "links"]

logging.basicConfig(filename="/tmp/org-source-feed.log", level=logging.DEBUG)
class EntryOfLinks(object):
    def __init__(self, source_type, remain_args):
        self.source_type = source_type
        self.remain_args = remain_args
        self.args = None

    def get_sources(self):
        useful_link_file = os.path.join(ORG_SOURCE_DIR, "useful-links.md")
        if not os.path.exists(useful_link_file):
            return []

        result = []
        with open(useful_link_file, "r") as f:
            for line in f.readlines():
                for matched_result in re.finditer(r"\[(?P<link_desc>.*?)\]\((?P<link>.*?)\)", line, re.M | re.I):
                    logging.debug("Find a link link_desc=[{link_desc}], link=[{link}]".format(
                        link_desc=matched_result["link_desc"],
                        link=matched_result["link"],
                    ))
                    result.append({
                        "source_file": "useful-links",
                        "name": matched_result["link_desc"],
                        "type": "link",
                        "code": matched_result["link"],
                    })
            return result

    def run(self):
        parser = argparse.ArgumentParser()
        parser.add_argument(
            "--operation",
            "-o",
            required=True,
            help="operation type"
        )
        self.args = parser.parse_args(self.remain_args)
        logging.debug("EntryOfLinks get args: {}".format(self.args))
        return EntryOfLinks.__dict__[self.args.operation.replace('-', '_')](self)

class EntryOfSourceCode(object):
    def __init__(self, source_type, remain_args):
        self.source_type = source_type
        self.remain_args = remain_args
        self.args = None

    def get_sources(self):
        result = []
        for file_name in filter(lambda s: re.match(TARGET_ORG_FILE_REGEX, s), os.listdir(ORG_SOURCE_DIR)):
            file_full_path = os.path.join(ORG_SOURCE_DIR, file_name)
            with open(file_full_path, "r") as f:
                content = f.read()
                for matched_result in re.finditer(SOURCE_CODE_BLOCK_REGEX, content, re.M | re.I | re.S | re.MULTILINE):
                    logging.debug("Find a match {}".format(matched_result["name"]))
                    result.append({
                        "source_file": file_name,
                        "name": matched_result["name"],
                        "type": matched_result["source_type"],
                        "code": EntryOfSourceCode.remove_prefix_ws(matched_result["source_code"]),
                    })
        return result

    @staticmethod
    def remove_prefix_ws(content):
        return "\n".join(map(lambda s: s[2:] if len(s) >= 2 else s, content.split("\n")))

    def run(self):
        parser = argparse.ArgumentParser()
        parser.add_argument(
            "--operation",
            "-o",
            required=True,
            help="operation type"
        )
        self.args = parser.parse_args(self.remain_args)
        logging.debug("EntryOfSourceCode get args: {}".format(self.args))
        return EntryOfSourceCode.__dict__[self.args.operation.replace('-', '_')](self)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--sourceType",
        "-t",
        required=True,
        help="source type",
        choices=SOURCE_TYPES,
    )
    args, remain = parser.parse_known_args()
    logging.debug("Main parse arguments result: args={} remain={}".format(args, remain))
    entry_name = 'EntryOf' + ''.join(
        map(lambda s: s[:1].upper() + s[1:],
            args.sourceType.split('-'))
    )
    print(json.dumps(globals()[entry_name](args, remain).run()))

if __name__ == "__main__":
    main()


import bme280
import smbus2
from time import sleep
import timestamp
port = 1
address = 0x77 # Adafruit BME280 address. Other BME280s may be different
bus = smbus2.SMBus(port)

bme280.load_calibration_params(bus,address)

while True:
    bme280_data = bme280.sample(bus,address)
    humidity  = bme280_data.humidity
    pressure  = bme280_data.pressure/33.8639
    temperature = (bme280_data.temperature * 9.0/5.0) +32
#    data = f"{timestamp.stamp()} Humidity {humidity:03.2f} % Pressure {pressure:03.2f} InHg Temperature {temperature:03.1f} F
"
    current = f"
{timestamp.stamp()}

 Humidity {humidity:03.2f} %

 Pressure {pressure:03.2f} InHg

 Temperature {temperature:03.1f} F

"
    print(current)
#    print(f"
 {timestamp.stamp()}
 Humidity {humidity:03.2f} % Pressure {pressure:03.2f} InHg Temperature {temperature:03.1f} F")

    try:

        with open("/var/www/html/current.html", "w") as outfile1:
            outfile1.write("<meta http-equiv='refresh' content='15'/>
")
            outfile1.write(str(current))

    except KeyboardInterrupt:
        print("Later Dude")

    finally:

        with open("/var/www/html/current.html", "a") as outfile1:
            outfile1.close()

    sleep(15)


from bs4 import BeautifulSoup
import audiojack
from threading import Thread

urllib = audiojack.urllib
youtube_dl = audiojack.youtube_dl

ydl_opts = {
    'format': 'bestaudio',
    'outtmpl': 'F:\\%(id)s.%(ext)s',
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'mp3',
        'preferredquality': '256'}]
    }

ydl = youtube_dl.YoutubeDL(params=ydl_opts)

class MusicDownloaderAPI:

    def __init__ (self, settings):
        self.settings = settings
        #PARSE SETTINGS

    def _get_thumbnail_url (self, video_id):
        url = f"http://i.ytimg.com/vi/{video_id}/mqdefault.jpg"
        return url

    def _get_image (self, url):
        img_req = urllib.request.urlopen(url)
        img = img_req.read()
        img_req.close()
        return img

    def _add_images (self, entries):
        res = []
        for video in entries:
            img_url = self._get_thumbnail_url(video['id'])
            img = self._get_image(img_url)
            video['image'] = img
            res.append(video)
        return res

    def set_param (self, name, value):
        ydl.params[name] = value

    def get_search_results (self, query, download=False, download_first=True, limit=3):
        query = query.replace(' ', '+')
        url = 'http://youtube.com/results?search_query=' + query
        info = ydl.extract_info(url, download=download, limit=limit)
        self.download(info['entries'][0]['webpage_url'])
        res = self._add_images(info['entries'])
        return res

    def get_lyrics (self, song_title, song_artist):
        artist = song_artist.lower()
        title = song_title.lower()
        artist = artist.replace(' ', '')
        title = title.replace(' ', '')

        url = "http://azlyrics.com/lyrics/"+artist+"/"+title+".html"
        print(url)

        content = urllib.request.urlopen(url).read()
        soup = BeautifulSoup(content, 'html.parser')
        lyrics = str(soup)
        # lyrics lies between up_partition and down_partition
        up_partition = ''
        down_partition = ''
        lyrics = lyrics.split(up_partition)[1]
        lyrics = lyrics.split(down_partition)[0]
        lyrics = lyrics.replace('
','').replace('
','').replace('
','').strip()
        return lyrics

    def download (self, video_urls):
        if not isinstance(video_urls, list):
            video_urls = [video_urls]
        thr = Thread(target=ydl.download, args=(video_urls,))
        thr.start()


import os
import re
import copy
import warnings
import flopy
import tempfile
import numpy as np
import prms_python as pp
from collections import OrderedDict
from datetime import datetime
from shutil import copyfile

class Control(OrderedDict):
    """
    Simple python data storage class that operates as an ordered dictionary.
    Class allows the user to create new, load existing, edit, and write
    GSFLOW control files.
    Simple consistancy checking is used to assure data is of proper type
    and number
    Record values can be called from the Control object using a dictionary key.

    Parameters:
    -----------
        controlfile: (str) name of the control file

    Methods:
    --------
        get_record: record handling class, retrieves complete metadata of a
                    record
        add_record: record handling class, adds a complete metadata record
        remove_record: record handling class, removes a record from the Control
                       object
        write: method to write data stored in the Control object to a control
               file
        load: static method to load existing an existing control file

    Attributes:
    -----------
        header: (str) Header string for the GSFLOW control file

    """
    def __init__(self, controlfile):
        # instantiate the OrderedDictionary override
        super(Control, self).__init__()

        self.header = 'GSFLOW control file built using GSFlowPy\n'
        self.name = controlfile

        if isinstance(controlfile, file):
            self.name = controlfile.name
            self.__controlfile = controlfile
            self.__get_params()

    def __getitem__(self, key):
        return super(Control, self).__getitem__(key)['val']

    def __setitem__(self, key, value):
        if type(value) is dict:
            rec = value
            rec = self.__check_record(rec)
        else:
            try:
                rec = self.get_record(key)
                rec['val'] = value
                rec = self.__check_record(rec)
            except KeyError:
                raise KeyError('record does not exist, create new record ' +
                               'using the <add_record> method')
        super(Control, self).__setitem__(key, rec)

    def __get_params(self):
        """
        Method to read parameters from an existing GSFLOW control file and
        create a record of data for each parameter within the Control object
        """
        position = 0
        name = None
        nval = None
        dtype = None
        val = []
        self.header = self.__controlfile.readline()
        # maybe use enumerate to track position?
        for line in self.__controlfile:
            if "#" in line:
                # we want to skip the fist instance of ###
                position = 0
                if name is not None:
                    # remove list dimension from data if it is not needed
                    if nval == 1:
                        val = val[0]

                    if name in ('start_time', 'end_time'):
                        # create a datetime object where appropriate
                        val = datetime(val[0], val[1], val[2],
                                       val[3], val[4], val[5])

                    rec = dict(name=name,
                               dtype=dtype,
                               nval=nval,
                               val=val)

                    super(Control, self).__setitem__(name, rec)

                    name = None
                    dtype = None
                    val = []

            elif position == 1:
                line = line.strip().split()
                name = str(line[0])

            elif position == 2:
                line = line.strip().split()
                nval = int(line[0])

            elif position == 3:
                line = line.strip().split()
                dtype = int(line[0])

            elif position > 3:
                line = line.strip().split()

                if dtype == 1:
                    val.append(int(line[0]))

                elif dtype in (2, 3):
                    val.append(float(line[0]))

                elif dtype == 4:
                    val.append(str(line[0]))

            position += 1

    def __check_record(self, rec):
        """
        Defensive method to check for consistancy of data type and length
        Parameters:
        -----------
            rec: (dict) Data record containing parameter metadata and data

        Returns:
        --------
            rec: (dict) Data record containing parameter metadata and data
        """
        if rec['name'] in ('start_time', 'end_time'):
            assert isinstance(rec['val'], datetime), 'time object:n {} must' + \
                                                     'be a datetime object'\
                                                     .format(rec['name'])
        else:

            rec = self.__check_nval(rec)
            if rec['nval'] == 1:
                self.__check_dtype(rec['nval'], rec['dtype'])

            elif rec['nval'] > 1:
                for val in rec['val']:
                    self.__check_dtype(val, rec['dtype'])

            else:
                raise StandardError('No data supplied for {}'
                                    .format(rec['name']))

        return rec

    def __check_dtype(self, val, dtype):
        """
        Convience method to check a value against its documented dtype
        Parameters:
        -----------
            val: value to be checked for consistancy with its listed dtype
            dtype: (int) GSFLOW control file dtype (1: int, 2: float, 3: float,
                         4: string
        """

        if dtype == 1:
            assert isinstance(val, int), 'listed dtype INT does not match' + \
                                         ' value supplied'

        elif dtype in (2, 3):
            try:
                float(val)
            except ValueError:
                assert isinstance(val, float), 'listed dtype FLOAT does not' + \
                                               ' match value supplied'

        elif dtype == 4:
            assert isinstance(val, str), 'listed dtype STR does not match' + \
                                         ' value supplied'

        else:
            raise AssertionError('dtype {} is not valid'.format(dtype))

    def __check_nval(self, rec):
        """
        Convience method to check adjust nval if necessary.

        Parameters
        ----------
            rec: (dict) GSFLOW control record

        Returns
        -------
            rec: (dict) GSFLOW control record
        """
        val = rec['val']

        if isinstance(val, int):
            rec['nval'] = 1

        elif isinstance(val, list):
            nval = len(val)
            rec['nval'] = nval

        return rec

    def get_record(self, key):
        """
        Method to get all data from a record of the control file dictionary
        Parameters:
        -----------
            key: (str) GSFLOW control record parameter name
        Returns:
        --------
            record: (dict) dictionary of data corresponding to the control
                           record
        """
        return super(Control, self).__getitem__(key)

    def add_record(self, name, nval, dtype, val):
        """
        Convience method to add records to Control object, or overwrite existing
        Control records.
        Parameters:
        -----------
            name: (str) gsflow record name
            nval: (int) number of values for the record (time objects = 6)
            dtype: (int) datatype specified in the gsflow documentation
            val: value(s) multiple values must be supplied in list format
        """
        rec = dict(name=name, nval=nval, dtype=dtype, val=val)
        self.__setitem__(name, rec)

    def remove_record(self, name):
        """
        Convience method to delete a record from the Control object
        Parameters:
        -----------
            name: (str) gsflow record name
        """
        self.pop(name, None)

    def write(self, controlfile=None):
        """
        Method to write records to a new gsflow control file
        Parameters:
        -----------
            controlfile: (str) file name for gsflow control file
        """
        if controlfile is None:
            controlfile = self.name
        else:
            self.name = controlfile

        # check if path exists
        controlfile = controlfile.replace('/', '\\')
        path = '\\'.join(controlfile.split('\\')[:-1])

        if os.path.exists(path):
            controlfile = open(controlfile, 'w')
        else:
            os.makedirs(path)
            controlfile = open(controlfile, 'w')

        # check header for newline
        if self.header[-1] != '\n':
            self.header += '\n'

        controlfile.write(self.header)

        for key in self:
            rec = self.get_record(key)

            if rec['name'] in ('start_time', 'end_time'):
                # re-format datetime object into string for gsflow control
                val = rec['val']
                dt = '{}\n{}\n{}\n{}\n{}\n{}'.format(val.year,
                                                     val.month,
                                                     val.day,
                                                     val.hour,
                                                     val.minute,
                                                     val.second)
                rec['val'] = dt

            elif rec['nval'] > 1:
                # re-format list object into string for gsflow control
                val = ''
                for i in rec['val']:
                    val += '{}\n'.format(i)
                rec['val'] = val[:-1]

            else:
                pass

            # write records to control file
            rec = '####\n{}\n{}\n{}\n{}\n'.format(rec['name'],
                                                  rec['nval'],
                                                  rec['dtype'],
                                                  rec['val'])

            controlfile.write(rec)

        controlfile.close()

    @staticmethod
    def load(controlfile):
        """
        Entry point to an existing GSFLOW control file
        Parameters:
        ----------
            controlfile: (str) GSFLOW control file name
        """
        return Control(open(controlfile, 'r'))

class ModelUtils(object):
    """
    Class containing useful model utilities to manage workspaces,
    etc ...
    Parameters:
    ----------
        ws: (str) model workspace (path containing the GSFLOW controlfile)
    """

    def __init__(self, ws):

        self.ws = ws
        # set up wsl

        self.__wsl = re.split('\/|\\\\|', ws)
        self.__wsl[0] += '\\'

    @property
    def _output_keys(self):
        return ('csv_output_file', 'gsflow_output_file',
                'mondel_output_file','humidity_day',
                'tmax_day', 'tmin_day', 'precip_day',
                'swrad_day', 'transp_day', 'windspeed_day',
                'potet_day', 'stat_var_file', 'var_init_file',
                'var_save_file', 'ani_output_file','nhruOutBaseFileName')

    def get_path(self, path):
        """
        Method to change the workspace path from control file
        to location of prms model files and/or modflow model files
        using path information contained in the control file.

        Parameters:
        -----------
            path: (str) relative or absolute file path

        Returns:
        --------
            path: (str) absolute file path
        """
        pathlist = re.split('\/|\\\\|', path)

        # check for absolute path
        if ':' in pathlist[0]:
            pathlist[0] += '\\'
            path = os.path.join(*pathlist)

        else:
            wsl = copy.copy(self.__wsl)
            for cd in pathlist:
                if cd == "..":
                    del wsl[-1]
                else:
                    wsl.append(cd)

            path = os.path.join(*wsl)

        return path

    def create_temp_modflow_nam_file(self, nam):
        """
        Creates a temporary nam file in the gsflow directory
        because flopy assumes a workspace based on location of the
        modflow nam, not the location of the gsflow.control file.

        Parameters:
        ----------
            nam: (str) path to modflow nam file

        Returns:
        --------
            temp_nam: (str) returns the temporary modflow nam file name
        """
        idx = - nam[::-1].index('\\')
        temp_nam = nam[idx:]
        temp_path = os.path.join(self.ws, temp_nam)
        copyfile(nam, temp_path)
        return temp_nam

    def create_temp_prms_file(self, prms):
        """
        Method to create temporary prms files if there is more than
        one file of that type. Necessary method for prms_python to load
        multiple parameter and data files.

        Parameters:
        -----------
            prms: (list) list of absolute file paths to prms files

        Returns:
        --------
            path: (string) path to temp prms file
            d: (OrderedDict) {filename: [final param]} to preseve user file
                              structure and orginization during the write
                              methods
        """

        suffix = prms[0].split('.')[-1]
        temp_file = tempfile.NamedTemporaryFile(suffix=suffix,
                                                prefix='GSFloPyTemp',
                                                dir=self.ws, delete=False)
        d = OrderedDict()
        temp = None
        for prms_file in prms:
            with open(prms_file) as f:
                reverse_prms_file = reversed(list(f))
                for line in reverse_prms_file:
                    if '#' in line:
                        d[prms_file.split('\\')[-1]] = temp.strip('\n')
                        break
                    temp = line

            with open(prms_file) as f:
                temp_file.writelines(f.readlines())

        temp_file.close()
        return temp_file.name, d

    @staticmethod
    def rename_and_move_files(src, dest):
        """
        Method to rename, and move temp files as needed while during the
        Loading, editing and writing process.

        Parameters:
        -----------
        src: (str) file source location and name
        dest: (str) file destination
        """
        os.rename(src, dest)

    @staticmethod
    def split_path(path):
        """
        Method to extract only the specific file name or path from any
        path. May be irrelevent with some additional work.

        Parameters:
        ----------
            path: (str) absolute path a file

        Returns:
        --------
            path, name (str) filename
        """
        path = re.split('\/|\\\\|', path)
        if len(path) == 1:
            return None, path[-1]
        else:
            return '\\'.join(list(path[:-1])), path[-1]

class PRMSModel(ModelUtils):
    """
    Wrapper class for the PRMS python module used as a wrapper around the prms
    python suite. Uses a GSFlow Control file to either load or create new prms
    objects

    Parameters:
    -----------
    controlfile: (str, object <gsflow.Control>) name of gsflow control file for
                 building new prms files. if <gsflow.Control> object class loads
                 existing prms files.

    ws: (str) gsflow model workspace

    Attributes:
    -----------

    Methods:
    --------
    load: loads existing PRMS model files
    """
    # todo: look into the PRMS init_vars_from_file and var_init_file,
    # todo: formatting and if we can read in/out these using existing code
    # todo: structures
    def __init__(self, controlfile, ws):
        super(PRMSModel, self).__init__(ws)
        self.__parameter_files = []
        self.__parameter_files_dict = {}
        self.__data_files = []
        self.__data_files_dict = {}
        self.__header = "PRMS parameters file created with GSFloPy 0.1\n"
        self.__version = "Version: 1.7\n"

        if isinstance(controlfile, Control):
            # load model!
            self.__controlfile = controlfile
            self.parameters = self.__load_parameter_files()
            self.data = self.__load_data_files()

        else:
            # assume that controlfile must exist even if it is an empty document
            self.__controlfile = Control(open(controlfile))

    def set_header(self, header):
        if not header.endswith('\n'):
            header += '\n'
        self.__header = header

    @property
    def data_files(self):
        """
        Returns a list of the prms data files
        """
        if len(self.__data_files) < 1:
            try:
                if self.__controlfile.get_record('data_file')['nval'] == 1:
                    self.__data_files = [self.get_path(
                        self.__controlfile['data_file'])]
                else:
                    self.__data_files = [self.get_path(i) for i in
                                         self.__controlfile['data_file']]
            except KeyError:
                self.__data_files = []

        return self.__data_files

    @property
    def parameter_files(self):
        """
        Returns a list of the prms parameter files
        """
        if len(self.__parameter_files) < 1:
            try:
                if self.__controlfile.get_record('param_file')['nval'] == 1:
                    self.__parameter_files = [self.get_path(
                        self.__controlfile['param_file'])]
                else:
                    self.__parameter_files = [self.get_path(i) for i in
                                              self.__controlfile['param_file']]
            except KeyError:
                self.__parameter_files = []

        return self.__parameter_files

    def __load_data_files(self):
        """
        Convience method to load prms data files listed in the
        GSFLOW control file information. Creates and removes a temporary
        file in the GSFLOW ws directory to manage multiple datafiles

        Returns:
        -------
        <class 'prms_python.data.Data>
        """
        if len(self.data_files) < 1:
            warnings.warn("Control file does not have prms data" +
                          " files included, skipping pmrs data import")
            return

        else:
            data_file, self.__data_files_dict = \
                self.create_temp_prms_file(self.data_files)
            df = pp.Data(data_file)
            # os.remove(data_file)
            return df

    def __load_parameter_files(self):
        """
        Convience method to load prms parameter files listed in the
        GSFLOW control file information. Creates a temporary file in the
        GSFLOW ws directory.

        Returns:
        --------
        <class 'prms_python.parameters.Parameters'>
        """
        if len(self.parameter_files) < 1:
            warnings.warn("Control file does not have prms paramter" +
                          " files included, skipping pmrs parameter import")
            return

        else:
            param_file, self.__parameter_files_dict = \
                self.create_temp_prms_file(self.parameter_files)
            p = pp.Parameters(param_file)
            p.base_file_reader.close()
            # os.remove(param_file)
            return p

    def write(self, ws):
        """
        Write function that replaces the PRMS python write function
        Improvements allow for writing of multiple files based on input
        structure.
        Parameters:
        -----------
            ws (str): workspace for GSFlow model
        """
        super(PRMSModel, self).__init__(ws)
        self.__parameter_files = []

        fnum = 0
        fname = self.parameter_files[fnum]
        path = '\\'.join(fname.split('\\')[:-1])
        end_of_file = self.__parameter_files_dict[fname.split('\\')[-1]]

        if not os.path.exists(path):
            os.makedirs(path)

        params_file = open(fname, 'w')
        params_file.write(self.__header)
        params_file.write(self.__version)
        # begin writing model dimensions
        params_file.write("** Dimensions **\n")
        for key, val in self.parameters.dimensions.items():
            params_file.write("####\n")
            params_file.write("{}\n".format(key))
            params_file.write("{}\n".format(val))

        # begin writing model parameters
        params_file.write("** Parameters **\n")
        for d in self.parameters.base_params:
            params_file.write("####\n")
            params_file.write("{}\n".format(d['name']))
            params_file.write("{}\n".format(d['ndims']))
            for dimname in d['dimnames']:
                params_file.write("{}\n".format(dimname))
            params_file.write("{}\n".format(d["length"]))
            params_file.write("{}\n".format(d["vartype"]))

            try:
                for val in self.parameters.param_arrays[d['name']]:
                    if d['ndims'] == 2:
                        for v in val:
                            params_file.write("{}\n".format(v))
                    else:
                        params_file.write("{}\n".format(val))

            except KeyError:
                try:
                    for val in self.parameters[d['name']]:
                        if d['ndims'] == 2:
                            for v in val:
                                params_file.write("{}\n".format(v))
                        else:
                            params_file.write("{}\n".format(val))
                except TypeError:
                    params_file.write("{}\n".format(self.parameters[d['name']]))

            if d['name'] == end_of_file:
                params_file.close()
                fnum += 1
                try:
                    fname = self.parameter_files[fnum]
                    params_file = open(fname, "w")
                    end_of_file = self.__parameter_files_dict[
                        fname.split('\\')[-1]]
                except IndexError:
                    pass

        os.remove(self.parameters.base_file)

        self.__data_files = []
        fname = self.data_files[0]
        self.data.write(fname)
        os.remove(self.data.base_file)

    @staticmethod
    def load(controlfile, ws):
        """
        Method to load prms files from a GSFlow control file object

        Parameters:
        ----------
        controlfile: (object) <gsflow.Control> object
        """
        if isinstance(controlfile, str):
            controlfile = Control.load(controlfile)

        elif isinstance(controlfile, Control):
            pass

        else:
            raise TypeError("Controlfile is not of recognized type")

        return PRMSModel(controlfile, ws)

class GSFlowModel(ModelUtils):
    """
    Object to load, create, and write a full GSFLOW model. Utilizes
    functionality from Flopy 3.2.6 and prms_python.

    Parameters
    ----------
        controlfile: (str) GSFLOW controlfile name
        ws: (str) model workspace (path containing the GSFLOW controlfile)
    Methods
    -------
        load: Entry point to load an existing GSFLOW model

    Attributes:
    -----------

    """
    def __init__(self, controlfile, ws=None):

        self.__modflow_nam_file_path = None
        self.__parameter_files = []
        self.__parameter_files_dict = {}
        self.__data_files = []
        self.__config_files_dict = {}

        if ws is None:
            self.__ws = os.path.dirname(os.path.realpath(__file__))
        else:
            self.__ws = ws

        super(GSFlowModel, self).__init__(self.__ws)

        if isinstance(controlfile, Control):
            self.controlfile = controlfile
            self.name = controlfile.name
            self.modflow = self.__load_modflow_model()
            self.modflow_nam = ModFlowNam(self.modflow_nam_file_path)
            self.prms = self.__load_prms_model()

        elif isinstance(controlfile, str):

            self.name = os.path.join(self.__ws, controlfile)
            self.controlfile = Control.load(self.name)

    def __load_prms_model(self):
        prms = PRMSModel.load(self.controlfile, self.__ws)
        return prms

    def __load_modflow_model(self):
        """
        Convience method to load a modflow model from the GSFLOW control
        file information, reads a temporary modflow NAM file beacause of
        workspace conflicts with GSFlow and Flopy 3.2.6 assumptions

        Returns:
        --------
        <class 'flopy.modflow.mf.Modflow'> ie. a Flopy model object
        """
        if self.modflow_nam_file_path is None:
            warnings.warn("control file does not have modflow_name" +
                          " skipping modflow model import")
            return

        else:
            # Create a temporary name file with absolute name space in
            # the modflow directory and remove after model instance is created
            nam = self.create_temp_modflow_nam_file(self.modflow_nam_file_path)
            mf = flopy.modflow.Modflow.load(f=nam, model_ws=self.__ws,
                                            version="mfnwt")
            os.remove(os.path.join(self.__ws, nam))
            return mf

    @property
    def modflow_nam_file_path(self):
        """
        Returns the modflow NAM file path
        """
        if self.__modflow_nam_file_path is None:
            try:
                self.__modflow_nam_file_path = self.get_path(self.controlfile
                                                             ['modflow_name'])
            except KeyError:
                self.__modflow_nam_file_path = None

        return self.__modflow_nam_file_path

    @staticmethod
    def load(controlfile, ws=None):
        """
        Entry point to an existing GSFLOW control file
        Parameters:
        ----------
            controlfile: (str) GSFLOW control file name
            ws: (str) model workspace (path containing the GSFLOW controlfile)
        """
        controlfile = Control.load(os.path.join(ws, controlfile))
        return GSFlowModel(controlfile, ws=ws)

    def write(self, ws=None, controlfile=None, clear_relative_paths=False):
        """
        Method to write all configuration files for a complete GSFlow simulation
        and create output directories if they do not exist

        Parameters:
        -----------
            controlfile: (str) name of the control file if user wants to
                               change names... allows for multiple controlfiles
                               to be saved to the same workspace
            ws: (str) workspace aka. GSFlow model path
        """
        # todo: write output directories for model to run.
        if ws is None:
            ws = self.__ws
        else:
            self.__ws = ws
            self.__modflow_nam_file_path = None
            super(GSFlowModel, self).__init__(ws)

        modflow_ws, nam = self.split_path(self.modflow_nam_file_path)
        modflow_ws = self.get_path(modflow_ws)

        self.modflow.change_model_ws(modflow_ws)
        self.modflow.write_input()
        self.modflow_nam.write(modflow_ws,
                               clear_relative_paths=clear_relative_paths)
        self.prms.write(ws)

        control_ws, name = self.split_path(self.name)
        control_ws = os.path.join(ws, name)
        self.name = control_ws
        self.controlfile.write(control_ws)

        # build the output directory structure from the control file
        for key in self._output_keys:
            try:
                fpath = self.controlfile[key]
                path = self.split_path(fpath)[0]
                if len(path) > 0:
                    path = self.get_path(path)
                    if not os.path.exists(path):
                        os.makedirs(path)
            except KeyError:
                pass

        mfoutput = self.get_path(self.modflow_nam.gsflow_output_path)
        if not os.path.exists(mfoutput):
            os.makedirs(mfoutput)

    def run(self, controlfile=None, gsflow_exe='gsflow.exe'):
        """
        Method to run the GSFlow model contained in the GSFlowModel object

        Parameters:
        -----------
            gsflow_exe: (str) name and/or location of the gsflow.exe
        """
        from subprocess import Popen, PIPE
        warnings.warn('Please write GSFlowModel object prior to running edited instances of the model')

        if controlfile is None:
            controlfile = self.controlfile

        if isinstance(controlfile, file):
            controlfile = controlfile.name

        elif isinstance(controlfile, Control):
            controlfile = controlfile.name

        elif isinstance(controlfile, str):
            pass

        else:
            raise TypeError('Unrecognised type for controlfile')

        p = Popen([gsflow_exe, controlfile], stdout=PIPE,
                  stderr=PIPE, cwd=self.ws)
        while True:
            stdout_line = p.stdout.readline()
            c = stdout_line.decode('utf-8')
            if c != '':
                c = c.strip('\r\n')
                print('{}'.format(c))

        p.kill()
        return

class ModFlowNam(OrderedDict):
    """
    Class to read & write custom Gsflow based modflow nam files
    maintain old relative paths.

    Parameters
    ----------
        nam: (str, file) if nam is a string, it must be the absolute path
                         to the modflow name file. Can also be an open
                         modflow name file.
    """
    def __init__(self, nam):
        super(ModFlowNam, self).__init__()
        self.__input_path = ''
        self.__output_path = ''
        self.__header = '#Name file for GSFLOW MODFLOW-NWT model,' + \
                        ' generated by GSFloPy\n'
        self.spatial_reference = ''

        if isinstance(nam, file):
            self.__nam = nam.name.split('\\')[-1]
            self.__read_nam_file(nam)

        elif isinstance(nam, str):
            nam = nam.replace('/', '\\')
            self.__nam = nam.split('\\')[-1]
            if os.path.exists(nam):
                self.__read_nam_file(open(nam, 'r'))
            else:
                raise AssertionError('NAM file not found')
        else:
            raise TypeError('nam file not of recognized type <file> or <str>')

    def __read_nam_file(self, nam):
        """
        Method to read the modflow nam file and set its values to an
        ordered dictionary, maintains relative paths for operation with
        GSFlow model.

        Parameters:
        -----------
            name: (file) open modflow nam file
        """
        data = nam.readlines()
        for i, pkg in enumerate(data):
            if i <= 1 and pkg.startswith('#'):
                    if 'xul' in pkg:
                        self.spatial_reference = pkg

            else:
                pkg = [i for i in pkg.strip('\n').split(' ') if i != '']

                if len(pkg) > 3:
                    replace = pkg.pop(-1)
                else:
                    replace = ''

                path = pkg.pop(-1).split('\\')
                name = path.pop(-1)
                path = '\\'.join(path)
                unit = pkg[1]
                pkg = pkg[0]

                if pkg.lower() == 'dis':
                    self.__input_path = path

                elif pkg.lower() == 'list':
                    self.__output_path = path

                else:
                    pass

                rec = dict(name=name,
                           rel_path=path,
                           unit=unit,
                           pkg=pkg,
                           replace=replace)

                super(ModFlowNam, self).__setitem__(name, rec)

    def __check_if_input(self, pkg):
        """
        Checks if the pkg is considered input or output

        Parameters:
        -----------
            pkg: (str) the modflow namefile pakage name

        Returns:
        --------
            boolean
        """
        output = ('list', 'data', 'data(binary)')
        # input = ()
        if pkg.lower() in output:
            return False
        else:
            return True

    def __check_units(self, rec, mfnam):
        """
        Checks the flopy modflow produced nam file for potential unit conflicts,
        especially when mfuzf1 package is used:

        Parameters:
        -----------
            d (dict) this an mfnam record from the flopy temp nam file.
            mfnam <ModFlowNam object> corresponding to the flopy temp nam file

        Returns:
        --------
        bool
        """
        name = rec['name']
        unit = rec['unit']
        extension = name.split('.')[-1]

        if name in self:
            return True
        else:
            if extension.lower() in ('uzfbt1', 'uzfbt2'):
                for key, record in mfnam.items():
                    if record['name'].split('.')[-1].lower() == 'cbc':
                        if unit == record['unit']:
                            return False
                return True

            elif 'uzfb' in extension.lower():
                for key, record in self.items():
                    if unit == record['unit']:
                        return False
                return True

            else:
                return True

    def clear_path(self):
        """
        Removes the stored relative path from metadata for each pkg
        """
        for key, rec in self.items():
            rec['rel_path'] = ''

    def set_input_relative_path(self, path):
        """
        Sets a new relative path in for all input files

        Parameters:
        -----------
            path: (str) relative path name from input
        """
        path = path.replace('/', '\\')
        for key, rec in self.items():
            input = self.__check_if_input(rec['pkg'])
            if input:
                rec['rel_path'] = path
        self.__input_path = path

    def set_output_relative_path(self, path):
        """
        Sets a new relative path in for all output files

        Parameters:
        -----------
            path: (str) relative path name from input
        """
        path = path.replace('/', '\\')
        for key, rec in self.items():
            input = self.__check_if_input(rec['pkg'])
            if not input:
                rec['rel_path'] = path
        self.__output_path = path

    @property
    def gsflow_output_path(self):
        return self.__output_path

    @property
    def gsflow_input_path(self):
        return self.__input_path

    @staticmethod
    def load(nam):
        """
        Simple method to open a modflow name file and instantiate the
        ModFlowNam class

        Parameters:
        -----------
            nam: (str) name of the modflow nam file

        Returns:
            <class gsflow.ModFlowNam> object
        """
        nam = open(nam, 'r')
        return ModFlowNam(nam)

    def write(self, mf_ws, clear_relative_paths=False):
        """
        Read and replace the flopy nam file with a nam file
        that includes the necessary relative paths for GSFlow to
        operate correctly

        Parameters:
        -----------
            mf_ws: (str) workspace for modflow model
            clear_relative_paths: (bool) if True assumes mf_ws is the same as
                                        the GSFlow control file workspace
        """
        __mfnam = os.path.join(mf_ws, self.__nam)
        mfnam = ModFlowNam.load(__mfnam)

        f = open(__mfnam, 'w')
        f.write(self.__header)
        f.write(mfnam.spatial_reference)
        if clear_relative_paths:
            for key, rec in mfnam.items():
                f.write('{:14} {:4} {} {}\n'.format(rec['pkg'],
                                                    rec['unit'],
                                                    rec['name'],
                                                    rec['replace']))

        for key, rec in mfnam.items():
            if self.__check_units(rec, mfnam):
                if key in self:
                    rec['rel_path'] = self[key]['rel_path']

                else:
                    mfinput = self.__check_if_input(mfnam[key]['pkg'])
                    if mfinput:
                        rec['rel_path'] = self.__input_path
                    else:
                        rec['rel_path'] = self.__output_path

                if rec['rel_path'] == '':
                    f.write('{:14} {:4} {}{} {}\n'.format(rec['pkg'],
                                                          rec['unit'],
                                                          rec['rel_path'],
                                                          rec['name'],
                                                          rec['replace']))
                else:

                    f.write('{:14} {:4} {}\\{} {}\n'.format(rec['pkg'],
                                                            rec['unit'],
                                                            rec['rel_path'],
                                                            rec['name'],
                                                            rec['replace']))
            else:
                pass

model = 'gsflow.control'
ws = r'C:\Users\jlarsen\Desktop\RussianRiver\GSFLOW_1.2.1\data\sagehen\windows'
# ws = r'C:\Users\jlarsen\Desktop\RussianRiver\WriteTest\gsflow'
gsflow_exe = r'C:\Users\jlarsen\Desktop\RussianRiver\GSFLOW_1.2.1\bin\gsflow.exe'
gsflow = GSFlowModel.load(model, ws=ws)
print(gsflow.modflow.get_package_list())
dis = gsflow.modflow.get_package('UZF')

ws = r'C:\Users\jlarsen\Desktop\RussianRiver\WriteTest\gsflow'
gsflow.write(ws)
gsflow.run(gsflow_exe=gsflow_exe)

# todo: compile list of parameters that RedRiver group would like to change
# todo: add, subtract. Create methods for these.

{'Popen': 'subprocess.Popen', 'PIPE': 'subprocess.PIPE'}.load


import requests
import csv
import time
import datetime
import dateutil.relativedelta

cookie = 'v_id=YOUR_ID; api_access_token=YOUR_API_TOKEN; csrftoken2=YOUR_CSRF_TOKEN'

start_date = datetime.datetime.strptime("2019-11-01", "%Y-%m-%d")

header_included = False

with open('venmo_data.csv', 'w', newline='') as csvfile:
    writer = csv.writer(csvfile, delimiter=',',
                            quotechar='"', quoting=csv.QUOTE_MINIMAL)
    for x in range(3):
        month_end_date = start_date + \
            dateutil.relativedelta.relativedelta(months=1, days=-1)

        url = 'https://api.venmo.com/v1/transaction-history?start_date=' + \
            start_date.strftime("%Y-%m-%d") + '&end_date=' + \
            month_end_date.strftime("%Y-%m-%d") + '&csv=true'

        r = requests.get(url, headers={"cookie": cookie})

        decoded_content = r.content.decode('utf-8')
        cr = csv.reader(decoded_content.splitlines(), delimiter=',')
        my_list = list(cr)
        length = len(my_list)
        printed = 0
        for row in my_list:
            printed += 1
            if (printed == 1):
                if (header_included):
                    # If this is the first row and we've already included the header we want to skip
                    continue
                else:
                    header_included = True
            if (printed == 2):
                # the second line is the begining balance for the period we are downloading
                continue
            if (printed > length - 4):
                # the last 4 lines of the csv are ending balance stuff, notes on
                # fees and some legal notes we want to ignore
                break

            # don't include Transfers (which is just transfering money back to the bank)
            # or merchant transactions (like paying for ubers or whatever)
            if(row[3] != 'Standard Transfer' and row[3] != 'Merchant Transaction'):
                print(row)
                writer.writerow(row)

        start_date = start_date - dateutil.relativedelta.relativedelta(months=1)
        # wait one second between requests just to avoid rate limiting issues or whatever
        time.sleep(1)


# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""GhostNet model define"""
from functools import partial
import math
import numpy as np
import mindspore.nn as nn
from mindspore.ops import operations as P
from mindspore import Tensor
from .quant import QuanConv

__all__ = ['ghostnet']

def _make_divisible(x, divisor=4):
    return int(np.ceil(x * 1. / divisor) * divisor)

class MyHSigmoid(nn.Cell):
    """
    Hard Sigmoid definition.

    Args:

    Returns:
        Tensor, output tensor.

    Examples:
        >>> MyHSigmoid()
    """

    def __init__(self):
        super(MyHSigmoid, self).__init__()
        self.relu6 = nn.ReLU6()

    def construct(self, x):
        return self.relu6(x + 3.) * 0.16666667

class Activation(nn.Cell):
    """
    Activation definition.

    Args:
        act_func(string): activation name.

    Returns:
         Tensor, output tensor.
    """

    def __init__(self, act_func):
        super(Activation, self).__init__()
        if act_func == 'relu':
            self.act = nn.ReLU()
        elif act_func == 'relu6':
            self.act = nn.ReLU6()
        elif act_func in ('hsigmoid', 'hard_sigmoid'):
            self.act = MyHSigmoid()
        elif act_func in ('hswish', 'hard_swish'):
            self.act = nn.HSwish()
        else:
            raise NotImplementedError

    def construct(self, x):
        return self.act(x)

class GlobalAvgPooling(nn.Cell):
    """
    Global avg pooling definition.

    Args:

    Returns:
        Tensor, output tensor.

    Examples:
        >>> GlobalAvgPooling()
    """

    def __init__(self, keep_dims=False):
        super(GlobalAvgPooling, self).__init__()
        self.mean = P.ReduceMean(keep_dims=keep_dims)

    def construct(self, x):
        x = self.mean(x, (2, 3))
        return x

class SE(nn.Cell):
    """
    SE warpper definition.

    Args:
        num_out (int): output channel.
        ratio (int): middle output ratio.

    Returns:
        Tensor, output tensor.

    Examples:
        >>> SE(4)
    """

    def __init__(self, num_out, ratio=4):
        super(SE, self).__init__()
        num_mid = _make_divisible(num_out // ratio)
        self.pool = GlobalAvgPooling(keep_dims=True)
        self.conv_reduce = QuanConv(in_channels=num_out, out_channels=num_mid,
                                    kernel_size=1, has_bias=True, pad_mode='pad')
        self.act1 = Activation('relu')
        self.conv_expand = QuanConv(in_channels=num_mid, out_channels=num_out,
                                    kernel_size=1, has_bias=True, pad_mode='pad')
        self.act2 = Activation('hsigmoid')
        self.mul = P.Mul()

    def construct(self, x):
        out = self.pool(x)
        out = self.conv_reduce(out)
        out = self.act1(out)
        out = self.conv_expand(out)
        out = self.act2(out)
        out = self.mul(x, out)
        return out

class ConvUnit(nn.Cell):
    """
    ConvUnit warpper definition.

    Args:
        num_in (int): Input channel.
        num_out (int): Output channel.
        kernel_size (int): Input kernel size.
        stride (int): Stride size.
        padding (int): Padding number.
        num_groups (int): Output num group.
        use_act (bool): Used activation or not.
        act_type (string): Activation type.

    Returns:
        Tensor, output tensor.

    Examples:
        >>> ConvUnit(3, 3)
    """

    def __init__(self, num_in, num_out, kernel_size=1, stride=1, padding=0, num_groups=1,
                 use_act=True, act_type='relu'):
        super(ConvUnit, self).__init__()
        self.conv = QuanConv(in_channels=num_in,
                             out_channels=num_out,
                             kernel_size=kernel_size,
                             stride=stride,
                             padding=padding,
                             group=num_groups,
                             has_bias=False,
                             pad_mode='pad')
        self.bn = nn.BatchNorm2d(num_out)
        self.use_act = use_act
        self.act = Activation(act_type) if use_act else None

    def construct(self, x):
        out = self.conv(x)
        out = self.bn(out)
        if self.use_act:
            out = self.act(out)
        return out

class GhostModule(nn.Cell):
    """
    GhostModule warpper definition.

    Args:
        num_in (int): Input channel.
        num_out (int): Output channel.
        kernel_size (int): Input kernel size.
        stride (int): Stride size.
        padding (int): Padding number.
        ratio (int): Reduction ratio.
        dw_size (int): kernel size of cheap operation.
        use_act (bool): Used activation or not.
        act_type (string): Activation type.

    Returns:
        Tensor, output tensor.

    Examples:
        >>> GhostModule(3, 3)
    """

    def __init__(self, num_in, num_out, kernel_size=1, stride=1, padding=0, ratio=2, dw_size=3,
                 use_act=True, act_type='relu'):
        super(GhostModule, self).__init__()
        init_channels = math.ceil(num_out / ratio)
        new_channels = init_channels * (ratio - 1)

        self.primary_conv = ConvUnit(num_in, init_channels, kernel_size=kernel_size, stride=stride, padding=padding,
                                     num_groups=1, use_act=use_act, act_type='relu')
        self.cheap_operation = ConvUnit(init_channels, new_channels, kernel_size=dw_size, stride=1, padding=dw_size//2,
                                        num_groups=init_channels, use_act=use_act, act_type='relu')
        self.concat = P.Concat(axis=1)

    def construct(self, x):
        x1 = self.primary_conv(x)
        x2 = self.cheap_operation(x1)
        return self.concat((x1, x2))

class GhostBottleneck(nn.Cell):
    """
    GhostBottleneck warpper definition.

    Args:
        num_in (int): Input channel.
        num_mid (int): Middle channel.
        num_out (int): Output channel.
        kernel_size (int): Input kernel size.
        stride (int): Stride size.
        act_type (str): Activation type.
        use_se (bool): Use SE warpper or not.

    Returns:
        Tensor, output tensor.

    Examples:
        >>> GhostBottleneck(16, 3, 1, 1)
    """

    def __init__(self, num_in, num_mid, num_out, kernel_size, stride=1, act_type='relu', use_se=False):
        super(GhostBottleneck, self).__init__()
        self.ghost1 = GhostModule(num_in, num_mid, kernel_size=1,
                                  stride=1, padding=0, act_type=act_type)

        self.use_dw = stride > 1
        self.dw = None
        if self.use_dw:
            self.dw = ConvUnit(num_mid, num_mid, kernel_size=kernel_size, stride=stride,
                               padding=self._get_pad(kernel_size), act_type=act_type, num_groups=num_mid, use_act=False)

        self.use_se = use_se
        if use_se:
            self.se = SE(num_mid)

        self.ghost2 = GhostModule(num_mid, num_out, kernel_size=1, stride=1,
                                  padding=0, act_type=act_type, use_act=False)

        self.down_sample = False
        if num_in != num_out or stride != 1:
            self.down_sample = True
        self.shortcut = None
        if self.down_sample:
            self.shortcut = nn.SequentialCell([
                ConvUnit(num_in, num_in, kernel_size=kernel_size, stride=stride,
                         padding=self._get_pad(kernel_size), num_groups=num_in, use_act=False),
                ConvUnit(num_in, num_out, kernel_size=1, stride=1,
                         padding=0, num_groups=1, use_act=False),
            ])
        self.add = P.Add()

    def construct(self, x):
        r"""construct of GhostNet BottleNeck"""
        shortcut = x
        out = self.ghost1(x)
        if self.use_dw:
            out = self.dw(out)
        if self.use_se:
            out = self.se(out)
        out = self.ghost2(out)
        if self.down_sample:
            shortcut = self.shortcut(shortcut)
        out = self.add(shortcut, out)
        return out

    def _get_pad(self, kernel_size):
        """set the padding number"""
        pad = 0
        if kernel_size == 1:
            pad = 0
        elif kernel_size == 3:
            pad = 1
        elif kernel_size == 5:
            pad = 2
        elif kernel_size == 7:
            pad = 3
        else:
            raise NotImplementedError
        return pad

class GhostNet(nn.Cell):
    """
    GhostNet architecture.

    Args:
        model_cfgs (Cell): number of classes.
        num_classes (int): Output number classes.
        multiplier (int): Channels multiplier for round to 8/16 and others. Default is 1.
        final_drop (float): Dropout number.
        round_nearest (list): Channel round to . Default is 8.
    Returns:
        Tensor, output tensor.

    Examples:
        >>> GhostNet(num_classes=1000)
    """

    def __init__(self, model_cfgs, num_classes=1000, multiplier=1., final_drop=0., round_nearest=8):
        super(GhostNet, self).__init__()
        self.cfgs = model_cfgs['cfg']
        self.inplanes = 16
        first_conv_in_channel = 3
        first_conv_out_channel = _make_divisible(multiplier * self.inplanes)

        self.conv_stem = QuanConv(in_channels=first_conv_in_channel,
                                  out_channels=first_conv_out_channel,
                                  kernel_size=3, padding=1, stride=2,
                                  has_bias=False, pad_mode='pad')
        self.bn1 = nn.BatchNorm2d(first_conv_out_channel)
        self.act1 = Activation('relu')

        self.blocks = []
        for layer_cfg in self.cfgs:
            self.blocks.append(self._make_layer(kernel_size=layer_cfg[0],
                                                exp_ch=_make_divisible(
                                                    multiplier * layer_cfg[1]),
                                                out_channel=_make_divisible(
                                                    multiplier * layer_cfg[2]),
                                                use_se=layer_cfg[3],
                                                act_func=layer_cfg[4],
                                                stride=layer_cfg[5]))
        output_channel = _make_divisible(
            multiplier * model_cfgs["cls_ch_squeeze"])
        self.blocks.append(ConvUnit(_make_divisible(multiplier * self.cfgs[-1][2]), output_channel,
                                    kernel_size=1, stride=1, padding=0, num_groups=1, use_act=True))
        self.blocks = nn.SequentialCell(self.blocks)

        self.global_pool = GlobalAvgPooling(keep_dims=True)
        self.conv_head = QuanConv(in_channels=output_channel,
                                  out_channels=model_cfgs['cls_ch_expand'],
                                  kernel_size=1, padding=0, stride=1,
                                  has_bias=True, pad_mode='pad')
        self.act2 = Activation('relu')
        self.squeeze = P.Flatten()
        self.final_drop = final_drop
        if self.final_drop > 0:
            self.dropout = nn.Dropout(p=1 - self.final_drop)

        self.classifier = nn.Dense(
            model_cfgs['cls_ch_expand'], num_classes, has_bias=True)

        self._initialize_weights()

    def construct(self, x):
        r"""construct of GhostNet"""
        x = self.conv_stem(x)
        x = self.bn1(x)
        x = self.act1(x)
        x = self.blocks(x)
        x = self.global_pool(x)
        x = self.conv_head(x)
        x = self.act2(x)
        x = self.squeeze(x)
        if self.final_drop > 0:
            x = self.dropout(x)
        x = self.classifier(x)
        return x

    def _make_layer(self, kernel_size, exp_ch, out_channel, use_se, act_func, stride=1):
        mid_planes = exp_ch
        out_planes = out_channel
        layer = GhostBottleneck(self.inplanes, mid_planes, out_planes,
                                kernel_size, stride=stride, act_type=act_func, use_se=use_se)
        self.inplanes = out_planes
        return layer

    def _initialize_weights(self):
        """
        Initialize weights.

        Args:

        Returns:
            None.

        Examples:
            >>> _initialize_weights()
        """
        self.init_parameters_data()
        for _, m in self.cells_and_names():
            if isinstance(m, (nn.Conv2d)):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.set_parameter_data(Tensor(np.random.normal(0, np.sqrt(2. / n),
                                                                    m.weight.data.shape).astype("float32")))
                if m.bias is not None:
                    m.bias.set_parameter_data(
                        Tensor(np.zeros(m.bias.data.shape, dtype="float32")))
            elif isinstance(m, nn.BatchNorm2d):
                m.gamma.set_parameter_data(
                    Tensor(np.ones(m.gamma.data.shape, dtype="float32")))
                m.beta.set_parameter_data(
                    Tensor(np.zeros(m.beta.data.shape, dtype="float32")))
            elif isinstance(m, nn.Dense):
                m.weight.set_parameter_data(Tensor(np.random.normal(
                    0, 0.01, m.weight.data.shape).astype("float32")))
                if m.bias is not None:
                    m.bias.set_parameter_data(
                        Tensor(np.zeros(m.bias.data.shape, dtype="float32")))

def ghostnet(model_name, **kwargs):
    """
    Constructs a GhostNet model
    """
    model_cfgs = {
        "1x": {
            "cfg": [
                # k, exp, c,  se,     nl,  s,
                # stage1
                [3, 16, 16, False, 'relu', 1],
                # stage2
                [3, 48, 24, False, 'relu', 2],
                [3, 72, 24, False, 'relu', 1],
                # stage3
                [5, 72, 40, True, 'relu', 2],
                [5, 120, 40, True, 'relu', 1],
                # stage4
                [3, 240, 80, False, 'relu', 2],
                [3, 200, 80, False, 'relu', 1],
                [3, 184, 80, False, 'relu', 1],
                [3, 184, 80, False, 'relu', 1],
                [3, 480, 112, True, 'relu', 1],
                [3, 672, 112, True, 'relu', 1],
                # stage5
                [5, 672, 160, True, 'relu', 2],
                [5, 960, 160, False, 'relu', 1],
                [5, 960, 160, True, 'relu', 1],
                [5, 960, 160, False, 'relu', 1],
                [5, 960, 160, True, 'relu', 1]],
            "cls_ch_squeeze": 960,
            "cls_ch_expand": 1280,
        },

        "nose_1x": {
            "cfg": [
                # k, exp, c,  se,     nl,  s,
                # stage1
                [3, 16, 16, False, 'relu', 1],
                # stage2
                [3, 48, 24, False, 'relu', 2],
                [3, 72, 24, False, 'relu', 1],
                # stage3
                [5, 72, 40, False, 'relu', 2],
                [5, 120, 40, False, 'relu', 1],
                # stage4
                [3, 240, 80, False, 'relu', 2],
                [3, 200, 80, False, 'relu', 1],
                [3, 184, 80, False, 'relu', 1],
                [3, 184, 80, False, 'relu', 1],
                [3, 480, 112, False, 'relu', 1],
                [3, 672, 112, False, 'relu', 1],
                # stage5
                [5, 672, 160, False, 'relu', 2],
                [5, 960, 160, False, 'relu', 1],
                [5, 960, 160, False, 'relu', 1],
                [5, 960, 160, False, 'relu', 1],
                [5, 960, 160, False, 'relu', 1]],
            "cls_ch_squeeze": 960,
            "cls_ch_expand": 1280,
        }
    }

    return GhostNet(model_cfgs[model_name], **kwargs)

ghostnet_1x = partial(ghostnet, model_name="1x", final_drop=0.8)
ghostnet_nose_1x = partial(ghostnet, model_name="nose_1x", final_drop=0.8)


'''
@author: Faizan-Uni-Stuttgart

Nov 11, 2020

4:01:31 PM

'''
import os
import time
import timeit
from pathlib import Path

from math import ceil

os.environ[str('MKL_NUM_THREADS')] = str(1)
os.environ[str('NUMEXPR_NUM_THREADS')] = str(1)
os.environ[str('OMP_NUM_THREADS')] = str(1)

import ogr
# import gdal
import numpy as np
import pandas as pd
import netCDF4 as nc
# import shapefile as shp

import matplotlib.pyplot as plt
from pathos.multiprocessing import ProcessPool

from faizpy import get_aligned_shp_bds_and_cell_size, cnvt_to_pt, chk_cntmt
from krige_mp_ftns_ft import (
    ordinary_kriging,)
#     simple_kriging,
#     external_drift_kriging,
#     inverse_distance_wtng)

plt.ioff()

DEBUG_FLAG = True

def main():

    main_dir = Path(
        r'P:\Synchronize\IWS\Testings\fourtrans_practice\multisite_phs_spec_corr\5min')

    os.chdir(main_dir)

    interp_var = 'ppt'

    ft_type = 'phs'

    #==========================================================================
    if interp_var == 'temp':
        # MEAN TEMPERATURE
        in_data_file = os.path.join(
            f'temperature_{ft_type}_spec_df.csv')

        in_vgs_file = os.path.join(
            r'temperature_cftns.csv')

        in_stns_coords_file = os.path.join(
            os.path.dirname(in_data_file),
            r'temperature_avg_coords.csv')

        out_dir = r'temperature_kriging'
        var_units = u'\u2103'  # 'centigrade'
        var_name = 'temperature'
        out_krig_net_cdf_file = f'kriging_1km_{ft_type}.nc'

        # interpolated values
        # can be int, float, 'min_in'/'max_in' or None
        # min_var_val = 'min_in'
        # max_var_val = 'max_in'
#         min_var_val = None
#         max_var_val = None

    #==========================================================================

    #==========================================================================
    elif interp_var == 'ppt':
        # PRECIPITATION
        in_data_file = os.path.join(
            f'ppt_median_{ft_type}_spec_df.csv')

        in_vgs_file = os.path.join(
            r'ppt_median_cftns.csv')

        in_stns_coords_file = os.path.join(
            os.path.dirname(in_data_file),
            r'metadata_ppt_gkz3_crds_subset.csv')

        out_dir = r'ppt_kg'
        var_units = 'mm'
        var_name = 'precipitation'
        out_krig_net_cdf_file = f'kg_1km_{ft_type}.nc'

        # interpolated values
        # can be int, float, 'min_in'/'max_in' or None
        # min_var_val = 'min_in'
        # max_var_val = 'max_in'
#         min_var_val = None
#         max_var_val = None

    #==========================================================================
    else:
        raise ValueError(f'Invalid value for interp_var: {interp_var}!')

    out_krig_net_cdf_file = out_krig_net_cdf_file

    # assuming in_drift_raster and in_stns_coords_file and in_bounds_shp_file
    # have the same coordinates system
    # assuming in_drift_rasters_list have the same cell sizes, bounds and NDVs
    # basically they are copies of each other except for the drift values
    in_drift_rasters_list = (
        [r'P:\Synchronize\IWS\QGIS_Neckar\raster\lower_de_gauss_z3_1km.tif'])

#     in_bounds_shp_file = (
#         os.path.join(r'P:\Synchronize\IWS\QGIS_Neckar\raster',
#                      r'taudem_out_spate_rockenau\watersheds.shp'))

    in_bounds_shp_file = (
        os.path.join(r'P:\Synchronize\IWS\QGIS_Neckar\raster\taudem_out_spate_rockenau\watersheds.shp'))

    align_ras_file = in_drift_rasters_list[0]

    out_figs_dir = os.path.join(out_dir, 'krige_figs')

    x_coords_lab = 'X'
    y_coords_lab = 'Y'
    time_dim_lab = 'freq'
    nc_mode = 'w'

#     min_ppt_thresh = 1.0

    idw_exp = 5
    n_cpus = 1
    buffer_dist = 20e3
    sec_buffer_dist = 2e3

    in_sep = str(';')

    ord_krige_flag = True
    sim_krige_flag = True
    edk_krige_flag = True
    idw_flag = True
    plot_figs_flag = True

#     ord_krige_flag = False
    sim_krige_flag = False
    edk_krige_flag = False
    idw_flag = False
    plot_figs_flag = False

    os.chdir(main_dir)

    if not os.path.exists(out_dir):
        os.mkdir(out_dir)

    if (not os.path.exists(out_figs_dir)) and plot_figs_flag:
        os.mkdir(out_figs_dir)

#     print('min_var_val:', min_var_val)
#     print('max_var_val:', max_var_val)
    print('idw_exp:', idw_exp)
    print('n_cpus:', n_cpus)
    print('nc_mode:', nc_mode)
    print('var_name:', var_name)
    print('out_dir:', out_dir)
    print('in_bounds_shp_file:', in_bounds_shp_file)
    print('out_krig_net_cdf_file:', out_krig_net_cdf_file)

    assert any([ord_krige_flag, sim_krige_flag, edk_krige_flag, idw_flag])

    #==========================================================================
    # read the data frames
    #==========================================================================
    in_data_df = pd.read_csv(
        in_data_file,
        sep=in_sep,
        index_col=0,
        encoding='utf-8')

    in_vgs_df = pd.read_csv(
        in_vgs_file,
        sep=in_sep,
        index_col=0,
        encoding='utf-8')

    in_stns_coords_df = pd.read_csv(
        in_stns_coords_file,
        sep=in_sep,
        index_col=0,
        encoding='utf-8')

    all_stns = in_data_df.columns.intersection(in_stns_coords_df.index)
    assert all_stns.shape[0]

    in_data_df = in_data_df.loc[:, all_stns]
    in_stns_coords_df = in_stns_coords_df.loc[all_stns, :]

    #==========================================================================
    # Get stations that are around/in the bounds_shp only
    #==========================================================================

    bds_vec = ogr.Open(in_bounds_shp_file)
    assert bds_vec

    bds_lyr = bds_vec.GetLayer(0)

    feat_buffs_list = []
    feat_sec_buffs_list = []
    for feat in bds_lyr:  # just to get the names of the catchments
        geom = feat.GetGeometryRef().Clone()
        assert geom

        feat_buffs_list.append(geom.Buffer(buffer_dist))
        feat_sec_buffs_list.append(geom.Buffer(sec_buffer_dist))

    bds_vec.Destroy()

    assert feat_buffs_list and feat_sec_buffs_list

    print(len(feat_buffs_list), 'polygons in the in_bounds_shp_file...')

    fin_stns = []
    for poly in feat_buffs_list:
        for stn in all_stns:
            if stn in fin_stns:
                continue

            curr_pt = cnvt_to_pt(
                *in_stns_coords_df.loc[stn, ['X', 'Y']].values)

            if chk_cntmt(curr_pt, poly):
                fin_stns.append(stn)

    assert fin_stns

    print('%d stations out of %d within buffer zone of in_bounds_shp_file' %
          (len(fin_stns), in_stns_coords_df.shape[0]))

    fin_stns = np.unique(fin_stns)
    in_data_df = in_data_df.loc[:, fin_stns]
    in_stns_coords_df = in_stns_coords_df.loc[fin_stns, :]

    #==========================================================================
    # Read the DEM
    #==========================================================================

#     if edk_krige_flag:
#         in_drift_arr_list = []
#         _rows_list = []
#         _cols_list = []
#
#         for in_drift_raster in in_drift_rasters_list:
#             in_drift_ds = gdal.Open(in_drift_raster)
#
#             assert in_drift_ds, 'GDAL cannot open %s' % in_drift_raster
#
#             drift_rows = in_drift_ds.RasterYSize
#             drift_cols = in_drift_ds.RasterXSize
#
#             drift_geotransform = in_drift_ds.GetGeoTransform()
#
#             _drift_x_min = drift_geotransform[0]
#             _drift_y_max = drift_geotransform[3]
#
#             drift_band = in_drift_ds.GetRasterBand(1)
#             drift_ndv = drift_band.GetNoDataValue()
#
#             cell_width = drift_geotransform[1]
#             cell_height = abs(drift_geotransform[5])
#
#             _drift_x_max = _drift_x_min + (drift_cols * cell_width)
#             _drift_y_min = _drift_y_max - (drift_rows * cell_height)
#
#             _arr = in_drift_ds.ReadAsArray()
#
#             in_drift_arr_list.append(_arr)
#             _rows_list.append(_arr.shape[0])
#             _cols_list.append(_arr.shape[1])
#
#         assert all(_ == _rows_list[0] for _ in _rows_list), (
#             'Drift raster have unequal number of rows!')
#
#         assert all(_ == _cols_list[0] for _ in _cols_list), (
#             'Drift raster have unequal number of columns!')

    #==========================================================================
    # Read the bounding shapefile
    #==========================================================================
#     sf = shp.Reader(in_bounds_shp_file)
#     polys_list = [i.__geo_interface__ for i in sf.iterShapes()]

    ((fin_x_min,
      fin_x_max,
      fin_y_min,
      fin_y_max),
     cell_width) = get_aligned_shp_bds_and_cell_size(
         in_bounds_shp_file, align_ras_file)

    cell_height = cell_width

    fin_x_min -= 2 * cell_width
    fin_x_max += 2 * cell_width
    fin_y_min -= 2 * cell_height
    fin_y_max += 2 * cell_height

#     if edk_krige_flag:
#         assert fin_x_min > _drift_x_min
#         assert fin_x_max < _drift_x_max
#         assert fin_y_min > _drift_y_min
#         assert fin_y_max < _drift_y_max
#
#         min_col = int(max(0, (fin_x_min - _drift_x_min) / cell_width))
#         max_col = int(ceil((fin_x_max - _drift_x_min) / cell_width))
#
#         min_row = int(max(0, (_drift_y_max - fin_y_max) / cell_height))
#         max_row = int(ceil((_drift_y_max - fin_y_min) / cell_height))
#
#     else:
    min_col = 0
    max_col = int(ceil((fin_x_max - fin_x_min) / cell_width))

    min_row = 0
    max_row = int(ceil((fin_y_max - fin_y_min) / cell_height))

    #==========================================================================
    # Calculate coordinates at which to krige
    #==========================================================================

    assert 0 <= min_col <= max_col, (min_col, max_col)
    assert 0 <= min_row <= max_row, (min_row, max_row)

    strt_x_coord = fin_x_min + (0.5 * cell_width)
    end_x_coord = strt_x_coord + ((max_col - min_col) * cell_width)

    strt_y_coord = fin_y_max - (0.5 * cell_height)
    end_y_coord = strt_y_coord - ((max_row - min_row) * cell_height)

    krige_x_coords = np.linspace(
        strt_x_coord, end_x_coord, (max_col - min_col + 1))

    krige_y_coords = np.linspace(
        strt_y_coord, end_y_coord, (max_row - min_row + 1))

    krige_x_coords_mesh, krige_y_coords_mesh = np.meshgrid(
        krige_x_coords, krige_y_coords)

    krige_coords_orig_shape = krige_x_coords_mesh.shape

#     if plot_figs_flag:
#         # xy coords for pcolormesh
#         pcolmesh_x_coords = np.linspace(
#             fin_x_min, fin_x_max, (max_col - min_col + 1))
#
#         pcolmesh_y_coords = np.linspace(
#             fin_y_max, fin_y_min, (max_row - min_row + 1))
#
#         krige_x_coords_plot_mesh, krige_y_coords_plot_mesh = (
#             np.meshgrid(pcolmesh_x_coords, pcolmesh_y_coords))
#
#     else:
#         krige_x_coords_plot_mesh, krige_y_coords_plot_mesh = None, None

    krige_x_coords_mesh = krige_x_coords_mesh.ravel()
    krige_y_coords_mesh = krige_y_coords_mesh.ravel()

#     print('\n\n')
#     print('#' * 10)
#
#     _beg_t = timeit.default_timer()
#
#     print(krige_x_coords_mesh.shape[0],
#           'cells to interpolate per step before intersection!')
#
    fin_cntn_idxs = np.ones(krige_x_coords_mesh.shape[0], dtype=bool)
#     fin_cntn_idxs = np.zeros(krige_x_coords_mesh.shape[0], dtype=bool)
#     ogr_pts = np.vectorize(cnvt_to_pt)(krige_x_coords_mesh, krige_y_coords_mesh)
#
#     for poly in feat_sec_buffs_list:
#         curr_cntn_idxs = np.vectorize(chk_cntmt)(ogr_pts, poly)
#         fin_cntn_idxs = fin_cntn_idxs | curr_cntn_idxs
#
#     print(fin_cntn_idxs.sum(),
#           'cells to interpolate per step after intersection!')
#
#     _end_t = timeit.default_timer()
#     _tot_t = _end_t - _beg_t
#
#     print(f'Took {_tot_t:0.4f} seconds!')
#     print('#' * 10)
#
#     krige_x_coords_mesh = krige_x_coords_mesh[fin_cntn_idxs]
#     krige_y_coords_mesh = krige_y_coords_mesh[fin_cntn_idxs]

#     if edk_krige_flag:
#         drift_vals_list = []
#
#         krige_cols = np.arange(min_col, max_col + 1, dtype=int)
#         krige_rows = np.arange(min_row, max_row + 1, dtype=int)
#
#         assert krige_x_coords.shape[0] == krige_cols.shape[0]
#         assert krige_y_coords.shape[0] == krige_rows.shape[0]
#
#         (krige_drift_cols_mesh,
#          krige_drift_rows_mesh) = np.meshgrid(krige_cols, krige_rows)
#
#         krige_drift_cols_mesh = krige_drift_cols_mesh.ravel()
#         krige_drift_rows_mesh = krige_drift_rows_mesh.ravel()
#
#         krige_drift_cols_mesh = krige_drift_cols_mesh[fin_cntn_idxs]
#         krige_drift_rows_mesh = krige_drift_rows_mesh[fin_cntn_idxs]
#
#         for _drift_arr in in_drift_arr_list:
#             _drift_vals = _drift_arr[
#                 krige_drift_rows_mesh, krige_drift_cols_mesh]
#
#             drift_vals_list.append(_drift_vals)
#
# #         drift_vals_arr = np.array(drift_vals_list, dtype=float)
#
#         drift_df_cols = list(range(len(in_drift_rasters_list)))
#         in_stns_drift_df = pd.DataFrame(
#             index=in_stns_coords_df.index,
#             columns=drift_df_cols,
#             dtype=float)
#
#         for stn in in_stns_drift_df.index:
#             stn_x = in_stns_coords_df.loc[stn, x_coords_lab]
#             stn_y = in_stns_coords_df.loc[stn, y_coords_lab]
#
#             stn_col = int((stn_x - _drift_x_min) / cell_width)
#             stn_row = int((_drift_y_max - stn_y) / cell_height)
#
#             for col, _arr in zip(drift_df_cols, in_drift_arr_list):
#                 try:
#                     _ = _arr[stn_row, stn_col]
#                     if not np.isclose(drift_ndv, _):
#                         in_stns_drift_df.loc[stn, col] = _
#
#                 except IndexError:
#                     pass
#
#         in_stns_drift_df.dropna(inplace=True)

    #==========================================================================
    # Open NC
    #==========================================================================
    out_nc = nc.Dataset(
        os.path.join(out_dir, out_krig_net_cdf_file), mode=str(nc_mode))

    if nc_mode == 'w':
        out_nc.set_auto_mask(False)
        out_nc.createDimension(x_coords_lab, krige_x_coords.shape[0])
        out_nc.createDimension(y_coords_lab, krige_y_coords.shape[0])
        out_nc.createDimension(time_dim_lab, in_data_df.shape[0])

        x_coords_nc = out_nc.createVariable(
            x_coords_lab, 'd', dimensions=x_coords_lab)

        x_coords_nc[:] = krige_x_coords

        y_coords_nc = out_nc.createVariable(
            y_coords_lab, 'd', dimensions=y_coords_lab)

        y_coords_nc[:] = krige_y_coords

        time_nc = out_nc.createVariable(
            time_dim_lab, 'i8', dimensions=time_dim_lab)

        time_nc[:] = np.arange(in_data_df.shape[0])

    else:
        raise RuntimeError('Not configured for this option!')

        time_nc = out_nc.variables[time_dim_lab]
        krige_y_coords = y_coords_nc[:]
        krige_x_coords = x_coords_nc[:]

    #==========================================================================
    # MP stuff
    #==========================================================================
    mp_cond = False

    if ((n_cpus > 1) and  (in_data_df.shape[0] > (n_cpus + 1))):
        idxs = pd.np.linspace(
            0,
            in_data_df.shape[0],
            (n_cpus) + 1,
            endpoint=True,
            dtype=int)

        idxs = np.unique(idxs)
        print('MP idxs:', idxs)

        if idxs.shape[0] == 1:
            idxs = np.concatenate((np.array([0]), idxs))

        mp_cond = True

    else:
        idxs = [0, in_data_df.shape[0]]

    #==========================================================================
    # Krige
    #==========================================================================
    if ord_krige_flag:
        print('\n\n')
        print('#' * 10)

        _beg_t = timeit.default_timer()

        print('Ordinary Kriging...')

        if 'OK' not in out_nc.variables:
            ok_nc = out_nc.createVariable(
                'OK',
                'd',
                dimensions=(time_dim_lab, y_coords_lab, x_coords_lab),
                fill_value=False)

        else:
            ok_nc = out_nc.variables['OK']

        ok_vars_gen = ((in_data_df.iloc[idxs[i]:idxs[i + 1]],
                        in_stns_coords_df,
                        in_vgs_df.loc[ft_type][0],
                        krige_x_coords_mesh,
                        krige_y_coords_mesh,
                        krige_coords_orig_shape,
                        (idxs[i], idxs[i + 1]),
                        fin_cntn_idxs) for i in range(n_cpus))

        if mp_cond:
            ok_krige_flds = np.full(
                (in_data_df.shape[0],
                 krige_coords_orig_shape[0],
                 krige_coords_orig_shape[1]),
                np.nan,
                dtype=np.float32)

            mp_ress = []

            try:
                mp_pool = ProcessPool(n_cpus)
                mp_pool.restart(True)

                mp_ress = list(mp_pool.uimap(ordinary_kriging, ok_vars_gen))

                mp_pool.clear()

            except Exception as msg:
                mp_pool.close()
                mp_pool.join()
                print('Error in ordinary_kriging:', msg)

            for mp_res in mp_ress:
                if (len(mp_res) != 3) and (not isinstance(list)):
                    print('\n', mp_res, '\n')
                    continue

                [strt_index, end_index, sub_ok_krige_flds] = mp_res
                ok_krige_flds[strt_index:end_index] = sub_ok_krige_flds

                # free memory
                mp_res[2], sub_ok_krige_flds = None, None

            ok_nc[:] = ok_krige_flds

        else:
            [strt_index,
             end_index,
             ok_krige_flds] = ordinary_kriging(next(ok_vars_gen))

            ok_nc[:] = ok_krige_flds

        ok_nc.units = var_units
        ok_nc.standard_name = var_name + ' (ordinary kriging)'

        ok_krige_flds = None

        _end_t = timeit.default_timer()
        _tot_t = _end_t - _beg_t

        print(f'Took {_tot_t:0.4f} seconds!')
        print('#' * 10)

#     if sim_krige_flag:
#         print('\n\n')
#         print('#' * 10)
#
#         _beg_t = timeit.default_timer()
#
#         print('Simple Kriging...')
#         if 'SK' not in out_nc.variables:
#             sk_nc = out_nc.createVariable(
#                 'SK',
#                 'd',
#                 dimensions=(time_dim_lab, y_coords_lab, x_coords_lab),
#                 fill_value=False)
#
#         else:
#             sk_nc = out_nc.variables['SK']
#
#         sk_vars_gen = ((in_data_df.iloc[idxs[i]:idxs[i + 1]],
#                         in_stns_coords_df,
#                         in_vgs_df.iloc[idxs[i]:idxs[i + 1]],
#                         min_ppt_thresh,
#                         var_name,
#                         krige_x_coords_mesh,
#                         krige_y_coords_mesh,
#                         krige_coords_orig_shape,
#                         min_var_val,
#                         max_var_val,
#                         (idxs[i], idxs[i + 1]),
#                         plot_figs_flag,
#                         krige_x_coords_plot_mesh,
#                         krige_y_coords_plot_mesh,
#                         var_units,
#                         polys_list,
#                         out_figs_dir,
#                         fin_cntn_idxs) for i in range(n_cpus))
#
#         if mp_cond:
#             sk_krige_flds = np.full(
#                 (in_data_df.shape[0],
#                  krige_coords_orig_shape[0],
#                  krige_coords_orig_shape[1]),
#                 np.nan,
#                 dtype=np.float32)
#
#             mp_ress = []
#
#             try:
#                 mp_pool = ProcessPool(n_cpus)
#                 mp_pool.restart(True)
#
#                 mp_ress = list(mp_pool.uimap(simple_kriging, sk_vars_gen))
#
#                 mp_pool.clear()
#
#             except Exception as msg:
#                 mp_pool.close()
#                 mp_pool.join()
#                 print('Error in simple_kriging:', msg)
#
#             for mp_res in mp_ress:
#                 if (len(mp_res) != 3) and (not isinstance(list)):
#                     print('\n', mp_res, '\n')
#                     continue
#
#                 [strt_index, end_index, sub_sk_krige_flds] = mp_res
#                 sk_krige_flds[strt_index:end_index] = sub_sk_krige_flds
#
#                 # free memory
#                 mp_res[2], sub_sk_krige_flds = None, None
#
#             sk_nc[:] = sk_krige_flds
#
#         else:
#             [strt_index,
#              end_index,
#              sk_krige_flds] = simple_kriging(next(sk_vars_gen))
#
#             sk_nc[:] = sk_krige_flds
#
#         sk_nc.units = var_units
#         sk_nc.standard_name = var_name + ' (simple kriging)'
#
#         sk_krige_flds = None
#
#         _end_t = timeit.default_timer()
#         _tot_t = _end_t - _beg_t
#
#         print(f'Took {_tot_t:0.4f} seconds!')
#         print('#' * 10)
#
#     if edk_krige_flag:
#         print('\n\n')
#         print('#' * 10)
#
#         _beg_t = timeit.default_timer()
#
#         print('External Drift Kriging...')
#         if 'EDK' not in out_nc.variables:
#             edk_nc = out_nc.createVariable(
#                 'EDK',
#                 'd',
#                 dimensions=(time_dim_lab, y_coords_lab, x_coords_lab),
#                 fill_value=False)
#
#         else:
#             edk_nc = out_nc.variables['EDK']
#
#         edk_vars_gen = ((in_data_df.iloc[idxs[i]:idxs[i + 1]],
#                          in_stns_drift_df,
#                          in_stns_coords_df,
#                          in_vgs_df.iloc[idxs[i]:idxs[i + 1]],
#                          min_ppt_thresh,
#                          var_name,
#                          krige_x_coords_mesh,
#                          krige_y_coords_mesh,
#                          drift_vals_arr,
#                          krige_coords_orig_shape,
#                          drift_ndv,
#                          min_var_val,
#                          max_var_val,
#                          (idxs[i], idxs[i + 1]),
#                          plot_figs_flag,
#                          krige_x_coords_plot_mesh,
#                          krige_y_coords_plot_mesh,
#                          var_units,
#                          polys_list,
#                          out_figs_dir,
#                          fin_cntn_idxs) for i in range(n_cpus))
#
#         if mp_cond:
#             edk_krige_flds = np.full(
#                 (in_data_df.shape[0],
#                  krige_coords_orig_shape[0],
#                  krige_coords_orig_shape[1]),
#                 np.nan,
#                 dtype=np.float32)
#
#             mp_ress = []
#
#             try:
#                 mp_pool = ProcessPool(n_cpus)
#                 mp_pool.restart(True)
#
#                 mp_ress = list(mp_pool.uimap(
#                     external_drift_kriging, edk_vars_gen))
#
#                 mp_pool.clear()
#
#             except Exception as msg:
#                 mp_pool.close()
#                 mp_pool.join()
#                 print('Error in external_drift_kriging:', msg)
#
#             for mp_res in mp_ress:
#                 if (len(mp_res) != 3) and (not isinstance(list)):
#                     print('\n', mp_res, '\n')
#                     continue
#
#                 [strt_index, end_index, sub_edk_krige_flds] = mp_res
#                 edk_krige_flds[strt_index:end_index] = sub_edk_krige_flds
#
#                 print('sub_min:', np.nanmin(sub_edk_krige_flds))
#                 print('sub_max:', np.nanmax(sub_edk_krige_flds))
#
#                 # free memory
#                 mp_res[2], sub_edk_krige_flds = None, None
#
#         else:
#             [strt_index,
#              end_index,
#              edk_krige_flds] = external_drift_kriging(next(edk_vars_gen))
#
#         edk_nc[:] = edk_krige_flds
#
#         edk_nc.units = var_units
#         edk_nc.standard_name = var_name + ' (external drift kriging)'
#
#         edk_krige_flds = None
#
#         _end_t = timeit.default_timer()
#         _tot_t = _end_t - _beg_t
#
#         print(f'Took {_tot_t:0.4f} seconds!')
#         print('#' * 10)
#
#     #==========================================================================
#     # IDW
#     #==========================================================================
#     if idw_flag:
#         print('\n\n')
#         print('#' * 10)
#
#         _beg_t = timeit.default_timer()
#
#         print('Inverse Distance Weighting...')
#         if 'IDW' not in out_nc.variables:
#             idw_nc = out_nc.createVariable(
#                 'IDW',
#                 'd',
#                  dimensions=(time_dim_lab, y_coords_lab, x_coords_lab),
#                  fill_value=False)
#
#         else:
#             idw_nc = out_nc.variables['IDW']
#
#         idw_vars_gen = ((in_data_df.iloc[idxs[i]:idxs[i + 1]],
#                         in_stns_coords_df,
#                         min_ppt_thresh,
#                         idw_exp,
#                         var_name,
#                         krige_x_coords_mesh,
#                         krige_y_coords_mesh,
#                         krige_coords_orig_shape,
#                         min_var_val,
#                         max_var_val,
#                         (idxs[i], idxs[i + 1]),
#                         plot_figs_flag,
#                         krige_x_coords_plot_mesh,
#                         krige_y_coords_plot_mesh,
#                         var_units,
#                         polys_list,
#                         out_figs_dir,
#                         fin_cntn_idxs) for i in range(n_cpus))
#
#         if mp_cond:
#             idw_flds = np.full(
#                 (in_data_df.shape[0],
#                  krige_coords_orig_shape[0],
#                  krige_coords_orig_shape[1]),
#                 np.nan,
#                 dtype=np.float32)
#
#             mp_ress = []
#             try:
#                 mp_pool = ProcessPool(n_cpus)
#                 mp_pool.restart(True)
#
#                 mp_ress = list(mp_pool.uimap(
#                     inverse_distance_wtng, idw_vars_gen))
#
#                 mp_pool.clear()
#
#             except Exception as msg:
#                 mp_pool.close()
#                 mp_pool.join()
#                 print('Error in inverse_distance_wtng:', msg)
#
#             for mp_res in mp_ress:
#                 if (len(mp_res) != 3) and (not isinstance(list)):
#                     print('\n', mp_res, '\n')
#                     continue
#
#                 [strt_index, end_index, sub_idw_flds] = mp_res
#                 idw_flds[strt_index:end_index] = sub_idw_flds
#
#                 # free memory
#                 mp_res[2], sub_idw_flds = None, None
#
#         else:
#             [strt_index,
#              end_index,
#              idw_flds] = inverse_distance_wtng(next(idw_vars_gen))
#
#         idw_nc[:] = idw_flds
#
#         idw_nc.units = var_units
#         idw_nc.standard_name = (
#             var_name + ' (IDW (exp=%0.3f))' % float(idw_exp))
#
#         idw_flds = None
#
#         _end_t = timeit.default_timer()
#         _tot_t = _end_t - _beg_t
#
#         print(f'Took {_tot_t:0.4f} seconds!')
#         print('#' * 10)

    out_nc.Author = 'Faizan IWS Uni-Stuttgart'
    out_nc.Source = out_nc.filepath()
    out_nc.close()
    return

if __name__ == '__main__':

    _save_log_ = False
    if _save_log_:
        from datetime import datetime
        from std_logger import StdFileLoggerCtrl

        # save all console activity to out_log_file
        out_log_file = os.path.join(
            r'P:\Synchronize\python_script_logs\\%s_log_%s.log' % (
            os.path.basename(__file__),
            datetime.now().strftime('%Y%m%d%H%M%S')))

        log_link = StdFileLoggerCtrl(out_log_file)

    print('#### Started on %s ####\n' % time.asctime())
    START = timeit.default_timer()

    #==========================================================================
    # When in post_mortem:
    # 1. "where" to show the stack
    # 2. "up" move the stack up to an older frame
    # 3. "down" move the stack down to a newer frame
    # 4. "interact" start an interactive interpreter
    #==========================================================================

    if DEBUG_FLAG:
        try:
            main()

        except:
            import pdb
            pdb.post_mortem()

    else:
        main()

    STOP = timeit.default_timer()
    print(('\n#### Done with everything on %s.\nTotal run time was'
           ' about %0.4f seconds ####' % (time.asctime(), STOP - START)))

    if _save_log_:
        log_link.stop()


#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.

from typing import Dict, Tuple

import ffmpeg  # @manual
from augly.utils import pathmgr
from augly.video.augmenters.ffmpeg import BaseFFMPEGAugmenter
from augly.video.helpers import get_audio_info, get_video_info
from ffmpeg.nodes import FilterableStream

class VideoAugmenterByAudioSwap(BaseFFMPEGAugmenter):
    def __init__(self, audio_path: str, offset: float):
        assert offset >= 0, "Offset cannot be a negative number"

        self.audio_path = pathmgr.get_local_path(audio_path)
        self.offset = offset

    def add_augmenter(
        self, in_stream: FilterableStream, **kwargs
    ) -> Tuple[FilterableStream, Dict]:
        """
        Swaps the audio of a video

        @param in_stream: the FFMPEG object of the video

        @returns: a tuple containing the FFMPEG object with the augmentation
            applied and a dictionary with any output arguments as necessary
        """
        audio_info = get_audio_info(self.audio_path)
        video_info = get_video_info(kwargs["video_path"])

        audio_duration = float(audio_info["duration"])
        audio_sample_rate = float(audio_info["sample_rate"])

        start = self.offset
        end = start + float(video_info["duration"])

        audio = ffmpeg.input(self.audio_path).audio

        if end > audio_duration:
            pad_len = (end - audio_duration) * audio_sample_rate
            audio = audio.filter_("apad", pad_len=pad_len)

        audio = audio.filter_("atrim", start=start, end=end).filter_(
            "asetpts", "PTS-STARTPTS"
        )

        return ffmpeg.concat(in_stream.video, audio, v=1, a=1, n=1), {}


#! coding=utf-8
import click
import glob

def _index_of_substr(the_list, substr):
    for i, val in enumerate(the_list):
        if substr in val:
            return i
    return None

@click.command()
@click.option('--in-dir', help='Slow queries log dir.')
@click.option('--out-file', help='Queries file.')
def log_to_query(in_dir, out_file):
    with open(out_file, 'w') as fwrite:
        errs = []
        slow_prefix = 'elasticsearch_dev_2.1_v2_index_search_slowlog'
        slow_files = glob.glob(in_dir + '/' + slow_prefix + '*')
        for s_file in slow_files:
            with open(s_file) as fread:
                for line in fread:
                    line_list = line.split(', ')
                    ind = _index_of_substr(line_list, 'source')
                    try:
                        query = line_list[ind][7:-1]
                        fwrite.write(query + '\n')
                    except IndexError:
                        errs.append(line)
                        pass
        if errs:
            print('There were {} errors.'.format(len(errs)))
            print(errs)

if __name__ == '__main__':
    log_to_query()


import scrapy
from ..items import RumahItem

class RumahSpider(scrapy.Spider):
    name = 'rumah'
    start_urls = ['https://www.rumah123.com/jual/bandung/gudang',]
    page = 1

    def get_number(self,text):
        result=''
        try:
            iter(text)
        except TypeError:
            return None
        for i in text:
            try:
                result+=str(int(i))
            except ValueError:
                continue
        if result:
            return int(result)
        else:
            return None

    def parse(self, response):
        list_rumah = response.css('ul.listing-list')
        print(list_rumah)
        for url in list_rumah.css('a::attr(href)'):
            yield response.follow(url, callback=self.parse_detail)
        pages = response.css('li.ant-pagination-item::attr(title)').getall()
        for i in pages:
            try:
                if int(i)==self.page+1:
                    self.page=int(i)
                    yield response.follow(self.start_urls[0]+'/?page={}'.format(self.page), callback=self.parse)
                    break
            except ValueError:
                continue

    def parse_detail(self, response):
        item = RumahItem()
        price = response.css('div.property-price').css('span::text').get()
        item['price']=self.get_number(price)
        item['address'] = response.css('span.property-address sale-default::text').get()
        item['supplyImageUrls'] = response.css('header').xpath('..').css('img::attr(src)').get()
        item['luastanah'] = self.get_number(response.css('div.property-areas-info').css('li::text').getall()[1])
        item['luasbangunan'] = self.get_number(response.css('div.property-areas-info').css('li::text').getall()[0])
        item['deskripsi'] = response.css('pre.property-description::text').get().replace('\n',',')
        item['title'] = response.css('h2.description-title::text').get()
        item['url'] = response.url
        yield item