import os
import json
import uuid
import datetime
from flask import Flask, jsonify, Response, request, session,\
    redirect, url_for
from sqlalchemy import Column, String, DateTime
from sqlalchemy.dialects.postgresql import JSON
from jianyue.database import init_db, db_session
from flask_restful import Api
from jianyue.models import JianYue, JianYueSchema
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow

from requests_oauthlib import OAuth2Session

import config

app = Flask(__name__)
api = Api(app)
app.config.from_object('config')
app.config.from_pyfile('config.py')
db = SQLAlchemy(app)
ma = Marshmallow(app)

class JianYue(db.Model):
    """

    """
    __tablename__ = 'jianyue'

    gid = Column(String, primary_key=True)
    openid = Column(String, unique=True)
    auth_type = Column(String)
    createtime = Column(DateTime)
    modifytime = Column(DateTime)
    jianyue = Column(JSON)

    def __init__(self, gid=None, openid=None, auth_type=None, createtime=None,
                 modifytime=None,jianyue=None):
        self.gid = gid
        self.openid = openid
        self.auth_type = auth_type
        self.createtime = createtime
        self.modifytime = modifytime
        self.jianyue = jianyue

    def __repr__(self):
        return self.__tablename__

class JianYueSchema(ma.Schema):
    class Meta:
        fields = ('gid', 'openid', 'auth_type',
                   'jianyue')

jianyue_schema = JianYueSchema()
jianyues_schema = JianYueSchema(many=True)

@app.teardown_request
def shutdown_seesion(exception=None):
    db_session.remove()

@app.route('/github', methods=['GET'])
def github_auth_login():
    github = OAuth2Session(config.client_id)
    authorization_url, state = github.authorization_url(config.authorization_base_url)
    session['oauth_state'] = state
    return redirect(authorization_url)

@app.route('/callback', methods=['GET'])
def callback():
    github = OAuth2Session(config.client_id, state=session['oauth_state'])
    token = github.fetch_token(config.token_url, client_secret=config.client_secret, authorization_response=request.url)
    session['oauth_token'] = token
    jy = JianYue.query.filter(JianYue.openid == token['access_token']).first()
    if jy == None:
        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
        gid = uuid.uuid3(uuid.NAMESPACE_DNS, now)
        jianyue = JianYue(gid=str(gid), openid=token['access_token'], createtime=now, modifytime=now)
        db.session.add(jianyue)
        db.session.commit()
    return redirect(url_for('.profile'))

@app.route("/profile", methods=["GET"])
def profile():
    """Fetching a protected resource using an OAuth 2 token.
    """
    github = OAuth2Session(config.client_id, token=session['oauth_token'])
    return jsonify(github.get('https://api.github.com/user').json())

@app.route('/users/<int:page>')
def get_all_users(page):
    jianyues = JianYue.query.limit(page)
    result = jianyues_schema.dump(jianyues)
    return Response(json.dumps(result.data), mimetype='Application/json')

@app.route('/user')
def get_user():
    gid = request.args.get('gid')
    auth_type = request.args.get('auth_type')
    jianyues = JianYue.query.filter(JianYue.gid == gid and JianYue.auth_type == auth_type)
    result = jianyues_schema.dump(jianyues)
    return Response(json.dumps(result.data), mimetype='Application/json')

@app.route('/jianyue/api/v0.1', methods=['post'])
def jianyue_api():
    jianyue_data = request.get_json(force=True)
    gid = jianyue_data['gid']
    openid = jianyue_data['openid']
    auth_type = jianyue_data['auth_type']
    createtime = jianyue_data['createtime']
    modifytime = jianyue_data['modifytime']
    jianyue = jianyue_data['jianyue']
    jianyue_model = JianYue(gid=gid, openid=openid, auth_type=auth_type, createtime=createtime,
                            modifytime=modifytime, jianyue=jianyue)
    db.session.add(jianyue_model)
    db.session.commit()
    return jsonify({'status': 'ok'})

@app.route('/jianyue/api/v0.1/<gid>', methods=['PUT'])
def update_jianyue(gid):
    jianyue = JianYue.query.filter(JianYue.gid == gid).first()
    json_data = request.get_json(force=True)
    jianyue.createtime = json_data['createtime']
    jianyue.modifytime = json_data['modifytime']
    jianyue.jianyue = json_data['jianyue']
    jianyue.auth_type = json_data['auth_type']
    db.session.commit()
    return jsonify({'status': 'ok'})

@app.route('/jianyue/api/v0.1/<gid>', methods=['DELETE'])
def delete_jianyue(gid):
    JianYue.query.filter(JianYue.gid == gid).delete()
    db.session.commit()
    return jsonify({'status': 'ok'})

if __name__ == '__main__':
    #init_db()
    #os.environ['DEBUG'] = "1"
    os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
    app.secret_key = os.urandom(24)
    app.run(host='127.0.0.1', port=5000, debug=True)


import spectranspo as stp
import pandas as pd
import numpy as np
from .tools import io

class Demand():
    '''
    Collection of profiles representing distributions between zones in an area

    Attributes
    ----------
    zones (array-like):
        Array of zone numbers
    profiles (pandas.DataFrame):
        Data frame of profiles where the index represents the origins and the columns represent the destinations
    K (int):
        Maximum number of circular moments stored
    '''
    def __init__(self, zones, profiles = None, K = None):

        self.zones = zones
        if type(profiles) == pd.DataFrame:
            self.profiles = profiles
        else:
            self.profiles = pd.DataFrame(index = zones, columns = zones)

        if K:
            self.K = K
        else:
            self.K = 0

    def __getitem__(self, od_pair):
        return self.profiles.loc[od_pair[0], od_pair[1]]

    def __setitem__(self, od_pair, TimeDist):
        assert isinstance(TimeDist, stp.TimeDist), 'Profile must be a TimeDist'
        self.set_profile(TimeDist, od_pair[0], od_pair[1])

    def __repr__(self):
        return self.profiles.sum().sum().__repr__().replace('TimeDist', 'Demand')

    @classmethod
    def from_omx(cls, filepath):
        '''
        Reads in a Demand object from an open matrix file. Tables in the file must be `Total`, and `mn` representing the n-th circular moments

        Parameters
        ----------
        filepath (str):
            filepath of the omx file
        '''
        try:
            import openmatrix as omx
        except ImportError:
            raise ImportError('Open Matrix not installed')

        #Open file
        f = omx.openFile(filepath, 'r')
        K = len(f.listMatrices()) - 1
        zones = f.mapping('zones').keys()

        #Read file contents into pandas.Panel
        data = pd.Panel(items = list(range(K+1)), major_axis = zones, minor_axis = zones)
        K = 0
        for k in data.items:
            K = max(k, K)
            if k:
                table = 'm%d'%(k)
            else:
                table = 'Total'

            for o_zone in zones:
                if table:
                    data.ix[k, o_zone, :] = f[table][o_zone]
                else:
                    data.ix[k, o_zone, :] = f[table][o_zone]

        #Convert panel to Demand object
        profiles = pd.DataFrame(index = zones, columns = zones)
        for o_zone in zones:
            for d_zone in zones:
                moments = data.ix[:, o_zone, d_zone]
                total = np.real(moments[0])
                moments[0] /= total
                profiles[d_zone][o_zone] = stp.TimeDist(np.array(moments), total, (0, 24))

        return cls(zones, profiles, K)

    @classmethod
    def from_survey(cls, n_moments, survey, event_column, origin_column, destination_column, weight_column = None):
        '''
        Creates a demand profile from a travel survey

        Parameters
        ----------
        survey: Data Frame
            The travel survey
        event_column:
            The column containing the events to aggregate
        origin_column:
            The column containing the origin of the trips
        destination_column:
            The column containing the destination of the trips
        weight_column:
            The column containing the column with the weights
        '''
        origins = survey[origin_column].value_counts().index.tolist()
        destinations = survey[destination_column].value_counts().index.tolist()
        zones = list(set(origins + destinations))

        profiles = pd.DataFrame(index = zones, columns = zones)

        for origin in origins:
            for destination in destinations:
                od = survey.query(origin_column + ' == @origin and ' + destination_column + ' == @destination')
                if len(od.index) == 0:
                    od_dist = stp.TimeDist(np.append(np.array(1), np.zeros(n_moments)), 0, (0, 24))

                else:
                    if weight_column:
                        od_dist = stp.TimeDist.from_events(od[event_column], n_moments, od[weight_column])
                    else:
                        od_dist = stp.TimeDist.from_events(od[event_column], n_moments)
                profiles[destination][origin] = od_dist

        return cls(zones, profiles, n_moments)

    def set_profile(self, TimeDist, o, d):
        '''
        Sets a profile for an origin and destination

        Parameters
        ----------
        TimeDist (spectranspo.TimeDist):
            TimeDist for the origin-destination pair
        o (int):
            Origin zone
        d (int):
            Destination zone
        '''
        self.profiles[d][o] = TimeDist
        self.K = max(self.K, len(TimeDist.moments) - 1)

    def get_trip_table(self, time_range = None):
        trip_table = pd.DataFrame(index = self.zones, columns = self.zones)
        if not time_range:
            for d_zone in self.zones:
                for o_zone in self.zones:
                    trip_table[d_zone][o_zone] = self.profiles[d_zone][o_zone].total
        else:
            for d_zone in self.zones:
                for o_zone in self.zones:
                    trip_table[d_zone][o_zone] = self.profiles[d_zone][o_zone].count_events(time_range[0], time_range[1])
        return trip_table

    def aggregate(self, aggregation_map):
        '''
        Aggregation from zones to larger districts

        Parameters
        ----------
        aggregation_map (dict):
            Dictionary mapping self's zones to aggregated zones

        Returns
        -------
        new_demand (spectranspo.Demand):
            Demand object based on aggregated zones
        '''
        #Create dictionary mapping larger zone to all of its subzones
        subzone_map = {}
        for zone in aggregation_map.keys():
            if aggregation_map[zone] in subzone_map:
                subzone_map[aggregation_map[zone]] += [zone]
            else:
                subzone_map[aggregation_map[zone]] = [zone]

        #Create new Demand
        new_zones = list(set(aggregation_map.values()))
        new_zones.sort()

        profiles = pd.DataFrame(index = new_zones, columns = new_zones)
        for ozone in new_zones:
            for dzone in new_zones:
                od_profiles = []
                for old_ozone in subzone_map[ozone]:
                    for old_dzone in subzone_map[dzone]:
                        try:
                            od_profiles += [self.profiles.loc[old_ozone, old_dzone]]
                        except KeyError:
                            continue

                profiles.loc[ozone, dzone] = stp.sum(od_profiles)

        return Demand(new_zones, profiles, self.K)

    def get_snapshots(self, N):
        '''
        Computes instantaneous values of the number of trips per time unit at N equally-spaced points throughout the day

        Parameters
        ----------
        N (int):
            Number of equally-spaced points to compute trip rate within day

        Returns
        -------
        assignment_inputs (pandas.Panel):
            A panel object where the items are the point numbers (starting at zero), the major axis is the origin zones, and the minor axis is the destination zones
        '''
        snapshots = pd.Panel(items = list(range(N)), major_axis = self.zones, minor_axis = self.zones)
        for ozone in self.zones:
            for dzone in self.zones:
                profile = self.profiles.loc[ozone, dzone]
                snapshots.ix[:, ozone, dzone] = profile.total * profile.get_snapshots(N)
        return snapshots

    def get_trip_table(self, breaks, time_range = (0, 24)):
        '''
        Creates a trip table based on the specified time periods

        Parameters
        ----------
        breaks (sequence):
            Starting point of each time period
        time_range (tuple):
            Start and end points of the day in the same units as the breaks

        Returns
        -------
        trip_table (pandas.Panel):
            Panel in which the items are the time periods, the major axis is the origin zones, and the minor axis is the destination zones
        '''
        N = len(breaks)
        items = []
        for i in range(N-1):
            if breaks[i] > breaks[i+1]:
                raise ValueError('Breaks must be sorted in ascending order')
            if breaks[i] < time_range[0] or breaks[i] > time_range[1]:
                raise ValueError('Breaks must be within time range')
            items += ['t{0}-{1}'.format(breaks[i], breaks[i+1])]
        items += ['t{0}-{1}'.format(breaks[-1], breaks[0] + time_range[1])]

        trip_table = pd.Panel(items = items, major_axis = self.zones, minor_axis = self.zones)
        for ozone in self.zones:
            for dzone in self.zones:
                profile = self.profiles.loc[ozone, dzone]
                profile.time_range = time_range
                for i in range(N-1):
                    trip_table.ix[items[i], ozone, dzone] = profile.count_events(breaks[i], breaks[i+1])
                trip_table.ix[items[-1], ozone, dzone] = profile.count_events(breaks[-1], breaks[0] + time_range[1])

        return trip_table

    def to_panel(self):
        '''
        Writes the Demand as a panel for exporting

        Returns
        -------
        out_data(pandas.Panel):
            Panel in which the first table is the total, and the rest are the circular moments
        '''
        out_data = pd.Panel(np.zeros((self.K+1, len(self.zones), len(self.zones))), major_axis = self.zones, minor_axis = self.zones)
        for d_zone in self.zones:
            for o_zone in self.zones:
                profile = self.profiles.loc[o_zone, d_zone]
                out_data.ix[:, o_zone, d_zone] = profile.moments
                out_data.ix[0, o_zone, d_zone] *= profile.total
        out_data[0] = np.real(out_data[0])

        return out_data

    def to_omx(self, filepath):
        '''
        Writes a Demand file as an open matrix file

        Parameters
        ----------
        filepath (str):
            Filepath to write to
        '''
        try:
            import openmatrix as omx
        except ImportError:
            print('Open Matrix not installed. File not written.')

        #Make sure filepath is an omx file, and open file for writing
        assert filepath[-4:] == '.omx', 'Demand must be written in open matrix format'

        #Convert data to panel with the total as well as circular moments
        out_data = self.to_panel()

        #Write panel to file
        f = omx.openFile(filepath, 'w')
        f['Total'] = out_data[0].as_matrix()
        for k in range(1, self.K+1):
            f['m%d'%k] = out_data[k].as_matrix()

        f.createMapping('zones', self.zones)
        f.close()

    def to_excel(self, filepath):
        '''
        Writes a Demand file as an Excel file.

        Parameters
        ----------
        filepath (str):
            Filepath to write to
        '''
        assert filepath[-4:] == '.xls' or filepath[-5:] == '.xlsx', 'Filepath must be an Excel format'

        items = ['Total']
        for i in range(1, self.K):
            items += ['%d Real'%(i), '%d Imaginary'%(i)]

        out_data = self.to_panel()
        for item in out_data.items:
            for element in out_data.major_axis:
                out_data.ix[item, element, :] = out_data[item, element, :].apply(io.remove_parentheses)
        out_data.to_excel(filepath)


import tweepy
from Utils import Utils
from GetTUserDetails import  user
import time
import  multiprocessing as mp

#Intialize twwepy API Connection and Utility Object
util=Utils()
configs=util.ReadConfig()
apiDict={}

cnt=0
for config in configs:
    auth = tweepy.OAuthHandler(config['CONSUMER_KEY'],config['CONSUMER_SECRET'])
    auth.set_access_token(config['ACCESS_TOKEN'],config['ACCESS_SECRET'])
    api=Utils.InitiateAPi(auth)
    apiDict[cnt]=api
    cnt+=1

#Load Seed List
seedList=[]
seed=open('seedIdstest.txt','r')
for u in seed.readlines():
    seedList.append(u.replace('\n',''))

cnt2=0
for u in seedList:
    api=cnt2%5
    cnt2+=1

    try:
      u=user(u,apiDict[api])
      print(str(u) +'processed')
      #print(u  ::' +str(u.auth.consumer_key))

    except:
      Utils.WriteLog(" excpetion occured while extracting details user "+ str(u.username))

    #Get Data for User u
    try:
       start=time.time()
       #extract followers
       u.GetFollowers()
       telapsed=time.time()-start
       #print(" time to extract "+str(u.numFollowers)+ " followers of user "+ str(u.username) + " is " + str(telapsed))
       Utils.WriteLog(" time to extract "+str(u.numFollowers)+ " followers of user "+ str(u.username) + " is " + str(telapsed))
    except:
       Utils.WriteLog(" excpetion occured while extracting followers of user "+ str(u.username))

    try:
       #extract followee
       start=time.time()
       u.GetFollowing(u)
       telapsed=time.time()-start
       #print(" time to extract "+str(u.numFollowed)+ " people followed  by user "+ str(u.username) + " is " + str(telapsed))
       Utils.WriteLog(" time to extract "+str(u.numFollowed)+ " people followed  by user "+ str(u.username) + " is " + str(telapsed))
    except:
       Utils.WriteLog(" excpetion occured while extracting people  followed by user "+ str(u.username))

    try:
        start=time.time()
        #extract followers
        u.GetTweetsasJson()
        telapsed=time.time()-start
        Utils.WriteLog(" time to extract "+str(u.numFTweets)+ " tweets posted  by user "+ str(u.username) + " is " + str(telapsed))
    except:
       print(" time to extract "+str(u.numFTweets)+ " tweets posted  by user "+ str(u.username) + " is " + str(telapsed))
       Utils.WriteLog(" excpetion occured while extracting tweets of user  "+ str(u.username))


#!/usr/bin/env python
from collections import defaultdict

d1 = dict()
d2 = {'a': 5, 'm': 16, 'e': 42}
d3 = {}
d4 = dict(a=5, m=16, e=42)

keys = ['a', 'm', 'e']
values = [5, 16, 42]

d5 = dict(zip(keys, values))
print(d5)

airports = {
    'EWR': 'Newark',
    'SFO': 'San Francisco',
    'RDU': 'Raleigh-Durham',
    'SJC': 'San Jose',
    'ABQ': 'Albuquerque',
    'OAK': 'Oakland',
    'SAC': 'Sacramento',
    'MLB': 'Melbourne',
    'IAD': 'Dulles',
}

print(airports['MLB'], '\n')

print(airports.get('MLB'))
print(airports.get('MIA'))
print(airports.get('MIA', "NOT FOUND"))
print()

colors = defaultdict(list)

colors['red'] = [25, 15]

print(colors['blue'])

colors['blue'].append(5)
colors['red'].append(20)
colors['blue'].append(35)
print(colors)

things = defaultdict(lambda: 0)

print(things['spam'])
print(things['ham'])

print(airports.setdefault('MIA', 'Miami'))

print(airports, '\n')

more_airports = {'JFK': "Kennedy", "ELM": "Elmira, NY", "HLN": "Helena, MT",
            'MIA': "Miami International"}

airports.update(more_airports)

print(airports, '\n')

for abbr in 'JFK', 'MLB', 'MCO', 'TPA', 'EWR':
    print(abbr, abbr in airports)
print()

for abbr, name in airports.items():
    print(abbr, name)
print()

for abbr, name in sorted(airports.items()):
    print(abbr, name)
print()

for abbr, name in sorted(airports.items(), key=lambda e: (e[1], e[0])):
    print(abbr, name)
print()


"""
Modified from https://github.com/mseitzer/pytorch-fid

Calculates the Frechet Inception Distance (FID) to evalulate GANs
The FID metric calculates the distance between two distributions of images.
Typically, we have summary statistics (mean & covariance matrix) of one
of these distributions, while the 2nd distribution is given by a GAN.
When run as a stand-alone program, it compares the distribution of
images that are stored as PNG/JPEG at a specified location with a
distribution given by summary statistics (in pickle format).
The FID is calculated by assuming that X_1 and X_2 are the activations of
the pool_3 layer of the inception net for generated samples and real world
samples respectively.
See --help to see further details.
Code apapted from https://github.com/bioinf-jku/TTUR to use PyTorch instead
of Tensorflow
Copyright 2018 Institute of Bioinformatics, JKU Linz
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
   http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import cv2
import glob
import pathlib
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser

import numpy as np
import pandas as pd
import nibabel as nib
import torch
import torchvision.transforms as TF
from PIL import Image
from scipy import linalg
from torch.nn.functional import adaptive_avg_pool2d

try:
    from tqdm import tqdm
except ImportError:
    # If tqdm is not available, provide a mock version of it
    def tqdm(x):
        return x

from inception import InceptionV3

wd = os.path.dirname(os.path.realpath(__file__))
import sys
sys.path.append(os.path.join(wd, '..'))
from models.classifier import Classifier

parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--batch-size', type=int, default=16,
                    help='Batch size to use')
parser.add_argument('--gpu', type=int, default=0,
                    help='Device to use. Like cuda, cuda:0 or cpu')
parser.add_argument('--dims', type=int, default=2048,
                    choices=list(InceptionV3.BLOCK_INDEX_BY_DIM),
                    help=('Dimensionality of Inception features to use. '
                          'By default, uses pool3 features'))
parser.add_argument('experiment', type=str, help='experiment to analyse.')

IMAGE_EXTENSIONS = {'bmp', 'jpg', 'jpeg', 'pgm', 'png', 'ppm',
                    'tif', 'tiff', 'webp'}

class ImagePathDataset(torch.utils.data.Dataset):
    def __init__(self, files, transforms=None):
        self.files = files
        self.transforms = transforms
        self.image_loader = None

    def __len__(self):
        return len(self.files)

    def __getitem__(self, i):
        path = self.files[i]
        img = Image.open(path).convert('RGB')
        if self.transforms is not None:
            img = self.transforms(img)
        return img

class ImagePathDatasetv2(torch.utils.data.Dataset):
    'Generalization of ImagePathDataset class with generic data loader.'
    def __init__(self, files, transforms=None, format='png'):
        self.files = files
        self.transforms = transforms
        self.image_loader = None
        if format == 'png':
            self.image_loader = lambda x: Image.open(x).convert('RGB')
        elif format == 'nifti':
            def load_nifti(x):
                aux = nib.load(x).get_fdata()[...,0,0]
                # Rescale to 0,1
                aux = (aux - aux.min())/(aux.max() - aux.min())
                aux = cv2.resize(aux, (256,256))
                return aux
            self.image_loader = lambda x: load_nifti(x)
        elif format == 'numpy':
            def load_numpy(x):
                aux = np.load(x)
                return aux
            self.image_loader = lambda x: load_numpy(x)
        else:
            raise ValueError('Invalid format for image', format)

    def __len__(self):
        return len(self.files)

    def __getitem__(self, i):
        path = self.files[i]
        img = self.image_loader(path)
        if self.transforms is not None:
            img = self.transforms(img)
        return img

def get_activations(files, model, batch_size=50, dims=2048, device='cpu',
                    num_workers=1):
    """Calculates the activations of the pool_3 layer for all images.
    Params:
    -- files       : List of image files paths
    -- model       : Instance of inception model
    -- batch_size  : Batch size of images for the model to process at once.
                     Make sure that the number of samples is a multiple of
                     the batch size, otherwise some samples are ignored. This
                     behavior is retained to match the original FID score
                     implementation.
    -- dims        : Dimensionality of features returned by Inception
    -- device      : Device to run calculations
    -- num_workers : Number of parallel dataloader workers
    Returns:
    -- A numpy array of dimension (num images, dims) that contains the
       activations of the given tensor when feeding inception with the
       query tensor.
    """
    model.eval()

    if batch_size > len(files):
        print(('Warning: batch size is bigger than the data size. '
               'Setting batch size to data size'))
        batch_size = len(files)

    # File format
    if files[0].endswith('.npz') or files[0].endswith('.npy'):
        format = 'numpy'
    elif files[0].endswith('.nii.gz'):
        format = 'nifti'
    else:
        format = 'png'
    dataset = ImagePathDatasetv2(files, transforms=TF.ToTensor(), format=format)
    dataloader = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size, shuffle=False,
        drop_last=False, num_workers=num_workers)

    pred_arr = np.empty((len(files), dims))

    start_idx = 0

    for batch in tqdm(dataloader):
        batch = batch.float()
        batch = batch.to(device)

        with torch.no_grad():
            # Get the logits before the FC layer
            pred = model(batch)[1]

        # If model output is not scalar, apply global spatial average pooling.
        # This happens if you choose a dimensionality not equal 2048.
        if pred.size(2) != 1 or pred.size(3) != 1:
            pred = adaptive_avg_pool2d(pred, output_size=(1, 1))

        pred = pred.squeeze(3).squeeze(2).cpu().numpy()

        pred_arr[start_idx:start_idx + pred.shape[0]] = pred

        start_idx = start_idx + pred.shape[0]

    return pred_arr

def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):
    """Numpy implementation of the Frechet Distance.
    The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
    and X_2 ~ N(mu_2, C_2) is
            d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).
    Stable version by Dougal J. Sutherland.
    Params:
    -- mu1   : Numpy array containing the activations of a layer of the
               inception net (like returned by the function 'get_predictions')
               for generated samples.
    -- mu2   : The sample mean over activations, precalculated on an
               representative data set.
    -- sigma1: The covariance matrix over activations for generated samples.
    -- sigma2: The covariance matrix over activations, precalculated on an
               representative data set.
    Returns:
    --   : The Frechet Distance.
    """

    mu1 = np.atleast_1d(mu1)
    mu2 = np.atleast_1d(mu2)

    sigma1 = np.atleast_2d(sigma1)
    sigma2 = np.atleast_2d(sigma2)

    assert mu1.shape == mu2.shape, \
        'Training and test mean vectors have different lengths'
    assert sigma1.shape == sigma2.shape, \
        'Training and test covariances have different dimensions'

    diff = mu1 - mu2

    # Product might be almost singular
    covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
    if not np.isfinite(covmean).all():
        msg = ('fid calculation produces singular product; '
               'adding %s to diagonal of cov estimates') % eps
        print(msg)
        offset = np.eye(sigma1.shape[0]) * eps
        covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))

    # Numerical error might give slight imaginary component
    if np.iscomplexobj(covmean):
        if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):
            m = np.max(np.abs(covmean.imag))
            raise ValueError('Imaginary component {}'.format(m))
        covmean = covmean.real

    tr_covmean = np.trace(covmean)

    return (diff.dot(diff) + np.trace(sigma1)
            + np.trace(sigma2) - 2 * tr_covmean)

def calculate_activation_statistics(files, model, batch_size=50, dims=2048,
                                    device='cpu', num_workers=1):
    """Calculation of the statistics used by the FID.
    Params:
    -- files       : List of image files paths
    -- model       : Instance of inception model
    -- batch_size  : The images numpy array is split into batches with
                     batch size batch_size. A reasonable batch size
                     depends on the hardware.
    -- dims        : Dimensionality of features returned by Inception
    -- device      : Device to run calculations
    -- num_workers : Number of parallel dataloader workers
    Returns:
    -- mu    : The mean over samples of the activations of the pool_3 layer of
               the inception model.
    -- sigma : The covariance matrix of the activations of the pool_3 layer of
               the inception model.
    """
    act = get_activations(files, model, batch_size, dims, device, num_workers)
    mu = np.mean(act, axis=0)
    sigma = np.cov(act, rowvar=False)
    return mu, sigma

def compute_statistics_of_filelist(
    filelist, model, batch_size, dims, device, num_workers=1):

    if len(filelist) == 0:
        if filelist.endswith('.npz'):
            with np.load(filelist) as f:
                m, s = f['mu'][:], f['sigma'][:]
        else:
            raise ValueError('Invalid file list', filelist)
    else:
        m, s = calculate_activation_statistics(filelist, model, batch_size,
                                               dims, device, num_workers)

    return m, s

def compute_statistics_of_path(path, model, batch_size, dims, device,
                               num_workers=1):
    if path.endswith('.npz'):
        with np.load(path) as f:
            m, s = f['mu'][:], f['sigma'][:]
    else:
        path = pathlib.Path(path)
        files = sorted([file for ext in IMAGE_EXTENSIONS
                       for file in path.glob('*.{}'.format(ext))])
        m, s = calculate_activation_statistics(files, model, batch_size,
                                               dims, device, num_workers)

    return m, s

def calculate_fid_given_filelists(lists_of_files, batch_size, device, dims, num_workers=1):
    """Calculates the FID of two paths"""
    block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[dims]

    model = InceptionV3([block_idx]).to(device)

    m1, s1 = compute_statistics_of_filelist(lists_of_files[0], model, batch_size,
                                            dims, device, num_workers)
    m2, s2 = compute_statistics_of_filelist(lists_of_files[1], model, batch_size,
                                            dims, device, num_workers)
    fid_value = calculate_frechet_distance(m1, s1, m2, s2)

    return fid_value

def calculate_fid_given_paths(paths, batch_size, device, dims, num_workers=1):
    """Calculates the FID of two paths"""
    for p in paths:
        if not os.path.exists(p):
            raise RuntimeError('Invalid path: %s' % p)

    block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[dims]

    model = InceptionV3([block_idx]).to(device)

    m1, s1 = compute_statistics_of_path(paths[0], model, batch_size,
                                        dims, device, num_workers)
    m2, s2 = compute_statistics_of_path(paths[1], model, batch_size,
                                        dims, device, num_workers)
    fid_value = calculate_frechet_distance(m1, s1, m2, s2)

    return fid_value

def compute_fid_from_lists(list_of_files, batch_size, dims=2048):
    device = torch.device('cuda' if (torch.cuda.is_available()) else 'cpu')

    num_avail_cpus = len(os.sched_getaffinity(0))
    num_workers = min(num_avail_cpus, 8)

    fid_value = calculate_fid_given_filelists(list_of_files,
                                              batch_size,
                                              device,
                                              dims,
                                              num_workers)

    print('FID:', fid_value)
    return fid_value

def compute_fid_for_experiment(name_of_experiment, batch_size, dims=2048, gpu=0):
    device = torch.device('cuda:{}'.format(gpu) if (torch.cuda.is_available()) else 'cpu')

    num_avail_cpus = len(os.sched_getaffinity(0))
    num_workers = min(num_avail_cpus, 4)

    # FID from ImageNet vs. FID from our dataset is quite different.
    # block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[dims]
    # model = InceptionV3([block_idx]).to(device)
    ckpt_model = 'repo_directory/checkpoints/sex-class-2d-inception3-v1/model-epoch=099-train_loss=0.00.ckpt'
    hpms_file = 'repo_directory/logs/sex-class-2d-inception3-v1/version_0/hparams.yaml'
    model = Classifier.load_from_checkpoint(
        checkpoint_path=ckpt_model,
        hparams_file=hpms_file,
        map_location=None).to(device)

    _files = [
        'path_to_list_of_testing_ids.csv'
    ]
    _ids = []
    for f in _files:
        _ids.extend(pd.read_csv(f).ID.values.tolist())
    _ids = np.array(_ids)
    _path = 'repo_directory/results/{}/numpy/*_image_step_{}.npy'

    list1 = list(sorted(glob.iglob(_path.format(name_of_experiment, 0))))
    aux = [int(l.split('_')[1]) for l in list1]
    mk = pd.Series(aux).isin(_ids)
    list1 = np.array(list1)[mk].tolist()
    print('list1', len(list1), list1[:3])
    m1, s1 = calculate_activation_statistics(
        list1, model, batch_size, dims, device, num_workers)

    fids = []
    for step in range(-29, 30):
        print('step', step)
        list2 = list(sorted(glob.iglob(_path.format(name_of_experiment, step))))
        aux = [int(l.split('_')[1]) for l in list2]
        mk = pd.Series(aux).isin(_ids)
        list2 = np.array(list2)[mk].tolist()
        print('list2', len(list2))
        m2, s2 = calculate_activation_statistics(
            list2, model, batch_size, dims, device, num_workers)
        fid_value = calculate_frechet_distance(m1, s1, m2, s2)
        fids.append([step, fid_value])

    pd.DataFrame(fids, columns=['step', 'fid']).to_csv(
        'repo_directory/results/{}/csv/fids.csv'.format(name_of_experiment),
        index=False)
    return fid_value

def main():
    args = parser.parse_args()

    if args.device is None:
        device = torch.device('cuda' if (torch.cuda.is_available()) else 'cpu')
    else:
        device = torch.device(args.device)

    if args.num_workers is None:
        num_avail_cpus = len(os.sched_getaffinity(0))
        num_workers = min(num_avail_cpus, 8)
    else:
        num_workers = args.num_workers

    fid_value = calculate_fid_given_paths(args.path,
                                          args.batch_size,
                                          device,
                                          args.dims,
                                          num_workers)
    print('FID:', fid_value)

if __name__ == '__main__':
    args = parser.parse_args()
    compute_fid_for_experiment(args.experiment, args.batch_size, args.dims, args.gpu)


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import numpy as np
from matplotlib import pyplot as plt

from astropy.stats import mad_std
from astropy.io import fits
from photutils import CircularAperture
from astroscrappy import detect_cosmics

from astropy.convolution import convolve_fft, Tophat2DKernel, Gaussian2DKernel
from pyfftw.interfaces.scipy_fftpack import fft2, ifft2

__all__ = ['init_centroids']

def init_centroids(first_image_path, master_flat, master_dark, target_centroid,
                   max_number_stars=10, min_flux=0.2, plots=False):

    first_image = (fits.getdata(first_image_path) - master_dark)/master_flat

    kernel = Gaussian2DKernel(5, 5)
    convolution = convolve_fft(first_image, kernel, fftn=fft2, ifftn=ifft2)

    convolution -= np.median(convolution)

    mad = mad_std(convolution)

    convolution[convolution < -5*mad] = 0.0

    from skimage.filters import threshold_otsu, threshold_yen
    from skimage.measure import label, regionprops

    thresh = threshold_yen(convolution)/4
    # thresh = threshold_otsu(image=convolution) # Use 10 for 20171104, 15 for all other nights

    masked = np.ones_like(convolution)
    masked[convolution <= thresh] = 0

    label_image = label(masked)

    plt.figure()
    plt.imshow(label_image, origin='lower', cmap=plt.cm.viridis)
    plt.show()

    regions = regionprops(label_image, convolution)

    # reject regions near to edge of detector
    buffer_pixels = 5
    regions = [region for region in regions
               if ((region.weighted_centroid[0] > buffer_pixels and
                   region.weighted_centroid[0] < label_image.shape[0] - buffer_pixels)
               and (region.weighted_centroid[1] > buffer_pixels and
                    region.weighted_centroid[1] < label_image.shape[1] - buffer_pixels))]

    # TYC 3561-1538-1 is a delta Scuti variable. Remove it:
    # variable_star = [1790.1645248,  1153.91737674]
    # tol = 100
    # regions = [region for region in regions
    #            if ((region.weighted_centroid[0] > variable_star[0] + tol) or
    #               (region.weighted_centroid[0] < variable_star[0] - tol)) and
    #               ((region.weighted_centroid[1] > variable_star[1] + tol) or
    #               (region.weighted_centroid[1] < variable_star[1] - tol))]

    centroids = np.array([region.weighted_centroid for region in regions])
    intensities = np.array([region.mean_intensity for region in regions])

    sort_order = np.argsort(intensities)[::-1]
    centroids = np.array(centroids)[sort_order]
    intensities = intensities[sort_order]

    positions = np.vstack([[y for x, y in centroids], [x for x, y in centroids]])

    flux_threshold = intensities > min_flux * intensities[0]
    positions = positions[:, flux_threshold]

    if plots:
        apertures = CircularAperture(positions, r=12.)
        apertures.plot(color='r', lw=2, alpha=1)
        plt.imshow(first_image, vmin=np.percentile(first_image, 0.01),
                   vmax=np.percentile(first_image, 99.9), cmap=plt.cm.viridis,
                   origin='lower')
        plt.show()
    return positions

    # target_index = np.argmin(np.abs(target_centroid - positions), axis=1)[0]
    # flux_threshold = sources['flux'] > min_flux * sources['flux'].data[target_index]
    #
    # fluxes = sources['flux'][flux_threshold]
    # positions = positions[:, flux_threshold]
    #
    # brightest_positions = positions[:, np.argsort(fluxes)[::-1][:max_number_stars]]
    # target_index = np.argmin(np.abs(target_centroid - brightest_positions),
    #                          axis=1)[0]
    #
    # apertures = CircularAperture(positions, r=12.)
    # brightest_apertures = CircularAperture(brightest_positions, r=12.)
    # apertures.plot(color='b', lw=1, alpha=0.2)
    # brightest_apertures.plot(color='r', lw=2, alpha=0.8)
    #
    # if plots:
    #     plt.imshow(first_image, vmin=np.percentile(first_image, 0.01),
    #                vmax=np.percentile(first_image, 99.9), cmap=plt.cm.viridis,
    #                origin='lower')
    #     plt.plot(target_centroid[0, 0], target_centroid[1, 0], 's')
    #
    #     plt.show()
    #
    # # Reorder brightest positions array so that the target comes first
    # indices = list(range(brightest_positions.shape[1]))
    # indices.pop(target_index)
    # indices = [target_index] + indices
    # brightest_positions = brightest_positions[:, indices]
    #
    # return brightest_positions


#!/usr/bin/env python
""" Start the containers """

import argparse

from lib.docker_compose_tools import set_up

def main():
    """ Start the containers """

    parser = argparse.ArgumentParser(description='Set up testing environment.')
    parser.add_argument('version', help='PostgreSQL version')
    args = parser.parse_args()

    version = args.version
    print('Starting environment with PostgreSQL {version}'.format(version=version))
    set_up(version)

if __name__ == "__main__":
    main()


import paddle
import data
from paddle.io import BatchSampler, DataLoader
import os
import classify
import cfg
from paddle import nn
from paddle import optimizer
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import regularizer
from stats import *
from metrics import ClsMetric
from postprocess import ClsPostProcess
from utils import save_model, get_logger

def make_argparse():
    args = ArgumentParser("paddle train params")
    args.add_argument("--batch_size", type=int, default=256)
    args.add_argument("--log_smooth_window", type=int, default=20)
    args.add_argument("--resume", type=bool, default=False)
    args.add_argument("--layer", type=int, default=18)
    args.add_argument("--best_acc", type=float, default=0.8)
    args.add_argument("--save_model_path", type=str,
                      default="/data/yujiangyong/Projects/products/model_files/paddle_softmax3")
    args.add_argument("--save_epoch_step", type=int, default=10000)
    args.add_argument("--eval_epoch_step", type=int, default=10000)
    args.add_argument("--print_step", type=int, default=20)
    args.add_argument("--epoch_num", type=int, default=20)
    args.add_argument("--learning_rate", type=float, default=0.0001)
    return args.parse_args()

def build_dataloader(char_idx_dict, idx_char_dict, batch_size,
                     img_h, img_w, train_txt_path1, train_txt_path2, train_txt_path3):
    dataset = data.SimpleDataSet(char_idx_dict, idx_char_dict, batch_size,
                                 img_h, img_w, train_txt_path1, train_txt_path2, train_txt_path3)
    batch_sampler = BatchSampler(
        dataset=dataset,
        batch_size=batch_size,
        shuffle=False,
        drop_last=True)

    data_loader = DataLoader(
        dataset=dataset,
        batch_sampler=batch_sampler,
        places='gpu:0',
        num_workers=16,
        return_list=True,
        use_shared_memory=False)
    return data_loader

def build_eval_dataloader(char_idx_dict, idx_char_dict, batch_size,
                          img_h, img_w, eval_txt_path, eval_root_path):
    dataset = data.EvalDataSet(char_idx_dict, idx_char_dict, batch_size,
                               img_h, img_w, eval_txt_path, eval_root_path)
    batch_sampler = BatchSampler(
        dataset=dataset,
        batch_size=batch_size,
        shuffle=False,
        drop_last=False)

    data_loader = DataLoader(
        dataset=dataset,
        batch_sampler=batch_sampler,
        places='gpu:0',
        num_workers=1,
        return_list=True,
        use_shared_memory=True)
    return data_loader

class ClassifyLoss(nn.Layer):
    def __init__(self, **kwargs):
        super(ClassifyLoss, self).__init__()
        self.loss_func = nn.CrossEntropyLoss(reduction="mean")

    def forward(self, predicts, batch):
        label = batch["label"].astype("int64")
        loss = self.loss_func(predicts, label)
        return {"loss": loss}

class Adam(object):
    def __init__(self,
                 learning_rate=0.001,
                 beta1=0.9,
                 beta2=0.999,
                 epsilon=1e-08,
                 parameter_list=None,
                 weight_decay=None,
                 grad_clip=None,
                 name=None,
                 lazy_mode=False,
                 **kwargs):
        self.learning_rate = learning_rate
        self.beta1 = beta1
        self.beta2 = beta2
        self.epsilon = epsilon
        self.parameter_list = parameter_list
        self.learning_rate = learning_rate
        self.weight_decay = weight_decay
        self.grad_clip = grad_clip
        self.name = name
        self.lazy_mode = lazy_mode

    def __call__(self, parameters):
        opt = optimizer.Adam(
            learning_rate=self.learning_rate,
            beta1=self.beta1,
            beta2=self.beta2,
            epsilon=self.epsilon,
            weight_decay=self.weight_decay,
            grad_clip=self.grad_clip,
            name=self.name,
            lazy_mode=self.lazy_mode,
            parameters=parameters)
        return opt

def build_optimizer(learning_rate, parameters):
    reg = regularizer.L2Decay()()
    optim = Adam(learning_rate=learning_rate, weight_decay=reg)
    return optim(parameters)

def model_eval(model,
               valid_dataloader,
               post_process_class,
               eval_class):
    model.eval()
    with paddle.no_grad():
        max_iter = len(valid_dataloader)
        for idx, batch in enumerate(valid_dataloader):
            if idx >= max_iter:
                break
            images = batch["image"]
            preds = model(images)
            post_result = post_process_class(preds, batch["label"])
            eval_class(post_result, batch)
        metric = eval_class.get_metric()
    model.train()
    return metric

def main():
    flags = make_argparse()
    global_step = 0
    save_epoch_step = flags.save_epoch_step
    eval_epoch_step = flags.eval_epoch_step
    print_step = flags.print_step
    start_epoch = 0
    log_smooth_window = flags.log_smooth_window
    epoch_num = flags.epoch_num
    save_model_dir = flags.save_model_path
    if not os.path.exists(save_model_dir):
        os.makedirs(save_model_dir)
    log_file = '{}/train.log'.format(save_model_dir)
    logger = get_logger(name='root', log_file=log_file)
    if flags.layer == 50:
        model = classify.Classify(1, cfg.NUMBER_CLASSES, 2048, 50)
    else:
        model = classify.Classify(1, cfg.NUMBER_CLASSES, 512, 18)
    print(flags.best_acc)
    if flags.resume:
        best_model_dict = {"acc": flags.best_acc}
        old_model_path = os.path.join(flags.save_model_path, "best_accuracy.pdparams")
        para_dict = paddle.load(old_model_path)
        model.set_state_dict(para_dict)
    else:
        best_model_dict = {"acc": 0}
    model.train()

    loss_class = ClassifyLoss()
    postprocess_class = ClsPostProcess(label_list=cfg.idx_to_char)
    acc_class = ClsMetric(main_indicator="acc")

    optimizer_ocr = build_optimizer(flags.learning_rate, model.parameters())

    eval_data_loader = build_eval_dataloader(char_idx_dict=cfg.char_to_idx,
                                             idx_char_dict=cfg.idx_to_char,
                                             batch_size=flags.batch_size,
                                             img_h=cfg.img_h,
                                             img_w=cfg.img_w,
                                             eval_txt_path=cfg.eval_txt_path,
                                             eval_root_path=cfg.eval_data_dir)
    train_data_loader = build_dataloader(char_idx_dict=cfg.char_to_idx,
                                         idx_char_dict=cfg.idx_to_char,
                                         batch_size=flags.batch_size,
                                         img_h=cfg.img_h,
                                         img_w=cfg.img_w,
                                         train_txt_path1=cfg.train_txt_path1,
                                         train_txt_path2=cfg.train_txt_path2,
                                         train_txt_path3=cfg.train_npy_path3)

    for epoch in range(start_epoch, epoch_num + 1):
        max_iter = len(train_data_loader)
        train_stats = TrainingStats(log_smooth_window, ['lr'])
        for idx, batch in enumerate(train_data_loader):
            if idx > max_iter:
                break
            lr = optimizer_ocr.get_lr()
            preds = model(batch["image"])
            loss = loss_class(preds, batch)
            avg_loss = loss['loss']
            avg_loss.backward()
            optimizer_ocr.step()
            optimizer_ocr.clear_grad()

            stats = {k: v.numpy().mean() for k, v in loss.items()}
            stats['lr'] = lr

            train_stats.update(stats)
            post_result = postprocess_class(preds, batch["label"])

            acc_class(post_result)
            metric = acc_class.get_metric()
            train_stats.update(metric)
            logs = train_stats.log()
            if global_step > 0 and global_step % print_step == 0:
                strs = "epoch {}, iter {}, {}".format(epoch, idx, logs)
                logger.info(strs)
            if global_step > 0 and global_step % eval_epoch_step == 0:
                cur_metric = model_eval(model,
                                        eval_data_loader,
                                        postprocess_class,
                                        acc_class)
                cur_metric_str = 'cur metric, {}'.format(', '.join(
                    ['{}: {}'.format(k, v) for k, v in cur_metric.items()]))
                logger.info(cur_metric_str)
                if cur_metric["acc"] >= best_model_dict["acc"]:
                    best_model_dict.update(cur_metric)
                    best_model_dict['best_epoch'] = epoch
                    save_model(
                        model,
                        optimizer_ocr,
                        save_model_dir,
                        logger,
                        is_best=True,
                        prefix='best_accuracy',
                        best_model_dict=best_model_dict,
                        epoch=epoch,
                        global_step=global_step)
            if global_step > 0 and global_step % save_epoch_step == 0:
                save_model(
                    model,
                    optimizer_ocr,
                    save_model_dir,
                    logger,
                    is_best=False,
                    prefix='last',
                    epoch=epoch,
                    global_step=global_step)
            optimizer_ocr.clear_grad()
            global_step += 1

if __name__ == "__main__":
    main()


import pandas as pd
import numpy as np
from graphviz import Graph
from copy import deepcopy
from sklearn.model_selection import KFold

class CausalTree:
    """ The CausalTree class fits a Causal Tree to given data and computes (potentially)
     heterogeneous treatment effects for new data.
    """

    def __init__(self, parent=None):
        """
        :param parent: an object of class CausalTree
        """
        self._parent = parent
        self._left_child = None
        self._right_child = None
        self._value = None
        self._min_leaf = None
        self._max_distance = None
        self._crit_num_obs = None
        self._split_var = None
        self._split_value = None
        self._node_loss = None
        self._branch_loss = None
        self._is_fitted = False
        self._feature_names = None
        self._num_features = None
        self._y = None
        self._y_transformed = None
        self._treatment_status = None
        self._cv_error = None

    def __str__(self):
        """ method for representing CausalTree object when called by print / str method

        :return: readable string representation of CausalTree
        """
        if self._is_fitted is False:
            string_dict = {"Loss": self.branch_loss, "Estimate": self.value}
        elif self._feature_names is not None:
            string_dict = {
                "Loss": self.branch_loss,
                "Splitting Variable (First split)": self._feature_names[
                    self._split_var
                ],
                "Splitting Value": self._split_value,
                "Tree Depth": self.depth,
                "Estimate": self.value,
            }
        else:
            string_dict = {
                "Loss": self.branch_loss,
                "Splitting Feature (First split)": self._split_var,
                "Splitting Value": self._split_value,
                "Tree Depth": self.depth,
                "Estimate": self.value,
            }
        return dict.__str__(string_dict)

    def __repr__(self):
        """ method for representing CausalTree object when called by evaluating / repr

        :return: unambiguous string representation of CausalTree
        """
        return f"Causal Tree; fitted = {str(self.is_fitted)}; id = {id(self)}"

    #  Property and Setter Functions

    @property
    def y(self):
        """ndarray: 1D array containing outcomes."""
        return self._y

    @property
    def y_transformed(self):
        """ndarray: 1D array containing transformed outcomes"""
        return self._y_transformed

    @property
    def treatment_status(self):
        """ndarray: 1D array containing treatment status of observations"""
        return self._treatment_status

    @property
    def parent(self):
        """CausalTree: Parent node."""
        return self._parent

    @property
    def left_child(self):
        """CausalTree: Left child."""
        return self._left_child

    @property
    def right_child(self):
        """CausalTree: Right child."""
        return self._right_child

    @right_child.setter
    def right_child(self, node):
        self._right_child = node

    @left_child.setter
    def left_child(self, node):
        self._left_child = node

    @property
    def cv_error(self):
        """float: Cross-validation error of tree with this node as root."""
        return self._cv_error

    @property
    def feature_names(self):
        """ndarray: 1D array containing feature names"""
        return self._feature_names

    @feature_names.setter
    def feature_names(self, value):
        if self.feature_names is None:
            self._feature_names = value

    @property
    def depth(self):
        """int: Tree depth starting from this node."""
        return CausalTree.detect_tree_depth(self)

    @property
    def is_leaf(self):
        """bool: True if given node is a leaf and false otherwise."""
        if self._is_fitted:
            return self._left_child is None
        else:
            print(
                "The tree has not been fitted yet, "
                "hence it is root and leaf at the same time.\n"
            )
            return True

    @property
    def is_root(self):
        """bool: True if node is root of tree and false otherwise."""
        return self._parent is None

    @property
    def is_fitted(self):
        """bool: True if method fit was already called without errors, false else."""
        return self._is_fitted

    @property
    def number_of_leafs(self):
        """int: Number of leafs originating from this node."""
        return CausalTree.get_number_of_leafs(self)

    @property
    def value(self):
        """float: Estimated treatment effect using observations contained in this
         node."""
        return self._value

    @property
    def node_loss(self):
        """float: Estimated loss arising from estimating treatment effects with
        attribute value on observations contained in this node."""
        return self._node_loss

    @property
    def branch_loss(self):
        """float: Estimated loss arising from summing attribute node_loss of all
        leafs originating from this node."""
        self.update_branch_loss()
        return self._branch_loss

    @property
    def min_leaf(self):
        """int: Minimum number of observations required in each leaf."""
        return self._min_leaf

    @property
    def max_distance(self):
        """int: Maximum number of absolute difference in observations with treatment
        and observations without treatment in each leaf."""
        return self._max_distance

    @property
    def crit_num_obs(self):
        """int: Critical value of observations in each leaf: When reached assertion
        using attribute max_distance is binding."""
        return self._crit_num_obs

    # Various Auxiliary Functions

    def update_branch_loss(self):
        """Computes loss in leafs originating from self and updates attribute
        branch_loss with sum of losses.
        """
        leaf_list = CausalTree.get_leafs_in_list(self)
        loss_array = np.array([leaf.node_loss for leaf in leaf_list])
        self._branch_loss = np.sum(loss_array)

    def output_partition_estimates(self):
        """Prints treatment effect estimate of each partition (leaf) --Mostly useful for
        debugging."""
        leaf_list = CausalTree.get_leafs_in_list(self)
        for i, leaf in enumerate(leaf_list):
            print("Leaf {:d}; Estimate: {:3.03f}".format(i, leaf.value))

    def splitting_info_to_string(self):
        """Constructs string encoding information about splitting feature and splitting
        point of given node.

        :return: str: Splitting information of given node.
        """
        if self.left_child is None:
            return ""
        else:
            if self._feature_names is None:
                return "Variable %d <= %3.3f" % (self._split_var, self._split_value)
            else:
                return "%s <= %3.3f" % (
                    self._feature_names[self._split_var],
                    self._split_value,
                )

    # Static Methods (Some of which should probably not be static methods)

    @staticmethod
    def transform_outcome(y, treatment_status, p=None):
        """Transforms outcomes using propensity scores and treatment status, such that
        expected value of individual outcomes is equal to individual treatment effects.
        If p=None we implicitly set p=1/2 for all observations.

        :param y: ndarray: 1D array containing outcomes.
        :param treatment_status: ndarray: 1D array containing treatment status.
        :param p: ndarray: 1D array containing individual propensity scores.
        :return: ndarray: 1D array containing transformed outcomes.
        """
        if treatment_status.dtype != "int":
            treatment_status = np.array(treatment_status, dtype="int")
        if p is None:
            y_star = 2 * y * treatment_status - 2 * y * (1 - treatment_status)
        else:
            y_star = y * (treatment_status - p) / (p * (1 - p))
        return y_star

    @staticmethod
    def get_leafs_in_list(tree):
        """
        :param tree: CausalTree.
        :return: list: Leafs originating from tree.
        """
        leaf_list = []
        if tree.left_child is None:
            leaf_list.append(tree)
        else:
            leaf_list.extend(CausalTree.get_leafs_in_list(tree.left_child))
            leaf_list.extend(CausalTree.get_leafs_in_list(tree.right_child))
        return leaf_list

    @staticmethod
    def get_number_of_leafs(tree):
        """
        :param tree: CausalTree
        :return: int: Number of leafs originating from tree.
        """
        return len(CausalTree.get_leafs_in_list(tree))

    @staticmethod
    def detect_tree_depth(tree):
        """
        :param tree: CausalTree.
        :return: int: Maximum number of nodes to cross when traveling from tree to any
        leaf originating from tree, i.e. depth of CausalTree starting at node tree.
        """
        depth_left, depth_right = 0, 0
        if tree.left_child is not None:
            depth_left += 1 + CausalTree.detect_tree_depth(tree.left_child)
            depth_right += 1 + CausalTree.detect_tree_depth(tree.right_child)
        return max(depth_left, depth_right)

    @staticmethod
    def validate(tree, X_test, y_transformed_test, metric=None):
        """Evaluates tree model on testing data given some user specific metric. If
        metric is None we use standard l2 distance.

        :param tree: CausalTree.
        :param X_test: ndarray: 2D array containing testing data on features.
        :param y_transformed_test: ndarray: 1D array containing testing data on
        transformed outcomes.
        :param metric: function: User specific metric.
        :return: float: loss obtained from predicting treatment effects in testing data
        using tree.
        """
        #  returns assumed validation metric on predicted and true outcomes
        tau_pred = tree.predict(X_test)
        if metric is None:
            return np.mean((tau_pred - y_transformed_test) ** 2)
        else:
            return metric(tau_pred, y_transformed_test)

    @staticmethod
    def get_first_subtree(fitted_tree, thresh=None):
        """This function executes the first step of the pruning process, that is,
        computing the smallest subtree which has equivalent predictive power as
        fitted_tree. Starting from the leafs, in-sample loss of parent and leafs
        are compared, where leafs are cut-off if parent loss is smaller than sum of leaf
        loss minus thresh.

        :param fitted_tree: CausalTree object which already had method fit called.
        :param thresh: float: Penalizer on larger trees.
        :return: CausalTree: Subtree of fitted_tree with equivalent in-sample
        predictive power.
        """
        subtree = deepcopy(fitted_tree)
        if thresh is None:
            thresh = 0  # np.sqrt(np.var(subtree.y)) / 50
        depth = subtree.depth
        if depth < 1:
            return subtree
        for i in range(depth):
            for parent_node in subtree.get_level_in_list(depth - i - 1):
                if parent_node.left_child is not None:
                    if (
                        parent_node.node_loss
                        <= parent_node.left_child.branch_loss  # gets updated automatic
                        + parent_node.right_child.branch_loss
                        + thresh
                    ):
                        parent_node.left_child, parent_node.right_child = None, None
        return subtree

    @staticmethod
    def get_pruned_tree_and_alpha_sequence(fitted_tree, thresh):
        """Implementation of the pruning process. Sequence of increasing cost-complexity
        parameters and corresponding optimal subtrees of fitted_tree are computed.

        :param fitted_tree: CausalTree object which already had method fit called.
        :param thresh: float: Penalizer on larger trees.
        :return: dict: alphas: Increasing list of cost-complexity paramaters.
                        subtrees: List of subtrees corresponding to alphas.
        """
        assert isinstance(
            fitted_tree, CausalTree
        ), "This method only works on CausalTree Trees"
        if not fitted_tree.is_fitted:
            raise ValueError("This method only works on fitted trees")

        alphas = [0]
        subtrees = [
            CausalTree.get_first_subtree(fitted_tree, thresh)
        ]  # get_first_subtree() does deepcopy

        index = 0
        while subtrees[index].left_child is not None:
            tmp_argmin, tmp_min = g(subtrees[index])
            tmp_subtree = deepcopy(subtrees[index])
            alphas.append(tmp_min)
            for node in tmp_argmin:
                node.left_child = None
                node.right_child = None
            subtrees.insert(index, tmp_subtree)
            index += 1

        if not test_monotonicity_list(alphas) and not test_monotonicity_list(
            alphas, strictly=False
        ):
            raise RuntimeError("Sequence of alphas is not increasing")
        if not test_monotonicity_list(alphas):
            print("Sequence of alphas is only weakly increasing.")
        return {
            "alphas": alphas,
            "subtrees": subtrees,
        }  # i think i want: return alphas, subtrees

    @staticmethod
    def get_subtree_given_alpha(tree, alpha, thresh):
        """Given a tree and cost-complexity parameter alpha, this function computes
        the unique optimal subtree of tree corresponding to alpha.

        :param tree: CausalTree.
        :param alpha: float: Positive cost-complexity parameter.
        :param thresh: float: Positive penalizer on larger trees.
        :return: CausalTree: Subtree of tree corresponding to alpha.
        """
        sequences = CausalTree.get_pruned_tree_and_alpha_sequence(tree, thresh)
        alphas = sequences["alphas"]
        subtrees = sequences["subtrees"]
        alphas = np.array(alphas)
        if alpha >= alphas[-1]:
            return subtrees[-1]
        else:
            index = np.where(alphas > alpha)[0][0]
            return subtrees[index]

    @staticmethod
    def apply_kFold_CV(
        X_learn,
        y_learn,
        treatment_status_learn=None,
        k=5,
        thresh=0,
        sparsity_bias=0.5,
        fitted_tree=None,
    ):
        """This static method computes the optimal causal tree given training data.
        The tree will be optimal in the sense that it minimizes out-of-sample loss
        estimated by k-fold cross validation. Since we often prefer sparser models, the
        function returns the optimal tree as well as an optimal sparse tree, which is
        computed by the one standard error rule of thumb.

        :param X_learn: ndarray: 2D array of training data on features.
        :param y_learn: ndarray: 1D array of training data on outcomes.
        :param treatment_status_learn: ndarray: 1D array of training data on treatment,
        if not given it is assumed that treatment status is a feature in X_learn.
        :param k: int: Indicating the number of folds for CV.
        :param thresh: float: Penalizer for larger trees.
        :param sparsity_bias: float between 0 and 1: larger number leading to sparser
        tree
        :param fitted_tree: CausalTree: Tree fitted using X_learn, y_learn and
        treatment_status_learn. If equal to none, a new tree is fitted on given data.
        :return: tuple: optimal_subtree computed via k-fold CV and
        optimal_sparse_subtree which uses the one standard error rule of thumb of
        selecting a sparser model.
        """
        try:
            feature_names = X_learn.columns.values
        except AttributeError:
            feature_names = None

        if fitted_tree is None:
            fitted_tree = CausalTree()
            fitted_tree.fit(X_learn, y_learn, treatment_status_learn)
        assert len(y_learn) == len(X_learn), (
            "Argument <<X_learn>> and <<y_learn>> must have the same number "
            "of observations."
        )
        X_learn = coerce_to_ndarray(X_learn)
        y_learn = coerce_to_ndarray(y_learn)

        kf = KFold(k)
        tree_max = fitted_tree  # complete maximal tree
        tree_max_sequences = CausalTree.get_pruned_tree_and_alpha_sequence(
            tree_max, thresh
        )
        tree_k_max_list = []  # list of maximal trees in each cross validation sample
        test_X = []
        test_y = []
        test_treatment_status = []

        for train_index, test_index in kf.split(X_learn, y_learn):
            tmp_tree = CausalTree()
            if treatment_status_learn is None:
                train_treatment_status = None
            else:
                train_treatment_status = treatment_status_learn[train_index]
            tmp_tree.fit(
                X_learn[train_index], y_learn[train_index], train_treatment_status
            )
            tree_k_max_list.append(tmp_tree)
            test_X.append(X_learn[test_index])
            test_y.append(y_learn[test_index])
            test_treatment_status.append(treatment_status_learn[test_index])

        alphas = tree_max_sequences["alphas"]
        potential_subtrees = tree_max_sequences["subtrees"]

        alpha_cv_errors = []
        for alpha in alphas:
            err_alpha = 0
            for k, cv_tree in enumerate(tree_k_max_list):
                cv_alpha_subtree = CausalTree.get_subtree_given_alpha(
                    cv_tree, alpha, thresh
                )
                test_y_transformed = CausalTree.transform_outcome(
                    test_y[k], test_treatment_status[k]
                )
                err_alpha += CausalTree.validate(
                    cv_alpha_subtree, test_X[k], test_y_transformed
                )
            alpha_cv_errors.append(err_alpha)

        alpha_cv_errors = np.array(alpha_cv_errors)
        optimal_index = int(np.where(alpha_cv_errors == alpha_cv_errors.min())[0][-1])
        sparsity_adjustment = sparsity_bias * np.std(alpha_cv_errors)
        optimal_sparse_index = int(
            np.where(
                alpha_cv_errors <= alpha_cv_errors[optimal_index] + sparsity_adjustment
            )[0][-1]
        )
        optimal_subtree = potential_subtrees[optimal_index]
        optimal_subtree.feature_names = feature_names
        optimal_subtree._cv_error = alpha_cv_errors[optimal_index]
        optimal_sparse_subtree = potential_subtrees[optimal_sparse_index]
        optimal_sparse_subtree.feature_names = feature_names
        optimal_sparse_subtree._cv_error = alpha_cv_errors[optimal_sparse_index]
        return optimal_sparse_subtree, optimal_subtree

    # Algorithm Implementation and Fitting Functions

    @staticmethod
    def estimate_treatment_in_leaf(y, treatment_status):
        """Computes treatment effect for given data y via simple mean differences.

        :param y: ndarray: 1D array containing outcomes of observations
        :param treatment_status: ndarray: 1D array containing treatment_status
        :return: float: Treatment effect estimate for given array.
        """
        if treatment_status.dtype != "bool":
            treatment_status = np.array(treatment_status, dtype="bool")
        y_treat = y[treatment_status]
        y_untreat = y[~treatment_status]
        assert len(y_treat) > 0 and len(y_untreat) > 0, "Error: Empty Outcome Vector"
        return y_treat.mean() - y_untreat.mean()

    def is_valid_split(self, sorted_treatment, index):
        """Checks if a split of sorted_treatment at index index would violate that each
        new region must not contain too little treated or untreated observations.

        :param sorted_treatment: ndarray: Treatment status sorted beforehand.
        :param index: int: Splitting index.
        :return: bool: True if split is valid, False else.
        """
        sorted_treat_left = sorted_treatment[: (index + 1)]
        sorted_treat_right = sorted_treatment[(index + 1) :]
        if np.sum(sorted_treat_right) == 0:
            return False
        if np.sum(sorted_treat_left) == 0:
            return False
        if np.sum(1 - sorted_treat_left) == 0:
            return False
        if np.sum(1 - sorted_treat_right) == 0:
            return False
        valid_left = (
            len(sorted_treat_left) > self.crit_num_obs
            or np.abs(np.sum(sorted_treat_left) - np.sum(1 - sorted_treat_left))
            <= self.max_distance
        )
        valid_right = (
            len(sorted_treat_right) > self.crit_num_obs
            or np.abs(np.sum(sorted_treat_right) - np.sum(1 - sorted_treat_right))
            <= self.max_distance
        )
        return valid_left and valid_right

    def find_best_splitting_point(self, X):
        """Standard implementation of the recursive binary splitting algorithm.

        :param X: ndarray: 2D array containing data on features used to find splits.
        :return: tuple: Feature at which to split the data (split_index), data point of
        given feature at which to split (split_value) and resulting loss of split.
        """
        n, p = X.shape
        split_index = None
        split_value = None
        loss = float("Inf")

        for var_index in range(p):
            # loop through covariates
            x = X[:, var_index]
            sort_index = np.argsort(x)
            sorted_x, sorted_y, sorted_treatment = (
                x[sort_index],
                self.y[sort_index],
                self.treatment_status[sort_index],
            )

            for i in range(self._min_leaf - 1, n - self._min_leaf - 1):
                # loop through potential splitting points

                xi = sorted_x[i]
                if xi == sorted_x[i + 1]:
                    continue

                if self.is_valid_split(sorted_treatment, i):
                    lhs_treat_effect = self.estimate_treatment_in_leaf(
                        sorted_y[: (i + 1)], sorted_treatment[: (i + 1)]
                    )

                    rhs_treat_effect = self.estimate_treatment_in_leaf(
                        sorted_y[(i + 1) :], sorted_treatment[(i + 1) :]
                    )

                    sorted_transformed_y = CausalTree.transform_outcome(
                        sorted_y, sorted_treatment
                    )
                    lhs_loss = np.sum(
                        (lhs_treat_effect - sorted_transformed_y[: (i + 1)]) ** 2
                    )
                    rhs_loss = np.sum(
                        (rhs_treat_effect - sorted_transformed_y[(i + 1) :]) ** 2
                    )

                    tmp_loss = lhs_loss + rhs_loss
                    if tmp_loss < loss:
                        split_index, split_value, loss = var_index, xi, tmp_loss

        return split_index, split_value, loss

    def fit(
        self, X, y, treatment_status=None, min_leaf=8, max_distance=4, crit_num_obs=None
    ):
        """Uses data X, y and treatment_status to fit a Causal Tree, only restricted by
        arguments min_leaf, max_distance and crit_num_obs. If treatment_status is None
        we assume that X is a pandas DataFrame containing a column named
        "treatment_status".

        :param X: ndarray: 2D array containing data on features.
        :param y: ndarray: 1D array containing data on outcomes.
        :param treatment_status: ndarray: 1D array containg data on treatment status.
        :param min_leaf: int: Minimum number of observations required in each leaf.
        :param max_distance: int: Maximum number of absolute difference in observations
        with treatment and observations without treatment in each leaf.
        :param crit_num_obs: int: Critical value of observations in each leaf: When
        reached assertion using attribute max_distance is binding.
        :return: CausalTree: Fitted tree object.
        """
        # Check Input Values and do stuff for root
        if self.is_root:
            assert min_leaf >= 1, "Parameter <<min_leaf>> has to be bigger than one."
            assert len(X) == len(
                y
            ), "Data <<X>> and <<y>> must have the same number of observations."
            assert len(y) >= 2 * min_leaf, (
                "Data has not enough observations for a single split to occur "
                "given value of <<min_leaf>>."
            )
            if treatment_status is None:
                assert "treatment_status" in X.columns
                treatment_status = X[["treatment_status"]]
                X = X.drop(["treatment_status"], axis=1)
            else:
                assert len(y) == len(treatment_status)

            try:
                self._feature_names = X.columns.values
            except AttributeError:
                pass
            X = coerce_to_ndarray(X)  # coerce to ndarray
            y = coerce_to_ndarray(y)
            treatment_status = coerce_to_ndarray(treatment_status)
            assert np.array_equiv(np.unique(treatment_status), np.array([0, 1]))
            self._num_features = X.shape[1]
            if crit_num_obs is None:
                crit_num_obs = 4 * max_distance

        # Set Parameters
        self._min_leaf = min_leaf
        self._max_distance = max_distance
        self._crit_num_obs = crit_num_obs
        self._value = self.estimate_treatment_in_leaf(y, treatment_status)
        self._y = y
        self._y_transformed = CausalTree.transform_outcome(y, treatment_status)
        self._treatment_status = treatment_status
        self._node_loss = np.sum((self.value - self.y_transformed) ** 2)

        # Actual Fitting
        self._split_var, self._split_value, tmp_loss = self.find_best_splitting_point(X)
        self._is_fitted = True

        if self._split_var is None:
            self._branch_loss = self._node_loss
        else:
            self._branch_loss = tmp_loss

        if self._split_var is not None:
            index = X[:, self._split_var] <= self._split_value

            self._left_child = CausalTree(parent=self)
            self._right_child = CausalTree(parent=self)
            self._left_child.feature_names = self.feature_names
            self._right_child.feature_names = self.feature_names

            self._left_child.fit(
                X[index],
                y[index],
                treatment_status[index],
                min_leaf,
                max_distance,
                crit_num_obs,
            )
            self._right_child.fit(
                X[~index],
                y[~index],
                treatment_status[~index],
                min_leaf,
                max_distance,
                crit_num_obs,
            )

        self.update_branch_loss()
        return self

    def get_level_in_list(self, level):
        """Traverses tree and saves all nodes in a given level in a list.
        Example:
            level:
                0: tree
                1: [tree.left_child, tree.right_child] (first level)
        :param level: int: Number representing desired depth.
        :return: list: Nodes in given level.
        """
        level_list = []
        if level == 0:
            level_list.append(self)
        else:
            if self.left_child is not None:
                level_list.extend(self.left_child.get_level_in_list(level - 1))
                level_list.extend(self.right_child.get_level_in_list(level - 1))
        return level_list

    def predict_row(self, xi):
        """
        :param xi: ndarray: 1D array of new data on features of a single observation.
        :return: float: Treatment prediction conditional on xi.
        """
        if self.is_leaf:
            return self.value
        child = (
            self._left_child
            if xi[self._split_var] <= self._split_value
            else self._right_child
        )
        return child.predict_row(xi)

    def predict(self, x):
        """
        :param x: ndarray: 2D array containing data on features.
        :return: ndarray: 1D array containing estimated treatment effects.
        """
        x = coerce_to_ndarray(x)
        assert (
            self._is_fitted
        ), "The tree has not yet been fitted; no prediction is possible."
        assert (
            x.shape[1] == self._num_features
        ), "New Data must have the same dimension as the Data used for Fitting."

        return np.array([self.predict_row(xi) for xi in x])

    def plot(self, render=False, save=False, filename=None):
        """Plotting function that nicely visualizes the information contained in the
        class by creating an (upside-down) tree-like structure plot. Inner nodes contain
        information on the splitting process of the feature space while leafs contain
        information on treatment predictions in regions of the partition.

        :param tree: CausalTree: Fitted tree.
        :param filename: str: Name of file that should be created.
        :param save: bool: Should the file be saved to disk.
        """
        if not self.is_fitted:
            print("The tree must be fitted in order to be plotted")
            return
        if filename is None:
            filename = "causal_tree.svg"
        dot = Graph(name="causal_tree", filename=filename, format="svg")
        dot.node(
            str(id(self)),
            self.splitting_info_to_string()
            + "\nestimate:"
            + str(round(float(self.value), 3)),
        )
        for i in range(self.depth):
            nodes = self.get_level_in_list(i + 1)
            for node in nodes:
                if node.left_child is None:
                    dot.node(
                        str(id(node)),
                        "This node is not split"
                        + "\nestimate:"
                        + str(round(float(node.value), 3)),
                    )
                    dot.edge(str(id(node.parent)), str(id(node)))
                else:
                    dot.node(
                        str(id(node)),
                        node.splitting_info_to_string()
                        + "\nestimate:"
                        + str(round(float(node.value), 3)),
                    )
                    dot.edge(str(id(node.parent)), str(id(node)))
        if render:
            dot.render(view=True)
        if save:
            dot.save()
        return dot

#  General Functions (Some of which might be static methods in a strict sense)

def pre_order_traverse_tree(root: CausalTree, func=None) -> list:
    """For many steps of multiple algorithms we need to traverse the tree and apply
    a function to each node one after the other. This function traverses the tree in a
    standard pre-order fashion. If func is None we simply append each node to a list and
    return that, otherwise func is applied before appending.

    :param root: CausalTree: Root node of a tree.
    :param func: Function that should be applied to nodes while traversing the tree.
    :return: list: Elements are either nodes given in pre-order or nodes in pre-order
    with func applied to each one.
    """
    values = []
    if func is None:
        if root is not None:
            values.append(root)
            values.extend(pre_order_traverse_tree(root.left_child))
            values.extend(pre_order_traverse_tree(root.right_child))
    else:
        if root is not None:
            values.append(func(root))
            values.extend(pre_order_traverse_tree(root.left_child, func=func))
            values.extend(pre_order_traverse_tree(root.right_child, func=func))
    return values

def g_value(node: CausalTree) -> float:
    return (
        float("Inf")
        if node.is_leaf
        else (node.node_loss - node.branch_loss) / (node.number_of_leafs - 1)
    )

def g(branch: CausalTree) -> tuple:
    nodes = pre_order_traverse_tree(branch)
    values = np.array(pre_order_traverse_tree(branch, g_value))
    minimum = values.min()
    argmin_index = np.where(values == minimum)[0]
    argmin = [nodes[ix] for ix in argmin_index]
    return argmin, minimum

def coerce_to_ndarray(obj) -> np.ndarray:
    if isinstance(obj, np.ndarray):
        return obj
    elif isinstance(obj, pd.Series):
        return obj.to_frame().values
    elif isinstance(obj, pd.DataFrame):
        return obj.values
    else:
        raise TypeError(
            "Object was given with inappropriate type;"
            "for matrices and vectors only use pandas Series, "
            "DataFrame or Numpy ndarrays"
        )

def test_monotonicity_list(lst: list, strictly=True) -> bool:
    """Test for (strict) monotonicity. In particular we expect the complexity-parameter
    in the pruning process to be an increasing sequence, which will be tested by this
    function.

    :param lst: Arbitrary list containing floats.
    :param strictly: bool: Should strict monotonicity be tested.
    :return: bool: True if lst is (strict) monotone increasing, False else.
    """
    if strictly:
        return all(x < y for x, y in zip(lst, lst[1:]))
    else:
        return all(x <= y for x, y in zip(lst, lst[1:]))


"""
Plot comparison of turbine models
"""
import sys
libpath = 'C:\\Users\\jrinker\\Documents\\GitHub\\dissertation'
if (libpath not in sys.path): sys.path.append(libpath)

import JR_Library.main as jr
import numpy as np
import os
import matplotlib.pyplot as plt

# set directory and turbine name
#turb_dir,TName = 'C:\\Users\\jrinker\\Documents\\GitHub' + \
#        '\\dissertation\\FAST_models\\verification\\WP0.75A08V00','WP0.75A08V00'
turb_dir,TName = 'C:\\Users\\jrinker\\Documents\\GitHub\\' + \
    'dissertation\\FAST_models\\FAST7\\WP0.75A08V00','WP0.75A08V00'
fileID,fignum = '42331',1
#fileID,fignum = '93013',2

#PlotFields = ['Time','WindVxi','RotSpeed','GenPwr',
#              'BldPitch1','TSR','GenTq','TTDspFA','OoPDefl1']
PlotFields = ['Time','WindVxi','RotSpeed',
              'BldPitch1','GenTq','TTDspFA','OoPDefl1']

# initialize figure
fig = plt.figure(fignum,figsize=(6.5,10))
plt.clf()

# load fast file
FASTfname = TName+'_'+fileID
FASTfpath = os.path.join(turb_dir,FASTfname+'.out')
FAST = jr.ReadFASTFile(FASTfpath)

# plot results
t = FAST['Data'][:,FAST['Fields'].index('Time')]

tplot = [150,155]
mask = (t >= tplot[0]) & (t <= tplot[1])

for i_plot in range(len(PlotFields)-1):
    PlotField = PlotFields[i_plot+1]

    ax = fig.add_subplot(len(PlotFields)-1,1,i_plot+1)

    if PlotField == 'WindVxi':
        ax.plot(t[mask],11*np.ones(t[mask].shape),'k:')

    y = FAST['Data'][:,FAST['Fields'].index(PlotField)]

    ax.plot(t[mask],y[mask])
    ax.set_title(PlotFields[i_plot+1])

    plt.locator_params(axis='y',nbins=4)
    #        if i_plot < len(PlotFields)-2:
    #            ax.set_ylim([np.mean(y)*0.95,np.mean(y)*1.05])
#    ax.set_xlim([150,160])
#    ax.margins(y=.1)

plt.tight_layout()
fig.suptitle(FASTfname,x=0.01,y=0.99,fontsize='large',ha='left')


#!/usr/bin/env python
"""
Copyright (c) 2017 Christian Uhsat <christian@uhsat.de>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import os
import sys

from epiphyte import Epiphyte

try:
    import pytest
except ImportError:
    sys.exit("Requires pytest")

class TestEpiphyte:
    """
    Epiphyte unit tests.
    """
    def test_fuzzy(self):
        """
        Fuzzy tests.
        """
        thread = os.urandom(32)
        tests = [os.urandom(4096) for n in range(10)]

        epiphyte = Epiphyte(thread, b'epiphyte_test')

        for test in tests:
            epiphyte.append(test)

        epiphyte = Epiphyte(thread, b'epiphyte_test')

        for test, data in zip(tests, epiphyte):
            assert test == data

if __name__ == "__main__":
    sys.exit(pytest.main(list(sys.argv)))


from lxml import etree
from urllib import parse
import re

#获取源代码
def get_source(html,xpath):
    tree=etree.HTML(html)
    try:
        section = tree.xpath(xpath)[0]
        source =etree.tostring(section, encoding="utf-8")
        source =source.decode()
    except:
        source=None

    return source
#获取内容
def get_text(html,xpath):
    tree = etree.HTML(html)
    text=""
    try:
        text = tree.xpath(xpath+"/text()")[0]
    except:
        print("无内容")
    return text
#获得图片
def get_image(html):
    tree=etree.HTML(html)
    srcList=tree.xpath("//img/@src")
    for i in range(len(srcList)):
        srcList[i]=parse.unquote(srcList[i])
    return srcList
#获得样式
def get_style(html):
    tree=etree.HTML(html)
    list=tree.xpath("//link[@type=\"text/css\"]/@href")
    return list

#获得样式

#减去无用元素
def minus_element(html,*rules,regax=False):
    if regax:
        for rule in rules:
            try:
                html=re.sub(rule,"",html)
            except:
                continue
    else:
        for rule in rules:
            try:
                old=get_source(html, rule)
                # print(old.encode("utf-8"))
                index=html.find(old[:20])
                html = html[:index]+html[index+len(old):]
            except:
                continue
    return html

#生成标签

def tag(name,*content,cls=None,**attrs):
    if cls is not None:
        attrs['class']=cls
    if attrs:
        attr_str = " ".join(' %s="%s"'%(attr,value) for attr,value in sorted(attrs.items()))
    else:
        attr_str=''
    if content:
        return '\n'.join('<%s%s>%s</%s>\n'%(name,attr_str,c,name) for c in content)
    else:
        return '<%s%s/>\n' %(name,attr_str)


import numpy as np
import scipy.stats
import matplotlib.pyplot as plt

# Primitives
alpha=0.4
beta=0.96
mu=0
s=0.1

def U_operator(w, grid, beta, u, f, shocks, cons_pol, Uw=None):
    """
    The approximate U operator, which computes and returns the
    updated value function Uw on the grid points.  An array to store
    the new set of values Uw is optionally supplied (to avoid having to
    allocate new arrays at each iteration).  If supplied, any existing data in
    Uw will be overwritten.

    Parameters
    ----------
    w : array_like(float, ndim = 1)
        The value of the input function on different grid points
    grid : array_like(float, ndim=1)
        The set of grid points
    beta : scalar
        The discount factor
    u : function
        The utility function
    f : function
        The production function
    shocks : numpy array
        An array of draws from the shock, for Monte Carlo integration (to
        compute expectations).
    Uw : array_like(float, ndim=1) optional (default=None)
        Array to write output values to
    cons_pol : Boolean to specify which consumption policy to use.
               If True, c = 0.95y. Otherwise, c = H(y)y

    """
    # === Apply linear interpolation to w === #
    w_func = lambda x: np.interp(x, grid, w)

    # == Initialize Uw if necessary == #
    if Uw is None:
        Uw = np.empty_like(w)

    # == set Uw[i] = u(c) + beta E w(f(y  - c) z) == #
    for i, y in enumerate(grid):
        if cons_pol:
            c = 0.95 * y
        else:
            c = scipy.stats.lognorm.cdf(y, 0.5)
        Uw[i] = u(c) + beta * np.mean(w_func(f(y - c) * shocks))
    return Uw

def u(c):
    " Utility Function "
    return np.log(c)

def f(k):
    " Deterministic part of production function.  "
    return k**alpha

# Generates Grids and Shocks
grid_max = 4         # Largest grid point
grid_size = 200      # Number of grid points
shock_size = 250     # Number of shock draws in Monte Carlo integral
n_iter = 300

grid = np.linspace(1e-5, grid_max, grid_size)
shocks = np.exp(mu + s * np.random.randn(shock_size))

w1 = 5 * np.log(grid)  # An initial condition
w2 = 5 * np.log(grid)

# Finds the value function for policy 1
for i in range(n_iter):
    w1 = U_operator(w1, grid, beta, u, f, shocks, cons_pol = True)

# Finds the value function for policy 2
for i in range(n_iter):
    w2 = U_operator(w2, grid, beta, u, f, shocks, cons_pol = False)
print(w1)
print(w2)
# Plots the value functions
fig, ax = plt.subplots(figsize=(9, 6))
ax.set_ylim(-40, 10)
ax.set_xlim(np.min(grid), np.max(grid))
ax.plot(grid, w1, 'k-', lw = 2, alpha = 0.7)
ax.plot(grid, w2, 'b-', lw = 2, alpha = 0.7)
plt.show()


# Python
import logging

# isort: split

# dependencies
from grpc.experimental import aio

# isort: split

# module
# isort: off
# from search.proto.genesearch_service.v1 import genesearch_pb2
# from search.proto.genesearch_service.v1 import genesearch_pb2_grpc
# from search.proto.chromosomesearch_service.v1 import chromosomesearch_pb2
# from search.proto.chromosomesearch_service.v1 import chromosomesearch_pb2_grpc
# from search.proto.chromosomeregion_service.v1 import chromosomeregion_pb2
# from search.proto.chromosomeregion_service.v1 import chromosomeregion_pb2_grpc
# NOTE: the following imports are a temporary workaround for a known protobuf
# bug; the commented imports above should be used when the bug is fixed:
# https://github.com/protocolbuffers/protobuf/issues/10075
from search import proto  # noqa: F401
from genesearch_service.v1 import genesearch_pb2, genesearch_pb2_grpc
from chromosomesearch_service.v1 import chromosomesearch_pb2, chromosomesearch_pb2_grpc
from chromosomeregion_service.v1 import chromosomeregion_pb2, chromosomeregion_pb2_grpc

# isort: on

async def gene_search(query, address):
    # fetch channel every time to support dynamic services
    channel = aio.insecure_channel(address)
    await channel.channel_ready()
    stub = genesearch_pb2_grpc.GeneSearchStub(channel)
    try:
        results = await stub.Search(genesearch_pb2.GeneSearchRequest(query=query))
        return results.genes
    except Exception as e:
        logging.error(e)
        return []

async def chromosome_search(query, address):
    # fetch channel every time to support dynamic services
    channel = aio.insecure_channel(address)
    await channel.channel_ready()
    stub = chromosomesearch_pb2_grpc.ChromosomeSearchStub(channel)
    try:
        results = await stub.Search(
            chromosomesearch_pb2.ChromosomeSearchRequest(query=query)
        )
        return results.chromosomes
    except Exception as e:
        logging.error(e)
        return []

async def chromosome_region(chromosome, start, stop, address):
    # fetch channel every time to support dynamic services
    channel = aio.insecure_channel(address)
    await channel.channel_ready()
    stub = chromosomeregion_pb2_grpc.ChromosomeRegionStub(channel)
    try:
        response = await stub.Get(
            chromosomeregion_pb2.ChromosomeRegionGetRequest(
                chromosome=chromosome, start=start, stop=stop
            )
        )
        return [response.region]
    except Exception as e:
        logging.error(e)
        return []


from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.gridlayout import GridLayout
from kivy.uix.bubble import Bubble
from kivy.garden.mapview import MapView, MapMarker, MapMarkerPopup
from mapview.geojson import GeoJsonMapLayer
from kivy.storage.jsonstore import JsonStore
from kivy.uix.image import Image
from kivy.uix.behaviors import ButtonBehavior

class ImageButton(ButtonBehavior, Image):
    pass

class MapIcons(GridLayout):
    def __init__(self, *args, **kwargs):
        super(MapIcons, self).__init__(**kwargs)

        self.rows = 3
        self.cols = 1
        self.padding = '8sp'
        self.spacing = '8sp'
        self.size_hint = (None, None)
        self.height = 3 * self.width

        self.btn_zoom_in = ImageButton(size_hint=(1, 1./3.),
                                       allow_stretch=True,
                                       keep_ratio=True,
                                       source="assets/icon_zoom_in.png")
        self.btn_zoom_out = ImageButton(size_hint=(1, 1./3.),
                                        allow_stretch=True,
                                        keep_ratio=True,
                                        source="assets/icon_zoom_out.png")
        self.btn_center_on = ImageButton(size_hint=(1, 1./3.),
                                         allow_stretch=True,
                                         keep_ratio=True,
                                         source="assets/icon_crosshair.png")

        self.add_widget(self.btn_zoom_in)
        self.add_widget(self.btn_zoom_out)
        self.add_widget(self.btn_center_on)

class ShowMap(BoxLayout):
    def __init__(self, *args, **kwargs):
        super(ShowMap, self).__init__(**kwargs)

        points = JsonStore("points_of_interest.json")

        self.view = MapView(lat=points["center"]['lat'],
                            lon=points["center"]['lon'],
                            zoom=15,
                            cache_dir="cache")

        # Add Route
        layer = GeoJsonMapLayer(source="route.json")
        self.view.add_layer(layer)

        # Add User
        user_marker = MapMarker(lat=points["center"]['lat'],
                                lon=points["center"]['lon'],
                                source='assets/icon_user.png')
        self.view.add_widget(user_marker)

        # Add Icons
        self.icons = MapIcons(width='60sp')

        def zoom_in(instance):
            print('Zoom In:', self.view.zoom)
            if self.view.zoom == 19:
                return
            self.view.zoom += 1
        self.icons.btn_zoom_in.bind(on_press=zoom_in)

        def zoom_out(instance):
            print('Zoom Out:', self.view.zoom)
            if self.view.zoom == 1:
                return
            self.view.zoom -= 1
        self.icons.btn_zoom_out.bind(on_press=zoom_out)

        def center_on(instance):
            print('Center On:', user_marker.lat, user_marker.lon)
            self.view.center_on(user_marker.lat, user_marker.lon)
        self.icons.btn_center_on.bind(on_press=center_on)

        self.view.add_widget(self.icons)

        for point in points["list"]:
            bubble = Bubble(orientation='vertical', background_image=point['image'])
            map_marker = MapMarkerPopup(id="marker",
                                        lat=point['lat'],
                                        lon=point['lon'],
                                        source='assets/icon_marker.png')
            map_marker.add_widget(bubble)
            self.view.add_marker(map_marker)

        self.add_widget(self.view)

        ### Begin Close Bubble ###
        def close_bubbles(arg, touch):
            for c in self.view.children:
                for cc in c.children:
                    if cc.id == 'marker':
                        if len(cc.children) and cc.is_open:
                            cc.is_open = False
                            cc.refresh_open_status()
        self.view.children[-1].bind(on_touch_up=close_bubbles)
        ### End Close Bubble ###

    def on_size(self, *args):
        if self.view:
            self.icons.pos = (self.width - self.icons.width, (self.height - self.icons.height)/2.)

class MapsApp(App):
    def build(self):
        self.title = 'Route Map'
        return ShowMap()

if __name__ == '__main__':
    MapsApp().run()