import tensorflow as tf
import numpy as np
import librosa
import os
import math
import pathlib
import matplotlib.pyplot as plt

from ..util import conversions

_FILES_PER_SHARD = 750
_BASE_PATH = os.path.join('/' 'home', 'kaspar', 'Glacier', 'data-zhaw-ba', 'unzipped', 'MIR-1K')
_VALIDATION_DATA_DIR = 'cv'
_TRAINING_DATA_DIR = 'tr'
_TEST_DATA_DIR = 'tt'
_SHARD_BASE_NAME = 'shard_mir1k_'
_SAMPLE_RATE = 16000

length_differences = []

def main():
    create_tfrecords_from_directory(_BASE_PATH)

def create_tfrecords_from_directory(dir):
    for data_dir in [_TRAINING_DATA_DIR, _TEST_DATA_DIR, _VALIDATION_DATA_DIR]:
        data_dir_path = os.path.join(dir, data_dir)

        wav_files = librosa.util.find_files(directory=data_dir_path, ext=['wav'], recurse=False, case_sensitive=False)
        number_of_shards = math.ceil(len(wav_files) / _FILES_PER_SHARD)

        for shard_number in range(number_of_shards):
            shard_name = f'{_SHARD_BASE_NAME}_{data_dir}_{str(shard_number).rjust(4, "0")}.tfrecord'
            shard_path = os.path.join(_BASE_PATH, 'tfrecords', data_dir, shard_name)

            with tf.io.TFRecordWriter(shard_path) as out:
                lowerIndex = shard_number * _FILES_PER_SHARD
                upperIndex = (shard_number + 1) * _FILES_PER_SHARD

                for fileIndex in range(lowerIndex, upperIndex if upperIndex <= len(wav_files) else len(wav_files)):
                    file_path = wav_files[fileIndex]
                    y, sr = librosa.load(file_path, sr=_SAMPLE_RATE, mono=False)
                    # _plot_stereo_audio(y, sr)
                    y = y[1]
                    # _plot_mono_audio(y, sr)

                    if y.dtype != np.dtype(np.int16):
                        y = (y * np.iinfo(np.int16).max).astype(np.int16)

                    file_name_stem = os.path.splitext(os.path.basename(file_path))[0]
                    ref_matches = list(pathlib.Path(_BASE_PATH).rglob(f'*{file_name_stem}*.[pP][vV]'))
                    if len(ref_matches) != 1:
                        print(f'Cannot find ref (.pv) file for [ {file_path} ], skipping...')
                        continue

                    ref_data = np.genfromtxt(ref_matches[0])
                    ref_data = conversions.convert_semitone_to_hz(ref_data, 10)  # Convert semitones to Hz
                    ref_data[ref_data <= 10] = 0  # Floor ref data 10 -> 0 Hz for model fitting
                    ref_data = np.insert(ref_data, 0, 0)
                    length_differences.append(len(y) / 16000 * 1000 - len(ref_data) * 20)

                    example = tf.train.Example(features=tf.train.Features(feature={
                        'data': _bytes_feature(y.tobytes()),
                        'data_sampling_rate': _int64_feature([sr]),
                        'data_num_channels': _int64_feature([1]),
                        'data_width': _int64_feature([len(y)]),
                        'pitch': _float_feature(ref_data),
                        'pitch_confidence': _float_feature(np.ones(len(ref_data))),
                    }))

                    out.write(example.SerializeToString())

    print('done')

def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _float_feature(value):
    return tf.train.Feature(float_list=tf.train.FloatList(value=value))

def _plot_stereo_audio(y, sr):
    fig, ax = plt.subplots(nrows=2, sharex='all', sharey='all')
    librosa.display.waveplot(y[0], sr=sr, ax=ax[0])
    ax[0].set(title='Channel 0')
    ax[0].label_outer()
    librosa.display.waveplot(y[1], sr=sr, ax=ax[1])
    ax[1].set(title='Channel 1')
    ax[1].label_outer()
    fig.show()

def _plot_mono_audio(y, sr):
    fig, ax = plt.subplots()
    librosa.display.waveplot(y, sr=sr)
    ax.set(title='Mono')
    ax.label_outer()
    fig.show()

main()


import copy
import math

from scipy import integrate
import numpy as np

import Materials

class Character():
    def __init__(self, skills, traits):
        self.skills = skills
        self.traits = traits
        self.items = []

class Creature():
    strength_scaler = None

class Thing(Materials.Material):
    def __init__(self):
        super().__init__()
        self.mass = 0
        self.volume = 0

    def calc_moment_of_inertia(self, extra_length=0):
        raise NotImplementedError

class Organ(Thing):
    def __init__(self, critical):
        super().__init__()
        self.critical = critical

class FloatingOrgan(Organ):
    def __init__(self, relative_size, **kwargs):
        super().__init__(**kwargs)
        self.relative_size = relative_size

class Brain(FloatingOrgan, Materials.Fleshy):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class Stomach(FloatingOrgan, Materials.Fleshy):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class Liver(FloatingOrgan, Materials.Fleshy):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class Intestines(FloatingOrgan, Materials.Fleshy):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class Lung(FloatingOrgan, Materials.Fleshy):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class Heart(FloatingOrgan, Materials.Muscle):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class WrapperOrgan(Organ):
    layer_padding = 0

    def __init__(self, thickness, **kwargs):
        super().__init__(**kwargs)
        self.thickness = thickness

class Larynx(WrapperOrgan, Materials.Fleshy):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class WrappingBone(WrapperOrgan, Materials.Bone):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class FloatingBone(FloatingOrgan, Materials.Bone):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class SkinLayer(WrapperOrgan, Materials.Fleshy):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

class MuscleLayer(WrapperOrgan, Materials.Muscle):
    def __init__(self, **kwargs):
        super().__init__(critical=False, **kwargs)

class FloatingOrganLayer(Organ, Materials.Fleshy):
    def __init__(self, contents, **kwargs):
        super().__init__(critical=False, **kwargs)
        self.contents = contents

# class BodyPlan:
#     def __init__(self):
#         self.parts_list = []

class CompoundItemLayers:
    def __init__(self, layers):
        self.layers = layers
        self.min_pierce_limit = self.layers[0].shear_plastic_limit

    def calc_damage(self, KE, volume):
        print("armour:")
        pl = self.shear_plastic_limit * volume
        el = self.shear_elastic_limit * volume

        print(KE, el, pl)

        if KE > pl:
            print('Pierced')
        elif KE > el:
            print('Bent')
            for s in np.linspace(0, self.strain_at_fracture, 100, endpoint=False):
                if integrate.quad(self.shear_stress_strain, 0, s)[0] * volume > KE:
                    print(s)
                    break
        else:
            print('Deflected')

    def calc_cutting_damage(self, KE, other):
        pass

    def calc_piercing_damage(self, KE, other):
        pass

    def calc_crushing_damage(self, KE, other):
        w = min(self.width, other.width)
        l = min(self.length, other.length)
        a = (2 * w + 2 * l) * self.thickness

class BodyPart(Thing):
    def __init__(self, parent, children, layers, can_grasp, can_attack, core=False):
        super().__init__()
        self.name = type(self).__name__
        self.parent = parent
        self.children = children
        self.layers = layers
        self.equipped = []
        self._can_grasp = can_grasp
        self.held = None
        self.can_attack = can_attack
        self.core = core
        self.length = None
        self.width = None
        self.thickness = None
        self.mom_inertia = None
        self.creature = None

    def scale(self, factor):
        self.length = self.length * factor
        self.width = self.width * factor
        self.volume = self.calc_volume()
        self.mass = self.volume * self.density

    def calc_volume(self):
        return self.length * self.width * self.thickness

    def calc_moment_of_inertia(self, extra_length=0):
        return (1 / 3) * self.mass * (self.length + extra_length) ** 2

    def calc_moment_of_inertia_below(self, len_to_part=0):
        I = self.calc_moment_of_inertia(len_to_part)

        if self.held is not None:
            I += self.held.calc_moment_of_inertia(len_to_part + self.length)

        if self.children is None:
            return I
        else:
            for part in self.children:
                I += part.calc_moment_of_inertia_below(len_to_part=len_to_part + self.length)

            return I

    def can_grasp(self):
        return self._can_grasp and self.held is None

    def equip(self, item, held=False):
        if held and self.can_grasp and self.held is None:
            self.held = item
        else:
            if item not in self.equipped:
                self.equipped.append(item)
                self.mass += item.mass

    def unequip(self, item, body_part):
        if body_part.can_grasp and item in self.equipped:
            body_part.equip(item, held=True)
            self.equipped.remove(item)
            self.mass -= item.mass

    def drop(self, item):
        if item in self.held:
            self.held = None

    def get_parts_below(self):
        parts = [self]

        if self.children is None:
            return parts
        else:
            for part in self.children:
                parts.extend(part.get_parts_below())

            return parts

    def get_parts_above(self, num_joints=-1, joints_found=0):
        parts = [self]
        if self.core:
            if num_joints == -1:
                return []
            elif joints_found < num_joints:
                raise Exception("That many joints don't exist above this part")
        if isinstance(self, Joint):
            joints_found += 1
            if joints_found == num_joints:
                return parts

        return parts + self.parent.get_parts_above(num_joints, joints_found)

    def calc_slash_speed(self, num_joints, joints_found=0):
        joint_speed = 0
        if self.core:
            if num_joints == -1:
                return 0, 0
            elif joints_found < num_joints:
                raise Exception("That many joints don't exist above this part")
        if isinstance(self, Joint):
            joints_found += 1
            joint_speed = self.rel_strength * self.creature.traits[
                'strength'] / self.calc_moment_of_inertia_below() * self.creature.strength_scaler
            if joints_found == num_joints:
                return joint_speed, self.length

        speed, length = self.parent.calc_slash_speed(num_joints, joints_found)
        return speed + joint_speed, length + self.length

    def calc_jab_speed(self, num_joints, joints_found=0):
        joint_speed = 0
        if self.core:
            if num_joints == -1:
                return 0, 0
            elif joints_found < num_joints:
                raise Exception("That many joints don't exist above this part")
        if isinstance(self, Joint):
            joints_found += 1
            joint_speed = self.rel_strength * self.creature.traits[
                'strength'] / self.calc_moment_of_inertia_below() * self.creature.strength_scaler
            if joints_found == num_joints:
                return joint_speed

        speed = self.parent.calc_jab_speed(num_joints, joints_found)
        return speed + joint_speed * self.length

    def calc_damage(self, KE, volume):
        print("armour:")
        pl = self.shear_plastic_limit * volume
        el = self.shear_elastic_limit * volume

        print(KE, el, pl)

        if KE > pl:
            print('Pierced')
        elif KE > el:
            print('Bent')
            for s in np.linspace(0, self.strain_at_fracture, 100, endpoint=False):
                if integrate.quad(self.shear_stress_strain, 0, s)[0] * volume > KE:
                    print(s)
                    break
        else:
            print('Deflected')

    def calc_cutting_damage(self, KE, other):
        pass

    def calc_piercing_damage(self, KE, other):
        pass

    def calc_crushing_damage(self, KE, other):
        w = min(self.width, other.width)
        l = min(self.length, other.length)
        a = (2 * w + 2 * l) * self.thickness

    def _swing_helper(self, other, num_joints):
        ang_vel, body_len = self.calc_slash_speed(num_joints)

        if self.held is not None:
            self.held.foreswing(other, ang_vel, body_len)
        else:
            impact_length = min(self.length, self.width, other.width, other.length)
            tip_velocity = ang_vel * (body_len + self.length)
            KE = .5 * self.mass * tip_velocity ** 2

            if impact_length == self.length or impact_length == self.width:
                self.calc_crushing_damage(KE, other)
            else:
                self.calc_cutting_damage(KE, other)
            other.calc_cutting_damage(KE, self)

    def foreswing(self, other, num_joints):
        self._swing_helper(other, num_joints)

    def backswing(self, other, num_joints):
        self._swing_helper(other, num_joints)

    def upswing(self, other, num_joints):
        self._swing_helper(other, num_joints)

    def downswing(self, other, num_joints):
        self._swing_helper(other, num_joints)

    def jab(self, other, num_joints):
        vel = self.calc_jab_speed(num_joints)
        print(vel)
        if self.held is not None:
            self.held.jab(other, vel)
        else:
            pass
            # impact_length = min(self.length, self.width, other.width, other.length)
            # tip_velocity = ang_vel * (body_len + self.length)
            # KE = .5 * self.mass * tip_velocity ** 2
            #
            # if impact_length == self.length or impact_length == self.width:
            #     self.calc_crushing_damage(KE, other)
            # else:
            #     self.calc_cutting_damage(KE, other)
            # other.calc_cutting_damage(KE, self)

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.name

class Digit(BodyPart, Materials.Fleshy):
    def __init__(self, parent=None, children=None, opposable=False, joints=3.0, length=95.0,
                 width=17.0, thickness=17.0, **kwargs):
        children = children if children is not None else []
        super().__init__(parent, children, **kwargs)
        self.num_joints = joints
        self.opposable = opposable
        self.length = length
        self.width = width
        self.thickness = thickness
        self.volume = self.calc_volume()
        self.mass = self.volume * self.density

class LimbTerminus(BodyPart, Materials.Fleshy):
    def __init__(self, parent=None, children=None, length=96.0, width=84.0, thickness=35.0,
                 **kwargs):
        children = children if children is not None else []
        super().__init__(parent, children, **kwargs)
        self.length = length
        self.width = width
        self.thickness = thickness
        self.volume = self.calc_volume()
        self.mass = self.volume * self.density

class LimbSegment(BodyPart, Materials.Fleshy):
    def __init__(self, parent=None, children=None, length=460.0, width=80.0, thickness=80.0,
                 **kwargs):
        children = children if children is not None else []
        super().__init__(parent, children, **kwargs)
        self.length = length
        self.width = width
        self.thickness = thickness
        self.volume = self.calc_volume()
        self.mass = self.volume * self.density

class Joint(BodyPart, Materials.Fleshy):
    def scale(self, factor):
        pass

    def __init__(self, rel_strength, parent=None, children=None, **kwargs):
        super().__init__(parent, children, layers=[], **kwargs)
        children = children if children is not None else []
        self.length = 0
        self.width = 0
        self.thickness = 0
        self.rel_strength = rel_strength

class TorsoSegment(BodyPart, Materials.Fleshy):
    def __init__(self, parent=None, children=None, length=300.0, width=142.0, thickness=86.0,
                 **kwargs):
        children = children if children is not None else []
        super().__init__(parent, children, **kwargs)
        self.length = length
        self.width = width
        self.thickness = thickness
        self.volume = self.calc_volume()
        self.mass = self.volume * self.density

class Neck(BodyPart, Materials.Fleshy):
    def __init__(self, parent=None, children=None, length=110.0, width=113.0, thickness=113.0,
                 **kwargs):
        children = children if children is not None else []
        super().__init__(parent, children, **kwargs)
        self.length = length
        self.width = width
        self.thickness = thickness
        self.volume = self.calc_volume()
        self.mass = self.volume * self.density

class Head(BodyPart, Materials.Fleshy):
    def __init__(self, parent=None, children=None, length=218.0, width=186.0, thickness=186.0,
                 **kwargs):
        children = children if children is not None else []
        super().__init__(parent, children, **kwargs)
        self.length = length
        self.width = width
        self.thickness = thickness
        self.volume = self.calc_volume()
        self.mass = self.volume * self.density

class Humanoid(Thing, Creature):
    num_arms = 2
    num_legs = 2
    num_fingers = 5
    num_toes = 5
    ideal_skin = SkinLayer(thickness=1 / 11.5 / 72.5, critical=False)
    ideal_cranium = WrappingBone(thickness=1 / 11.5 / 20, critical=False)
    ideal_arm_bone = WrappingBone(thickness=1 / 11.5 / 29, critical=False)
    ideal_hand_bone = WrappingBone(thickness=.2 / 11.5, critical=False)
    ideal_leg_bone = WrappingBone(thickness=1 / 11.5 / 18, critical=False)
    ideal_foot_bone = WrappingBone(thickness=1 / 11.5 / 20, critical=False)
    ideal_ribcage = WrappingBone(thickness=1 / 11.5 / 14.5, critical=False)
    ideal_spine = WrappingBone(thickness=1 / 11.5 / 11, critical=True)
    ideal_brain = Brain(relative_size=.9, critical=True)
    ideal_stomach = Stomach(relative_size=.2, critical=True)
    ideal_liver = Liver(relative_size=.3, critical=True)
    ideal_intestine = Intestines(relative_size=.5, critical=False)
    ideal_larynx = Larynx(thickness=1 / 11.5 / 15, critical=True)
    ideal_heart = Heart(relative_size=.1, critical=True)
    ideal_lung = Lung(relative_size=.4, critical=True)

    ideal_head = Head(layers=[ideal_skin, ideal_cranium, FloatingOrganLayer(contents=[ideal_brain])], length=1 / 7.5, width=1 / 11.5,
                      thickness=1 / 11.5,
                      can_attack=True, core=True, can_grasp=False)
    ideal_neck = Neck(layers=[ideal_skin, MuscleLayer(thickness=1 / 11.5 / 3), ideal_larynx, ideal_spine],
                      length=1 / 11.5, width=1 / 11.5, thickness=1 / 11.5, can_attack=False, core=True, can_grasp=False)
    ideal_upper_torso = TorsoSegment(layers=[ideal_skin, MuscleLayer(thickness=1 / 11.5 / 20), ideal_ribcage, FloatingOrganLayer(contents=[ideal_lung, ideal_lung, ideal_heart]),
                                             ideal_spine], length=1.5 / 7.5, width=2 / 11.5, thickness=1.25 / 11.5,
                                     can_attack=True, core=True, can_grasp=False)
    ideal_upper_torso.name = 'Upper Torso'
    ideal_lower_torso = TorsoSegment(layers=[ideal_skin, MuscleLayer(thickness=1 / 11.5 / 15), FloatingOrganLayer(contents=[ideal_intestine, ideal_liver, ideal_stomach]), ideal_spine],
                                     length=1.25 / 7.5, width=1.75 / 11.5, thickness=1.5 / 11.5, can_attack=False,
                                     core=True, can_grasp=False)
    ideal_lower_torso.name = 'Lower Torso'
    ideal_upper_leg = LimbSegment(layers=[ideal_skin, MuscleLayer(thickness=1 / 11.5 / 2), ideal_leg_bone],
                                  length=2 / 7.5, width=1 / 11.5, thickness=1 / 11.5, can_attack=False, can_grasp=False)
    ideal_lower_leg = LimbSegment(layers=[ideal_skin, MuscleLayer(thickness=1 / 11.5 / 3), ideal_leg_bone],
                                  length=1.75 / 7.5, width=.5 / 11.5, thickness=.5 / 11.5, can_attack=False,
                                  can_grasp=False)
    ideal_foot = LimbTerminus(layers=[ideal_skin, ideal_foot_bone], length=1.5 / 11.5, width=.5 / 11.5,
                              thickness=.4 / 11.5, can_attack=True, can_grasp=False)
    ideal_toe = Digit(layers=[ideal_skin, ideal_foot_bone], length=.2 / 11.5, width=.1 / 11.5,
                      thickness=.1 / 11.5, can_attack=False, can_grasp=False)
    ideal_upper_arm = LimbSegment(layers=[ideal_skin, MuscleLayer(thickness=1 / 11.5 / 3), ideal_arm_bone],
                                  length=2 / 11.5, width=.4 / 11.5, thickness=.4 / 11.5, can_attack=False,
                                  can_grasp=False)
    ideal_lower_arm = LimbSegment(layers=[ideal_skin, MuscleLayer(thickness=1 / 11.5 / 4), ideal_arm_bone],
                                  length=2 / 11.5, width=.25 / 11.5, thickness=.2 / 11.5, can_attack=False,
                                  can_grasp=False)
    ideal_palm = LimbTerminus(layers=[ideal_skin, ideal_hand_bone], length=.8 / 11.5, width=.8 / 11.5,
                              thickness=.2 / 11.5, can_attack=True, can_grasp=True)
    ideal_finger = Digit(layers=[ideal_skin, ideal_hand_bone], length=.5 / 11.5, width=.15 / 11.5,
                         thickness=.15 / 11.5, can_attack=False, can_grasp=False)
    ideal_hip = Joint(.5, can_attack=True, can_grasp=False)
    ideal_knee = Joint(.3, can_attack=True, can_grasp=False)
    ideal_shoulder = Joint(.4, can_attack=True, can_grasp=False)
    ideal_elbow = Joint(.2, can_attack=True, can_grasp=False)

    def __init__(self, height):
        super().__init__()
        self.head = copy.deepcopy(self.ideal_head)
        self.head.contents = [self.ideal_skin, self.ideal_cranium, self.ideal_brain]
        self.neck = copy.deepcopy(self.ideal_neck)
        self.neck.contents = []
        self.upper_torso = copy.deepcopy(self.ideal_upper_torso)
        self.lower_torso = copy.deepcopy(self.ideal_lower_torso)

        self.hip = copy.deepcopy(self.ideal_hip)
        self.hip.name = 'Hip'

        self.left_upper_leg = copy.deepcopy(self.ideal_upper_leg)
        self.left_upper_leg.name = 'Left Upper Leg'

        self.right_upper_leg = copy.deepcopy(self.ideal_upper_leg)
        self.right_upper_leg.name = 'Right Upper Leg'

        self.left_knee = copy.deepcopy(self.ideal_knee)
        self.left_knee.name = 'Left Knee'

        self.right_knee = copy.deepcopy(self.ideal_knee)
        self.right_knee.name = 'Right Knee'

        self.left_lower_leg = copy.deepcopy(self.ideal_lower_leg)
        self.left_lower_leg.name = 'Left Lower Leg'

        self.right_lower_leg = copy.deepcopy(self.ideal_lower_leg)
        self.right_lower_leg.name = 'Right Lower Leg'

        self.left_foot = copy.deepcopy(self.ideal_foot)
        self.left_foot.name = 'Left Foot'

        self.right_foot = copy.deepcopy(self.ideal_foot)
        self.right_foot.name = 'Right Foot'

        self.left_toes = copy.deepcopy([copy.deepcopy(self.ideal_toe) for i in range(5)])
        self.right_toes = copy.deepcopy([copy.deepcopy(self.ideal_toe) for i in range(5)])

        self.left_shoulder = copy.deepcopy(self.ideal_shoulder)
        self.left_shoulder.name = 'Left Shoulder'

        self.right_shoulder = copy.deepcopy(self.ideal_shoulder)
        self.right_shoulder.name = 'Right Shoulder'

        self.left_upper_arm = copy.deepcopy(self.ideal_upper_arm)
        self.left_upper_arm.name = 'Left Upper Arm'

        self.right_upper_arm = copy.deepcopy(self.ideal_upper_arm)
        self.right_upper_arm.name = 'Right Upper Arm'

        self.left_elbow = copy.deepcopy(self.ideal_elbow)
        self.left_elbow.name = 'Left Elbow'

        self.right_elbow = copy.deepcopy(self.ideal_elbow)
        self.right_elbow.name = 'Right Elbow'

        self.left_lower_arm = copy.deepcopy(self.ideal_lower_arm)
        self.left_lower_arm.name = 'Left Lower Arm'

        self.right_lower_arm = copy.deepcopy(self.ideal_lower_arm)
        self.right_lower_arm.name = 'Right Lower Arm'

        self.left_palm = copy.deepcopy(self.ideal_palm)
        self.left_palm.name = 'Left Hand'

        self.right_palm = copy.deepcopy(self.ideal_palm)
        self.right_palm.name = 'Right Hand'

        self.left_fingers = copy.deepcopy([copy.deepcopy(self.ideal_finger) for i in range(5)])
        self.right_fingers = copy.deepcopy([copy.deepcopy(self.ideal_finger) for i in range(5)])

        self.head.children = [self.neck]
        self.neck.parent = self.head
        self.neck.children = [self.upper_torso]
        self.upper_torso.parent = self.neck
        self.upper_torso.children = [self.lower_torso, self.right_shoulder, self.left_shoulder]
        self.lower_torso.parent = self.upper_torso
        self.lower_torso.children = [self.hip]
        self.hip.parent = self.lower_torso
        self.hip.children = [self.left_upper_leg, self.right_upper_leg]
        self.left_upper_leg.parent = self.hip
        self.left_upper_leg.children = [self.left_knee]
        self.right_upper_leg.parent = self.hip
        self.right_upper_leg.children = [self.right_knee]
        self.right_knee.parent = self.right_upper_leg
        self.right_knee.children = [self.right_lower_leg]
        self.left_knee.parent = self.left_upper_leg
        self.left_knee.children = [self.left_lower_leg]
        self.left_lower_leg.parent = self.left_knee
        self.left_lower_leg.children = [self.left_foot]
        self.right_lower_leg.parent = self.right_knee
        self.right_lower_leg.children = [self.right_foot]
        self.left_foot.parent = self.left_lower_leg
        self.left_foot.children = self.left_toes
        self.right_foot.parent = self.right_lower_leg
        self.right_foot.children = self.right_toes
        self.left_shoulder.parent = self.upper_torso
        self.left_shoulder.children = [self.left_upper_arm]
        self.right_shoulder.parent = self.upper_torso
        self.right_shoulder.children = [self.right_upper_arm]
        self.left_upper_arm.parent = self.right_shoulder
        self.left_upper_arm.children = [self.left_elbow]
        self.right_upper_arm.parent = self.right_shoulder
        self.right_upper_arm.children = [self.right_elbow]
        self.left_elbow.parent = self.left_upper_arm
        self.left_elbow.children = [self.left_lower_arm]
        self.right_elbow.parent = self.right_upper_arm
        self.right_elbow.children = [self.right_lower_arm]
        self.left_lower_arm.parent = self.left_elbow
        self.left_lower_arm.children = [self.left_palm]
        self.right_lower_arm.parent = self.right_elbow
        self.right_lower_arm.children = [self.right_palm]
        self.left_palm.parent = self.left_lower_arm
        self.left_palm.children = self.left_fingers
        self.right_palm.parent = self.right_lower_arm
        self.right_palm.children = self.right_fingers

        for i, finger in enumerate(self.right_fingers):
            finger.name = "Right Finger %s" % str(i + 1)
            finger.parent = self.right_palm
            finger.children = []
        for i, finger in enumerate(self.left_fingers):
            finger.name = "Left Finger %s" % str(i + 1)
            finger.parent = self.left_palm
            finger.children = []

        for i, toe in enumerate(self.right_toes):
            toe.name = "Right toe %s" % str(i + 1)
            toe.parent = self.right_foot
            toe.children = []
        for i, toe in enumerate(self.left_toes):
            toe.name = "Left toe %s" % str(i + 1)
            toe.parent = self.left_foot
            toe.children = []

        self.parts_list = [self.head, self.neck, self.upper_torso, self.lower_torso, self.left_upper_leg,
                           self.right_upper_leg, self.left_lower_leg, self.right_lower_leg, self.left_foot,
                           self.right_foot, self.left_upper_arm, self.right_upper_arm, self.left_lower_arm,
                           self.right_lower_arm, self.left_palm, self.right_palm, self.right_shoulder, self.hip,
                           self.left_shoulder, self.right_elbow, self.left_elbow, self.left_knee, self.right_knee]

        self.parts_list.extend(self.left_fingers)
        self.parts_list.extend(self.right_fingers)
        self.parts_list.extend(self.left_toes)
        self.parts_list.extend(self.right_toes)

        for part in self.parts_list:
            part.scale(height)

        self.attacking_parts = [part for part in self.parts_list if part.can_attack]

class Human(Humanoid, Character):
    max_height = 2.7
    min_height = .6
    max_bmi = 70
    min_bmi = 10
    strength_scaler = .04

    def __init__(self, height, bmi, skills, traits):
        super().__init__(height)
        self.traits = traits
        for part in self.parts_list:
            part.creature = self

h = Human(2, 20, {}, {'strength': 40})
print(h.right_palm.calc_slash_speed(1))
# TODO Attacks don't come from body level but from part level - i.e. leg can attack
# TODO if part is wielding an object use the corresponding attack method from the object
# TODO Attacks come in: Jab Foreswing Backswing Downswing Upswing (Swing come in slap and strike)


# -*- coding: utf-8 -*-

import numpy as np
import warnings
import sys
import os
import argparse
import re

from keras.models import Model
from keras.models import Sequential
from keras.layers import Flatten
from keras.layers import Dense
from keras.layers import Input
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.preprocessing import image
from keras.utils import layer_utils
from keras import backend as K
from keras.applications.imagenet_utils import preprocess_input
from keras.applications.imagenet_utils import _obtain_input_shape
from keras.engine.topology import get_source_inputs

from sklearn.svm import SVC
from sklearn.decomposition import IncrementalPCA
from sklearn.decomposition import PCA
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score

def VGG16(include_top=True, weights='imagenet',
          input_tensor=None, input_shape=None,
          pooling=None,
          classes=1000):

    WEIGHTS_PATH = 'vgg16_weights_tf_dim_ordering_tf_kernels.h5'
    # Determine proper input shape
    input_shape = _obtain_input_shape(input_shape,
                                      default_size=224,
                                      min_size=48,
                                      data_format=K.image_data_format(),
                                      include_top=include_top)

    if input_tensor is None:
        img_input = Input(shape=input_shape)
    else:
        if not K.is_keras_tensor(input_tensor):
            img_input = Input(tensor=input_tensor, shape=input_shape)
        else:
            img_input = input_tensor
    # Block 1
    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)
    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)

    # Block 2
    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)
    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)

    # Block 3
    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)
    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)

    # Block 4
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)

    # Block 5
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)

    x = Flatten(name='flatten')(x)
    x = Dense(4096, activation='relu', name='fc1')(x)
    x = Dense(4096, activation='relu', name='fc2')(x)
    x = Dense(classes, activation='softmax', name='predictions')(x)

    # Ensure that the model takes into account
    # any potential predecessors of `input_tensor`.
    if input_tensor is not None:
        inputs = get_source_inputs(input_tensor)
    else:
        inputs = img_input
    # Create model.
    model = Model(inputs, x, name='vgg16')

    # load weights
    model.load_weights(WEIGHTS_PATH)

    return model

def natural_sort(l):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key=alphanum_key)

if __name__ == '__main__':
    #PARSE PROGRAM ARGUMENTS
    parser = argparse.ArgumentParser()
    parser.add_argument('imgBasePath', help="Insert the path to the Image Database you want to use\nI.E. \tCaltech101_600/\n\timages/Caltech101_600/")
    parser.add_argument('train', help='True for training, False for predicting', type=str)
    parser.add_argument('--n_components', help='Number of components to reduce in PCA', type=int, nargs='*', default=[16])
    parser.add_argument('-k', help='Number of neighbors in k-NN', nargs='*', type=int, default=[1])
    #parser.add_argument('--predict', help='Predict the class for the input Image Database, using the pre-trained VGG-16')
    args = parser.parse_args()

    if args.train.lower() == 'train':
        #LOAD THE PRE-TRAINED MODEL AND EXTRACT THE 15TH LAYER
        base_model = VGG16(weights='imagenet')
        model = Model(inputs=base_model.input, outputs=base_model.get_layer('fc2').output)

        ipca_preds = np.array([[]])
        ipca_classes = np.array([])

        #GETS ALL IMAGES IN THE SUBDIRECTORIES
        for root, dirs, files in os.walk(args.imgBasePath):
            for f in natural_sort(files):
                if f.endswith(".jpg") or f.endswith('.jpeg'):
                    #print('File:\t' + os.path.join(root,f) + '\t|Class:\t' + root.split('/')[-1])

                    if ipca_classes.size > 0:
                        ipca_classes = np.append(ipca_classes, root.split('/')[-1])
                    else:
                        ipca_classes = np.array([root.split('/')[-1]])

                    #LOAD AND PREPROCESS THE IMAGE
                    img = image.load_img(os.path.join(root, f), target_size=(224, 224))
                    x = image.img_to_array(img)
                    x = np.expand_dims(x, axis=0)
                    x = preprocess_input(x)

                    prediction = model.predict(x)

                    #RUN THE IMAGE THROUGH THE MODEL
                    if ipca_preds.size > 0:
                        ipca_preds = np.append(ipca_preds, prediction, axis=0)
                    else:
                        ipca_preds = prediction

        print('Database: ', args.imgBasePath)

        f_name = args.imgBasePath.split('/')[-2];
        if not os.path.exists('build/'):
            os.makedirs('build/')

        with open('build/'+f_name+'_4096_weights.npy','wb+') as f_ipca :
            np.save(f_ipca, ipca_preds)

        with open('build/'+f_name+'_4096_classes.npy','wb+') as c_ipca:
            np.save(c_ipca, ipca_classes)

        f_ipca.close()
        c_ipca.close()

        print('N_Components: 4096')
        print('Predictions ', ipca_preds.shape)
        print('IPCA Output None')

        #RUN THE EXTRACTED VALUES THROUGH PCA TO REDUCE THE NUMBER OF FEATURES
        for n in args.n_components:
            #CREATE THE WEIGHTS AND CLASSES FILES

            ipca = IncrementalPCA(n_components=n, whiten=True, copy=True)
            ipca_output = ipca.fit_transform(ipca_preds)

            with open('build/'+f_name+'_'+str(n)+'_weights.npy','wb+') as f_ipca:
                np.save(f_ipca, ipca_output)

            with open('build/'+f_name+'_'+str(n)+'_classes.npy','wb+') as c_ipca:
                np.save(c_ipca, ipca_classes)

            f_ipca.close()
            c_ipca.close()

            print('N_Components: ', n)
            print('Predictions ', ipca_preds.shape)
            print('IPCA Output ', ipca_output.shape)

    if args.train.lower() == 'test':
        f_name = args.imgBasePath.split('/')[-2];
        print('Database: ', f_name)

        for n in args.n_components:
            print()
            for a in args.k:

                with open('build/'+f_name+'_'+str(n)+'_weights.npy','rb') as f_data :
                    data = np.load(f_data)
                    for d in f_data:
                        data = np.append(data, d)
                with open('build/'+f_name+'_'+str(n)+'_classes.npy','rb') as f_class:
                    classes = np.load(f_class)
                    for c in f_class:
                        classes = np.append(classes, c)
                f_data.close()
                f_class.close()

                knn = KNeighborsClassifier(n_neighbors=a)
                scores = cross_val_score(knn, data, classes, cv=5, n_jobs=-1)
                print('k-NN: %d Neighbors' % (a))
                print('\tN_Components: ', n)
                print('\tAccuracy: %0.4f +/- %0.4f' % (scores.mean(), scores.std()))

            svm = SVC(kernel='linear')
            scores = cross_val_score(svm, data, classes, cv=5, n_jobs=-1)
            print('SVC Linear')
            print('\tN_Components: ', n)
            print('\tAccuracy: %0.4f +/- %0.4f' % (scores.mean(), scores.std()))


#!/usr/bin/env python

from setuptools import find_packages, setup

tests_require = [
    'pytest>=5.0.0,<6.0.0',
]

extras_require = {
    "docs": [
        'bdc-readthedocs-theme @ git+git://github.com/brazil-data-cube/bdc-readthedocs-theme.git#egg=bdc-readthedocs-theme',
        'Sphinx>=2.1.2',
    ],
    "tests": tests_require
}

setup(
    name='bdc-core',
    version='0.0.4',
    description='Brazilian Data Cube Core Package',
    author='Admin',
    author_email='admin@admin.com',
    license="MIT",
    packages=find_packages(),
    url='https://github.com/brazil-data-cube/',
    install_requires=[
        'Flask>=1.0.3',
        'flask-restplus>=0.12.1',
        'Flask-JWT>=0.3.2',
        'jsonschema>=3.0.1',
        'secure-smtplib==0.1.1',
        'Mako==1.1.1'
    ],
    extras_require=extras_require,
    tests_require=tests_require
)


"""

This script converts different inputs to a waterscenario for BIVAS

Jurjen de Jong, 24 april 2020

"""

from pathlib import Path
from klimaatbestendige_netwerken.pyBIVAS import pyBIVAS
from klimaatbestendige_netwerken.waterdepthgrid import WaterdepthGrid
from klimaatbestendige_netwerken.externals.SOBEK_to_Pandas import read_nc as read_sobek_nc
from klimaatbestendige_netwerken.externals.dini_to_json_via_xml import ini_to_json
import logging
import numpy as np
import networkx as nx
import pandas as pd
from shapely.ops import nearest_points

class CreateWaterScenario:
    def __init__(self):
        self.reach = {}
        pass

    def load_sobek(self, SOBEK_output_reachsegments: Path, SOBEK_output_gridpoints: Path, SOBEK_network_file: Path,
                   exclude_sobek_reaches_from_network=[]):
        logging.info('Loading SOBEK results')

        self.gridpoints_data, self.gridpoints = read_sobek_nc(SOBEK_output_gridpoints, return_attributes=True)
        self.reachseg_data, self.reachseg = read_sobek_nc(SOBEK_output_reachsegments, return_attributes=True)

        self.gridpoints['branchname'] = self.gridpoints['gridpoint_id'].str.rsplit('_', 1, expand=True)[0]
        self.reachseg['branchname'] = self.reachseg['reach_segment_id'].str.rsplit('_', 1, expand=True)[0]

        logging.info('Create networkx')
        self.sobek_networkx = self.networkx_from_sobek3(SOBEK_network_file, exclude_sobek_reaches_from_network)

        logging.info('Computing stats')

        def sobek_stats(df):
            time_end = df.index[-1]
            time_start = time_end - pd.Timedelta(hours=24, minutes=50)
            df = df.loc[time_start:time_end]

            df_stats = df.describe(percentiles=[0.05, 0.25, 0.5, 0.75, 0.95]).drop('count').stack().unstack(level=0)
            return df_stats

        self.gridpoint_stats = sobek_stats(self.gridpoints_data)
        self.reachseg_stats = sobek_stats(self.reachseg_data)

    def load_bivas(self, BIVAS_database):
        logging.info('Loading BIVAS network')
        # Read BIVAS-network to networkx
        self.BIVAS = pyBIVAS(BIVAS_database)
        self.BIVAS.set_scenario()
        self.BIVAS_networkx = self.BIVAS.networkx_generate()
        self.BIVAS_arcs = self.BIVAS.network_arcs()
        self.BIVAS_nodes = self.BIVAS.network_nodes()

    def create_mapping(self, bivas_reaches, sobek_reaches):
        """
        Within a subset of reaches in BIVAS and SOBEK, compute the closest SOBEK points to each BIVAS-arc

        (reminder: SOBEK-results are only used for flow velocity (and water level/discharge). The grid is used for the
        depth)

        """
        logging.info('Creating koppeling')
        koppeling = []
        for reach, (node_start, node_end) in bivas_reaches.items():
            logging.info(f'Branch: {reach}')
            # Select BIVAS arcs  between two given arcs
            _, bivas_arcs = self.shortestpath(self.BIVAS_networkx, node_start, node_end, weight_col='Length__m',
                                              edge_name='ID')

            # Select SOBEK branches between two given nodes
            SOBEK_node_start, SOBEK_node_end = sobek_reaches[reach]

            _, sobek_branches = self.shortestpath(self.sobek_networkx, SOBEK_node_start, SOBEK_node_end,
                                                  weight_col='length',
                                                  edge_name='id')

            gridpoint_subset = self.gridpoints[self.gridpoints['branchname'].isin(sobek_branches)]
            gridpoint_subset_coordinates = gridpoint_subset[['x_coordinate', 'y_coordinate']].apply(tuple, axis=1)

            reachseg_subset = self.reachseg[self.reachseg['branchname'].isin(sobek_branches)]
            reachseg_subset_coordinates = reachseg_subset[['x_coordinate', 'y_coordinate']].apply(tuple, axis=1)

            # For each BIVAS-arc, get closest gridpoint and reachsegment
            for arc in bivas_arcs:
                BIVAS_arc = self.BIVAS_arcs.loc[arc]
                bivas_arcs_center = tuple(BIVAS_arc[['XM', 'YM']])

                closestgridpoint = self.closestpoint(node1=bivas_arcs_center,
                                                     list_node2=list(gridpoint_subset_coordinates),
                                                     labels_node2=gridpoint_subset_coordinates.index)
                closestreachseg = self.closestpoint(node1=bivas_arcs_center,
                                                    list_node2=list(reachseg_subset_coordinates),
                                                    labels_node2=reachseg_subset_coordinates.index)

                koppeling.append({
                    'Branch': arc,
                    'BIVAS_arc_name': BIVAS_arc['Name'],
                    'BIVAS_arc_id': arc,
                    'SOBEK_gridpoint': closestgridpoint,
                    'SOBEK_reachseg': closestreachseg,
                    'koppeling_gridpoint': 'LINESTRING ({b[0]:.0f} {b[1]:.0f}, {s[0]:.0f} {s[1]:.0f})'.format(
                        b=bivas_arcs_center,
                        s=gridpoint_subset_coordinates.loc[closestgridpoint]
                    ),
                    'koppeling_reachseg': 'LINESTRING ({b[0]:.0f} {b[1]:.0f}, {s[0]:.0f} {s[1]:.0f})'.format(
                        b=bivas_arcs_center,
                        s=reachseg_subset_coordinates.loc[closestreachseg]
                    )
                })

        self.mapping = pd.DataFrame(koppeling)
        return self.mapping

    def mapping_load(self, koppeling_csv):
        # TODO

        self.koppeling = pd.read_csv(koppeling_csv)

    def waterscenario_from_mapping(self):
        """
        Get SOBEK information (water level, discharge, flow velocity) based on the mapping

        """
        logging.info('Apply mapping')
        # Use koppeling

        waterscenario = {}

        for _, mapping_row in self.mapping.iterrows():
            arc = mapping_row['BIVAS_arc_id']
            gridpoint_id = mapping_row['SOBEK_gridpoint']
            reachseg_id = mapping_row['SOBEK_reachseg']

            water_level_cell = self.gridpoint_stats.loc[gridpoint_id, ('water_level', 'max')]
            # water_depth_cell = self.gridpoint_stats.loc[gridpoint_id, ('water_depth', 'max')]
            water_discharge_cell = self.reachseg_stats.loc[reachseg_id, ('water_discharge', 'mean')]
            water_velocity_cell = self.reachseg_stats.loc[reachseg_id, ('water_velocity', 'mean')]

            waterscenario[arc] = {
                'SeasonID': 1,
                'WaterLevel__m': water_level_cell,
                'RateOfFlow__m3_s': water_discharge_cell,
                'WaterSpeed__m_s': water_velocity_cell,
                'WaterDepth__m': np.nan, # water_depth_cell
            }

        self.waterscenario = pd.DataFrame(waterscenario).T
        self.waterscenario.index.name = 'ArcID'
        self.waterscenario['SeasonID'] = self.waterscenario['SeasonID'].astype(int)

        return self.waterscenario

    def load_waterdepthgrids(self, gridfiles, waterdepth):
        """
        Load water depth grids using the module waterdepthgrid.py

        """
        logging.info('Loading and processing waterlevel grids')

        assert gridfiles.keys() == waterdepth.keys(), 'Should have equal keys'
        datagrids = {}

        for g in waterdepth:
            WD = WaterdepthGrid()
            WD.load_grid(gridfiles[g])

            WD.add_waterdepth(waterdepth[g])

            WD.compute_width_depth_table()
            datagrids[g] = WD

        self.datagrids = datagrids

    def waterdepth_from_grids(self, sections, add_extra_columns=True, manual_points=None):
        """

        Compute the lowest waterdepth for a reach from the water depth grids

        :param sections: dict with structure {branchname: {point_start, point_end, channel_width, depth_at_fullwidth, sideslope, grid}}
        :param add_extra_columns: Boleaan. Add additional columns to waterscenario file
        :param manual_points: dict for adding manual points: {name: {BIVAS_arc, SOBEK_grid, bedlevel}
        :return:
        """
        logging.info('Getting waterdepth from grids')
        # Project results on the grid
        for label, section in sections.items():
            logging.info(f'  {label}')

            if 'sideslope' not in section:  # Old mode
                df_depth = self.datagrids[section['grid']].compute_depth_forwidth(
                    channelwidth=section['channel_width'],
                    min_channelwidth=section['min_channel_width'],
                    min_channel_depth=section['min_channel_depth']
                )
            else:
                df_depth = self.datagrids[section['grid']].compute_depth_forwidth(
                    channelwidth=section['channel_width'],
                    depth_at_fullwidth=section['depth_at_fullwidth'],
                    sideslope=section['sideslope']
                )

            # Select current reach in the grid
            ii_start = self.nearest_df_row(df_depth, section['point_start'])
            ii_end = self.nearest_df_row(df_depth, section['point_end'])
            reach = df_depth.loc[ii_start: ii_end].copy()
            self.reach[label] = reach

            # Get list of BIVAS arcs that need to be updated
            BIVAS_node_start = self.nearest_df_row(self.BIVAS_nodes, section['point_start'])
            BIVAS_node_end = self.nearest_df_row(self.BIVAS_nodes, section['point_end'])

            _, bivas_arcs = self.shortestpath(self.BIVAS_networkx, BIVAS_node_start, BIVAS_node_end,
                                              weight_col='Length__m', edge_name='ID')

            # First link all grid results to the nearest arc (based on centerpoint of arc)
            BIVAS_arcs_centroid = self.BIVAS_arcs.loc[bivas_arcs].copy()
            BIVAS_arcs_centroid['geometry'] = self.BIVAS_arcs.centroid

            for row in reach.itertuples():
                arc_id = self.nearest_df_row(BIVAS_arcs_centroid, row.geometry)
                reach.loc[row.Index, 'nearest_arc'] = arc_id

            # Second, loop through all arcs to find lowest depth (or closest depth)
            for arc in bivas_arcs:
                logging.debug(f'  {arc}')
                if arc in reach.nearest_arc.values:
                    min_depth = reach[reach.nearest_arc == arc]['depth'].min()
                    min_width = reach[reach.nearest_arc == arc]['width'].min()

                    reach_depth_nrow = reach[reach.nearest_arc == arc]['depth'].idxmin()
                    reach_width_nrow = reach[reach.nearest_arc == arc]['width'].idxmin()
                else:
                    nearest_row = self.nearest_df_row(reach, self.BIVAS_arcs.loc[arc, 'geometry'])
                    min_depth = reach.loc[nearest_row]['depth']
                    min_width = reach.loc[nearest_row]['width']

                    reach_depth_nrow = nearest_row
                    reach_width_nrow = nearest_row

                self.waterscenario.loc[arc, 'WaterDepth__m'] = min_depth

                # Add some extra columns for convenvenience. Should not be loaded to BIVAS
                if add_extra_columns:
                    self.waterscenario.loc[arc, 'Width'] = min_width

                    BIVAS_arc = self.BIVAS_arcs.loc[arc]

                    self.waterscenario.loc[arc, 'geometry'] = \
                        'LINESTRING ({b.X1:.0f} {b.Y1:.0f}, {b.X2:.0f} {b.Y2:.0f})'.format(
                            b=BIVAS_arc
                        )

                    self.waterscenario.loc[arc, 'nrow_depth'] = reach_depth_nrow
                    self.waterscenario.loc[arc, 'nrow_width'] = reach_width_nrow
                    self.waterscenario.loc[arc, 'mapping_to_grid'] = \
                        'LINESTRING ({b.XM:.0f} {b.YM:.0f}, {g.x:.0f} {g.y:.0f})'.format(
                            b=BIVAS_arc,
                            g=reach.loc[reach_depth_nrow]['geometry']
                        )

        if manual_points is not None:
            for name, params in manual_points.items():
                arc = params['BIVAS_arc']

                waterlevel = self.gridpoint_stats.loc[params['SOBEK_grid'], ('water_level', 'max')]
                waterdepth = waterlevel - params['bedlevel']

                BIVAS_arc = self.BIVAS_arcs.loc[arc]
                self.waterscenario.loc[arc, 'SeasonID'] = 1
                self.waterscenario.loc[arc, 'WaterDepth__m'] = waterdepth
                self.waterscenario.loc[arc, 'WaterLevel__m'] = waterlevel
                self.waterscenario.loc[arc, 'WaterSpeed__m_s'] = 0.0
                self.waterscenario.loc[arc, 'RateOfFlow__m3_s'] = 0.0

                self.waterscenario.loc[arc, 'geometry'] = \
                    'LINESTRING ({b.X1:.0f} {b.Y1:.0f}, {b.X2:.0f} {b.Y2:.0f})'.format(
                        b=BIVAS_arc
                    )
            self.waterscenario['SeasonID'] = self.waterscenario['SeasonID'].astype(int)
        return self.waterscenario

    @staticmethod
    def networkx_from_sobek3(SOBEK_network_file, exclude_sobek_reaches_from_network=None):
        """
        Build a networkx from a SOBEK3 network file
        Only includes nodes and branches not intermediate gridpoints

        :param SOBEK_network_file:
        :return: networkx
        """
        network_json = ini_to_json(SOBEK_network_file)
        network_pd = pd.DataFrame(network_json['data']['Branch'])
        length = [float(o.split(' ')[-1]) for o in network_pd['gridPointOffsets']]
        network_pd['length'] = length

        network_pd = network_pd[['id', 'fromNode', 'toNode', 'length']]

        network_pd.set_index('id', inplace=True)
        network_pd.drop(exclude_sobek_reaches_from_network, axis=0, inplace=True)

        sobek_networkx = nx.from_pandas_edgelist(
            network_pd.reset_index(), 'fromNode', 'toNode', edge_attr=True)

        # Is this necessary?
        # for n in network_json['data']['Node']:
        #     sobek_networkx.node[n['id']]['X'] = n['x']
        #     sobek_networkx.node[n['id']]['Y'] = n['y']

        return sobek_networkx

    @staticmethod
    def shortestpath(networkx, node_start, node_end, weight_col='Length__m', edge_name='ID'):
        # List of nodes for shortest path
        pathnodes = nx.dijkstra_path(
            networkx, node_start, node_end, weight=weight_col)

        # Name of edges between those nodes
        pathedges = []
        for i in range(len(pathnodes) - 1):
            pathedges.append(
                networkx[pathnodes[i]][pathnodes[i + 1]][edge_name])
        return pathnodes, pathedges

    @staticmethod
    def nearest_df_row(gdf, point):
        queried_geom, nearest_geom = nearest_points(point, gdf.unary_union)
        row_index = gdf.index[gdf['geometry'] == nearest_geom][0]
        return row_index

    @staticmethod
    def closestpoint(node1, list_node2, labels_node2=None):
        from scipy.spatial.distance import cdist
        id = cdist([node1], list_node2).argmin()

        if labels_node2 is None:
            return list_node2[id]
        else:
            return labels_node2[id]


'''
Created on Aug 23, 2018

@author: devalshah
'''
import datetime

from django.dispatch import receiver
from justclub.clubs.clubs_models import Club, BookClub, ClubComment, ClubMember,\
    ClubMemberJoinRequest
from django.db.models.signals import pre_save
from django.db.models.signals import post_save
from guardian.shortcuts import assign_perm

@receiver(pre_save, sender=Club)
def update_dates(sender, instance, **kwargs):

    if(instance.status == Club.IN_PROGRESS and (not instance.start_dt)):
        instance.start_dt = datetime.datetime.now()

    if((instance.status == Club.CANCELED or instance.status == Club.COMPLETED)
         and (not instance.close_dt)):
        instance.close_dt = datetime.datetime.now()

@receiver(post_save, sender=ClubMember)
def update_permission(sender, instance, **kwargs):

    user_added = instance.member
    club_added = instance.club

    assign_perm(Club.VIEW_COMMENT_PERM,user_added,club_added)

@receiver(post_save, sender=Club)
def add_owner_as_clubmember(sender, instance, **kwargs):
    ClubMember.objects.update_or_create(club=instance, member = instance.owner)

@receiver(post_save, sender=ClubMemberJoinRequest)
def add_approved_as_clubmember(sender, instance, **kwargs):
    if(instance.status == ClubMemberJoinRequest.APPROVED_YES):
        ClubMember.objects.update_or_create(club=instance.club, member = instance.member)


from setuptools import find_packages, setup
from package import Package

try:
    setup(
        version="1.0",
        author="Shreyansh Kumar",
        author_email="shreyansh.kumar2012@vitalum.ac.in",
        packages=find_packages(),
        include_package_data=True,
        cmdclass={
            "package": Package
        }
    )
except Exception as e:
    msg = e
    print("Error occurred:",msg)


import gc
import hashlib
import multiprocessing as mp
import os
import random
import shutil
import ssl
import sys
import tarfile
import time
import traceback
import warnings
from ctypes import c_bool, c_int
from functools import partial
from glob import glob
from io import BytesIO
from typing import Dict, Generator, List, Optional, Tuple
from urllib.parse import urljoin, urlparse
from uuid import uuid1, uuid4

import asks
import ftfy
import numpy as np
import pandas as pd
import pycld2 as cld2
import requests
import trio
import ujson
from PIL import Image, ImageFile, ImageOps, UnidentifiedImageError
from random_user_agent.params import OperatingSystem, SoftwareName
from random_user_agent.user_agent import UserAgent

import crawlingathome_client as cah

ImageFile.LOAD_TRUNCATED_IMAGES = True  # https://stackoverflow.com/a/47958486

warnings.filterwarnings('ignore')

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

class DownloadProgressInstrument(trio.abc.Instrument):
    def __init__(self, processing_count: mp.Value, finished_count: mp.Value, http_error_count: mp.Value, image_error_count: mp.Value, lock: mp.Lock):
        self._processing_count = processing_count
        self._finished_count = finished_count
        self._http_error_count = http_error_count
        self._image_error_count = image_error_count
        self._lock = lock

    def task_exited(self, task: trio.lowlevel.Task):
        if task.custom_sleep_data in [0, 1, 2]:
            with self._lock:
                self._processing_count.value -= 1
                self._finished_count.value += 1
            if task.custom_sleep_data == 1:
                with self._lock:
                    self._http_error_count.value += 1
            if task.custom_sleep_data == 2:
                with self._lock:
                    self._image_error_count.value += 1

class FileData:
    def __init__(self, filename: str):
        self._filename = filename
        self._line_to_position = [0]
        self._length = 0

        with open(self._filename, 'r') as f:
            while f.readline():
                self._line_to_position.append(f.tell())
                self._length += 1
        gc.collect()

    def __getitem__(self, line: int) -> int:
        return self._line_to_position[line]

    def __len__(self) -> int:
        return self._length

class InvalidImageError(Exception):
    pass

def chunk_using_generators(lst: list, n: int) -> Generator[list, None, None]:
    for i in range(0, len(lst), n):
        yield lst[i: i + n]

def log(ex: Exception, msg: str = ''):
    if msg:
        msg = f'- {msg} '
    with open('err.log', 'a') as f:
        f.write(f'{ex.__class__.__name__} {msg} -> {ex}\n')

def remove_bad_chars(text: str) -> str:
    return ''.join(c for c in text if c.isprintable())

def bloom_server_filter(hashes: bytes, bloom_ip: str, key: Optional[str] = 'clipped', loc: Optional[str] = 'deduplicate') -> List[str]:
    files = {
        'file': ('hash.txt', BytesIO(hashes)),
        'key': (None, key),
    }

    for _ in range(5):
        response = requests.post(
            f'http://{bloom_ip}:8000/{loc}/', files=files)
        if response.status_code != 200:
            cah.print('Bloom Server Error, retrying...')
            time.sleep(1)
            continue
        return response.content.decode('utf-8').split('\n')

    cah.print('Cannot contact bloom server, shutting down worker')
    sys.exit(1)

def parse_wat_worker(file_name: str, start: int, line_count: int, bloom_ip1: Optional[str] = '116.202.162.146', bloom_ip2: Optional[str] = '94.130.167.172') -> Tuple[List[Tuple], int, int, int]:
    blocked_formats = ('.svg', '.gif', '.webp',
                       'data:image', 'javascript:', 'mailto:')

    valid_data = []
    added_urls = set()

    with open(file_name, 'r') as content:
        content.seek(start)
        for _ in range(line_count):
            line = content.readline()

            if 'IMG@' not in line:
                continue

            line_str = line.strip()
            data = ujson.loads(line_str)

            linklist = data['Envelope']['Payload-Metadata']['HTTP-Response-Metadata'][
                'HTML-Metadata'
            ]['Links']

            base_url = os.path.dirname(
                data['Envelope']['WARC-Header-Metadata']['WARC-Target-URI']
            )

            license = '?'
            for e in linklist:
                if 'url' in e and 'creativecommons.org/licenses/' in e['url']:
                    license = e['url']

                if 'alt' not in e:
                    continue

                url = e['url']

                if any(bf in url for bf in blocked_formats):
                    continue

                url_domain = 'unknown'
                try:
                    url_domain = urlparse(url).netloc
                except:
                    continue

                alt_text = ftfy.fix_text(e['alt'].replace('\n', ' ')).strip()
                try:
                    _, _, details = cld2.detect(alt_text)
                except:
                    alt_text = remove_bad_chars(alt_text)
                    _, _, details = cld2.detect(alt_text)

                if details[0][1] != 'en':
                    continue

                if not url.startswith('http'):
                    url = urljoin(base_url, url)

                if url in added_urls:
                    continue

                hash = hashlib.md5(
                    (url + alt_text).encode('utf-8')).hexdigest()

                valid_data.append((url, alt_text, license, url_domain, hash))

        orig_len = len(valid_data)
        valid_data = [
            t for t in {tuple(i) for i in valid_data}
        ]
        shard_dups = orig_len - len(valid_data)

        valid_hashes = bloom_server_filter(
            '\n'.join(data[-1].strip() for data in valid_data).encode('utf-8'), bloom_ip1, key='clipped', loc='deduplicate')

        before_clipped = len(valid_data)
        valid_data = [data
                      for data in valid_data if data[-1].strip() in valid_hashes]
        clipped = before_clipped - len(valid_data)

        valid_hashes = bloom_server_filter(
            '\n'.join(data[-1].strip() for data in valid_data).encode('utf-8'), bloom_ip2, key='parsed', loc='deduplicate')

        before_parsed_check = len(valid_data)
        valid_data = [data
                      for data in valid_data if data[-1].strip() in valid_hashes]
        parsed_check = before_parsed_check - len(valid_data)

        return valid_data, clipped, shard_dups, parsed_check

def parse_wat(file_name: str, shard: int, fd: Optional[FileData] = None, bloom_ip1: Optional[str] = '116.202.162.146', bloom_ip2: Optional[str] = '94.130.167.172') -> Tuple[List[Tuple], int, int, int]:
    if fd is None:
        fd = FileData(file_name)

    if shard == 0:
        start_line = 0
    elif shard == 1:
        start_line = len(fd)//2

    line_count = len(fd)//2

    return parse_wat_worker(file_name, fd[start_line], line_count, bloom_ip1=bloom_ip1, bloom_ip2=bloom_ip2)

def torchvision_resize(img: Image.Image) -> Image.Image:
    size = 224
    w, h = img.size

    short, long = (w, h) if w <= h else (h, w)
    if short == size:
        return img

    new_short, new_long = size, int(size * long / short)

    new_w, new_h = (new_short, new_long) if w <= h else (new_long, new_short)
    return img.resize((new_w, new_h), Image.BICUBIC)

def torchvision_parse_fill(img: Image.Image) -> Dict[str, Tuple[float, ...]]:
    fill = 0
    name = 'fill'

    num_bands = len(img.getbands())
    if num_bands > 1:
        fill = tuple([fill] * num_bands)

    return {name: fill}

def torchvision_pad(img: Image.Image, padding: Tuple[int]) -> Image.Image:
    opts = torchvision_parse_fill(img)
    if img.mode == "P":
        palette = img.getpalette()
        image = ImageOps.expand(img, border=padding, **opts)
        image.putpalette(palette)
        return image

    return ImageOps.expand(img, border=padding, **opts)

def torchvision_crop(img: Image.Image,
                     top: int,
                     left: int,
                     height: int,
                     width: int,
                     ) -> Image.Image:

    return img.crop((left, top, left + width, top + height))

def torchvision_centercrop(img: Image.Image) -> Image.Image:
    size = (224, 224)
    output_size = size

    image_width, image_height = img.size
    crop_height, crop_width = output_size

    if crop_width > image_width or crop_height > image_height:
        padding_ltrb = (
            (crop_width - image_width) // 2 if crop_width > image_width else 0,
            (crop_height - image_height) // 2 if crop_height > image_height else 0,
            (crop_width - image_width + 1) // 2 if crop_width > image_width else 0,
            (crop_height - image_height + 1) // 2 if crop_height > image_height else 0,
        )
        img = torchvision_pad(img, padding_ltrb)
        image_width, image_height = img.size
        if crop_width == image_width and crop_height == image_height:
            return img

    crop_top = round((image_height - crop_height) // 2)
    crop_left = round((image_width - crop_width) // 2)
    return torchvision_crop(img, crop_top, crop_left, crop_height, crop_width)

def clip_preprocessing(img: Image.Image) -> Image.Image:
    img = torchvision_resize(img)
    img = torchvision_centercrop(img)
    img = img.convert('RGB')
    return img

def process_img_content(response, alt_text: str, license: str, sample_id: int) -> Tuple[str, str, str, str, int, int, str]:
    img_output_folder = 'save/images/'

    try:
        if len(response.content) < 5000:
            raise InvalidImageError('image is too small')

        img_data = BytesIO(response.content)
        with Image.open(img_data) as im:
            width, height = im.size

            if width * height > 89478484:  # Might be a DOS decompression bomb
                raise InvalidImageError(
                    'image is too large (dos decompression)')

            im_format = im.format.lower()
            if im_format not in set(['jpeg', 'jpg', 'png', 'webp']):
                raise InvalidImageError('invalid image format')

            out_fname = f'{img_output_folder}{sample_id}.{im_format}'

            im = clip_preprocessing(im)
            im.save(out_fname)

        return (str(sample_id), out_fname, response.url, alt_text, width, height, license)
    except (AttributeError, KeyError, UnidentifiedImageError) as ex:
        raise InvalidImageError(
            f'Unidentified Image Error: {ex.__class__.__name__}')

async def request_image(datas: list, start_sampleid: int, user_agent: str, processing_count: mp.Value, lock: mp.Lock, connections: int = 165):
    tmp_data = []

    limit = trio.CapacityLimiter(2000)
    session = asks.Session(connections=connections, ssl_context=ssl_ctx)

    session.headers = {
        'User-Agent': user_agent,
        'Accept-Language': 'en-US,en;q=0.5',
        'Accept-Encoding': 'gzip, deflate',
        'Referer': 'https://www.google.com',
        'DNT': '1',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    }

    async def _request(data_index, sample_id):
        with lock:
            processing_count.value += 1

        task = trio.lowlevel.current_task()

        url, alt_text, license, _, _ = datas[data_index]

        try:
            async with limit:
                response = await session.get(url, timeout=10, connection_timeout=20)
            tmp_data.append(process_img_content(
                response, alt_text, license, sample_id
            ))
            task.custom_sleep_data = 0
        except InvalidImageError as ex:
            log(ex, msg='Image Error')
            task.custom_sleep_data = 1
        except Exception as ex:
            log(ex, msg='HTTP Error')
            task.custom_sleep_data = 2
        return

    async with trio.open_nursery() as n:
        for id, index in enumerate(range(len(datas)), start=start_sampleid):
            async with limit:
                n.start_soon(_request, index, id)

    with open(f'.tmp/dl-{uuid1()}.json', 'w') as f:
        ujson.dump(tmp_data, f)
    gc.collect()

def dl_wat_worker(data: list, start_sample_id: int, user_agent: str, processing_count: mp.Value, finished_count: mp.Value, http_error_count: mp.Value, image_error_count: mp.Value, lock: mp.Lock) -> None:
    trio.run(request_image, data, start_sample_id, user_agent, processing_count, lock, instruments=[
             DownloadProgressInstrument(processing_count, finished_count, http_error_count, image_error_count, lock)])

def dl_progress(len_data: int, processing_count: mp.Value, finished_count: mp.Value, http_error_count: mp.Value, image_error_count: mp.Value, update_tqdm: mp.Value, lock: mp.Lock, isnotebook: Optional[bool] = False) -> None:
    if isnotebook:
        from tqdm import tqdm
    else:
        from tqdm import tqdm

    progress_bar = tqdm(total=len_data, unit='links')
    while True:
        with lock:
            if not update_tqdm.value:
                break
            progress_bar.desc = f'Processing {processing_count.value} links, {http_error_count.value} http errors, {image_error_count.value} image errors'
            progress_bar.update(finished_count.value - progress_bar.n)
        time.sleep(1)
    progress_bar.close()

def dl_wat(valid_data: list, first_sample_id: str, user_agent_rotator: UserAgent, isnotebook: Optional[bool] = False) -> pd.DataFrame:
    # Download every image available
    processed_samples = []
    n_processes = mp.cpu_count()

    manager = mp.Manager()
    update_tqdm = manager.Value(c_bool, True)

    processing_count = manager.Value(c_int, 0)
    finished_count = manager.Value(c_int, 0)
    http_error_count = manager.Value(c_int, 0)
    image_error_count = manager.Value(c_int, 0)

    lock = manager.Lock()

    t = mp.Process(target=dl_progress, args=(
        len(valid_data), processing_count, finished_count, http_error_count, image_error_count, update_tqdm, lock, isnotebook))
    t.start()

    if n_processes == 1:
        dl_wat_worker(valid_data, first_sample_id, user_agent_rotator.get_random_user_agent(), processing_count,
                      finished_count, http_error_count, image_error_count, update_tqdm, lock)
    else:
        chunk_size = len(valid_data) // n_processes + 1
        worker = partial(dl_wat_worker, processing_count=processing_count,
                         finished_count=finished_count, http_error_count=http_error_count, image_error_count=image_error_count, lock=lock)

        with mp.Pool(n_processes) as pool:
            pool.starmap(worker, [(data, first_sample_id + i * chunk_size, user_agent_rotator.get_random_user_agent())
                                  for (i, data) in enumerate(chunk_using_generators(valid_data, chunk_size))])

    time.sleep(1)
    with lock:
        update_tqdm.value = False

    t.join()
    t.close()

    for tmpf in glob('.tmp/dl-*.json'):
        with open(tmpf, 'r') as f:
            processed_samples.extend(ujson.load(f))
    return pd.DataFrame(
        processed_samples,
        columns=['SAMPLE_ID', 'PATH', 'URL',
                 'TEXT', 'HEIGHT', 'WIDTH', 'LICENSE'],
    )

def upload(source: str, target: str):
    with tarfile.open(f'{source}.tar.gz', 'w:gz') as tar:
        tar.add(source, arcname=os.path.basename(source))

    result = 1
    while result:
        result = os.system(
            f'rsync -av {source}.tar.gz {target} > /dev/null 2>&1')
    if os.path.exists(f'{source}.tar.gz'):
        os.remove(f'{source}.tar.gz')

def safe_client_function(client_function, *args, **kwargs):
    while True:
        try:
            return client_function(*args, **kwargs)
        except cah.WorkerTimedOutError:
            cah.print('worker timed out, retrying')

def check_current_worker_version() -> None:
    while True:
        try:
            r = requests.get(
                'https://raw.githubusercontent.com/ARKseal/crawlingathome-worker/master/version.txt')
            r.raise_for_status()
            break
        except requests.HTTPError:
            pass

    version = r.content.decode('utf-8')
    current_version = os.getenv('CAHVERSION', default=None)

    if current_version is None:
        cah.print("Worker doesn't use the correct docker env")
        exit(-1)

    if version == current_version:
        return

    cah.print("Worker is outdated, please repull the docker image")
    sys.exit(1)

def main(name: str, url: str, debug: bool, isnotebook: bool, isdocker: bool) -> None:
    if isdocker:
        check_current_worker_version()

    client = cah.FullWATClient(
        url=url, nickname=name
    )

    if not hasattr(client, 'bye'):
        client.bye = client._c.bye

    output_folder = './save/'
    img_output_folder = output_folder + 'images/'

    uid = ''

    user_agent_rotator = UserAgent(software_names=[SoftwareName.CHROME.value], operating_systems=[
                                   OperatingSystem.LINUX.value], limit=2000)

    while safe_client_function(client.jobCount) > 0:
        if isdocker:
            check_current_worker_version()

        try:
            completing_arg = {}
            start = time.time()

            if not safe_client_function(client.isAlive):
                safe_client_function(client.recreate)

            safe_client_function(client.newJob)
            safe_client_function(client.downloadWat)

            fd = FileData('shard.wat')

            for shard_of_chunk in range(2):
                shutil.rmtree(output_folder, ignore_errors=True)
                shutil.rmtree(uid, ignore_errors=True)
                shutil.rmtree('.tmp', ignore_errors=True)

                os.mkdir(output_folder)
                os.mkdir(img_output_folder)
                os.mkdir('.tmp')

                first_sample_id = np.int64(
                    client.shards[shard_of_chunk][1]['start_id'])
                last_sample_id = np.int64(
                    client.shards[shard_of_chunk][1]['end_id'])

                out_fname = \
                    f'FIRST_SAMPLE_ID_IN_SHARD_{first_sample_id}_LAST_SAMPLE_ID_IN_SHARD_{last_sample_id}_{shard_of_chunk}'
                cah.print(
                    f'shard identification: {out_fname}'
                )  # in case test fails, we need to remove bad data

                safe_client_function(client.log, 'Processing shard')
                start_processing = time.time()

                parsed_data, cliped, shard_dups, parsed_check = parse_wat(
                    'shard.wat', shard_of_chunk, fd=fd)

                num_links = len(parsed_data)

                parsed_df = pd.DataFrame(parsed_data, columns=[
                                         'URL', 'TEXT', 'LICENSE', 'DOMAIN', 'HASH'])
                parsed_df = parsed_df.drop_duplicates(subset=['URL'])
                parsed_df.to_csv(
                    f'{output_folder}{out_fname}_parsed.csv', index=False, sep='|')

                random.shuffle(parsed_data)

                end_processing = time.time()
                cah.print(
                    f'Processed shard in {(end_processing-start_processing):.2f} seconds'
                    '\n\t'
                    f'cliped found: {cliped}, shard dups found: {shard_dups}, parsed check found: {parsed_check}')

                safe_client_function(client.log, 'Downloading images')
                start_dl = time.time()

                dlparse_df = dl_wat(
                    parsed_data, first_sample_id, user_agent_rotator, isnotebook)
                dlparse_df.to_csv(
                    f'{output_folder}{out_fname}.csv', index=False, sep='|')

                end_dl = time.time()
                cah.print(
                    f'Downloaded {len(dlparse_df)} images out of {num_links} links in {(end_dl - start_dl):.2f} seconds')
                cah.print(
                    f'Download efficiency: {(len(dlparse_df) / (end_dl - start_dl)):.2f} img/sec OR {(num_links / (end_dl - start_dl)):.2f} links/sec')

                safe_client_function(client.log, 'Uploading Temporary Job')

                uid = uuid4().hex
                shutil.move('save', uid)

                completing_arg[str(
                    client.shards[shard_of_chunk][0])] = f'rsync {uid}'
                upload(uid, client.upload_address)

            safe_client_function(client.completeJob, completing_arg)
            end = time.time()
            cah.print(
                f'job completed in {(end - start):.2f} seconds')
            cah.print(
                f'job efficiency {(len(dlparse_df) / (end - start)):.2f} pairs/sec')
        except KeyboardInterrupt:
            cah.print('stopping crawler')
            break
        except Exception as ex:
            cah.print(f'ERROR: {ex}')
            if debug:
                traceback.print_exc()
            try:
                if safe_client_function(client.isAlive):
                    safe_client_function(client.log, 'Error, restarting job')
            except:
                cah.print("Couldn't log to client")
        finally:
            if debug:
                break
    shutil.rmtree(uid, ignore_errors=True)
    shutil.rmtree(output_folder, ignore_errors=True)
    shutil.rmtree(img_output_folder, ignore_errors=True)
    shutil.rmtree('.tmp', ignore_errors=True)
    try:
        if safe_client_function(client.isAlive):
            safe_client_function(client.bye)
    except:
        pass


import cv2
import numpy as np

pc = 1
bx = 150
by = 150
bh = 550
bw = 370
c1 = 0  # pedestrian
c2 = 1  # car
c3 = 0  # traffic light

y = [pc, bx, by, bh, bw, c1, c2, c3]

print(y)

# Python program to explain cv2.rectangle() method

# Reading an image in default mode
image = cv2.imread('mer.png')

# Window name in which image is displayed
window_name = 'Image'

# Start coordinate, here (5, 5)
# represents the top left corner of rectangle
start_point = (bx, by)

# Ending coordinate, here (220, 220)
# represents the bottom right corner of rectangle
end_point = (bh, bw)

# Blue color in BGR
color = (255, 0, 0)

# Line thickness of 2 px
thickness = 2

# Using cv2.rectangle() method
# Draw a rectangle with blue line borders of thickness of 2 px
image = cv2.rectangle(image, start_point, end_point, color, thickness)

# Displaying the image
cv2.imshow(window_name, image)
cv2.waitKey(0)
cv2.destroyAllWindows()


import Global
import pygame
#constant
class Local(object):
	@staticmethod
	def init():
		Global.load_const(Local, "Combo")
		Local.font=pygame.font.SysFont(Local.font_name,
				Local.text_size, True, False)
Local.init()

class Combo(object):
	@classmethod
	def start(cls):
		cls.combo=0

	@classmethod
	def end(cls):
		pass

	@classmethod
	def draw(cls):
		if cls.combo == 0 :
			return
		if cls.combo > 1 :
			string="%d combos!" % cls.combo
		else :
			string="%d combo!" % cls.combo
		text=Local.font.render(string, True, Local.text_color)
		Global.screen.blit(text, Local.left_up_point)

	@classmethod
	def tick(cls):
		pass

	@classmethod
	def reset_combo(cls):
		cls.combo=0

	@classmethod
	def add_combo(cls, combo):
		cls.combo+=combo

	@classmethod
	def get_bonus(cls):
		return cls.combo

start=Combo.start
end=Combo.end
draw=Combo.draw
tick=Combo.tick
add_combo=Combo.add_combo
reset_combo=Combo.reset_combo
get_bonus=Combo.get_bonus


from root import ROOT_DIR
from cffi import FFI

ffi = FFI()
ffi.cdef("int times2(int x);")

C = ffi.dlopen(ROOT_DIR + "/target/release/liballegory.dylib")

if __name__ == "__main__":
    print(C.times2(9))