text:
###############################################################################
# WaterTAP Copyright (c) 2021, The Regents of the University of California,
# through Lawrence Berkeley National Laboratory, Oak Ridge National
# Laboratory, National Renewable Energy Laboratory, and National Energy
# Technology Laboratory (subject to receipt of any required approvals from
# the U.S. Dept. of Energy). All rights reserved.
#
# Please see the files COPYRIGHT.md and LICENSE.md for full copyright and license
# information, respectively. These files are also available online at the URL
# "https://github.com/watertap-org/watertap/"
#
###############################################################################
import os
from pyomo.environ import (
    ConcreteModel,
    Set,
    Expression,
    value,
    TransformationFactory,
    units as pyunits,
    assert_optimal_termination,
)
from pyomo.network import Arc, SequentialDecomposition
from pyomo.util.check_units import assert_units_consistent

from idaes.core import FlowsheetBlock
from idaes.core.solvers import get_solver
from idaes.models.unit_models import Product
import idaes.core.util.scaling as iscale
from idaes.core import UnitModelCostingBlock

from watertap.core.util.initialization import assert_degrees_of_freedom

from watertap.core.wt_database import Database
import watertap.core.zero_order_properties as prop_ZO
from watertap.unit_models.zero_order import (
    FeedZO,
    PumpElectricityZO,
    NanofiltrationZO,
)
from watertap.core.zero_order_costing import ZeroOrderCosting

def main():
    m = build()

    set_operating_conditions(m)
    assert_degrees_of_freedom(m, 0)
    assert_units_consistent(m)

    initialize_system(m)

    results = solve(m)
    assert_optimal_termination(results)

    display_results(m)

    add_costing(m)
    m.fs.costing.initialize()
    assert_degrees_of_freedom(m, 0)

    results = solve(m)
    assert_optimal_termination(results)
    display_costing(m)
    return m, results

def build():
    # flowsheet set up
    m = ConcreteModel()
    m.db = Database()

    m.fs = FlowsheetBlock(default={"dynamic": False})
    m.fs.prop = prop_ZO.WaterParameterBlock(default={"solute_list": ["dye", "tds"]})

    # unit model
    m.fs.feed = FeedZO(default={"property_package": m.fs.prop})

    m.fs.P1 = PumpElectricityZO(
        default={
            "property_package": m.fs.prop,
            "database": m.db,
            "process_subtype": "default",
        }
    )

    m.fs.nanofiltration = NanofiltrationZO(
        default={
            "property_package": m.fs.prop,
            "database": m.db,
            "process_subtype": "rHGO_dye_rejection",
        }
    )

    m.fs.permeate1 = Product(default={"property_package": m.fs.prop})
    m.fs.retentate1 = Product(default={"property_package": m.fs.prop})

    # connections
    m.fs.s01 = Arc(source=m.fs.feed.outlet, destination=m.fs.P1.inlet)
    m.fs.s02 = Arc(source=m.fs.P1.outlet, destination=m.fs.nanofiltration.inlet)
    m.fs.s03 = Arc(source=m.fs.nanofiltration.treated, destination=m.fs.permeate1.inlet)
    m.fs.s04 = Arc(
        source=m.fs.nanofiltration.byproduct, destination=m.fs.retentate1.inlet
    )

    TransformationFactory("network.expand_arcs").apply_to(m)

    # scaling
    iscale.calculate_scaling_factors(m)

    return m

def set_operating_conditions(m):
    # feed
    flow_vol = 120 / 3600 * pyunits.m**3 / pyunits.s
    conc_mass_dye = 2.5 * pyunits.kg / pyunits.m**3
    conc_mass_tds = 50.0 * pyunits.kg / pyunits.m**3

    m.fs.feed.flow_vol[0].fix(flow_vol)
    m.fs.feed.conc_mass_comp[0, "dye"].fix(conc_mass_dye)
    m.fs.feed.conc_mass_comp[0, "tds"].fix(conc_mass_tds)
    solve(m.fs.feed)

    # nanofiltration
    m.fs.nanofiltration.load_parameters_from_database(use_default_removal=True)

    # pump
    m.fs.P1.load_parameters_from_database(use_default_removal=True)
    m.fs.P1.applied_pressure.fix(m.fs.nanofiltration.applied_pressure.get_values()[0])
    m.fs.P1.lift_height.unfix()

def initialize_system(m):
    seq = SequentialDecomposition()
    seq.options.tear_set = []
    seq.options.iterLim = 1
    seq.run(m, lambda u: u.initialize())

def solve(blk, solver=None, tee=False, check_termination=True):
    if solver is None:
        solver = get_solver()
    results = solver.solve(blk, tee=tee)
    if check_termination:
        assert_optimal_termination(results)
    return results

def add_costing(m):
    source_file = os.path.join(
        os.path.dirname(os.path.abspath(__file__)),
        "dye_desalination_global_costing.yaml",
    )

    m.fs.costing = ZeroOrderCosting(default={"case_study_definition": source_file})

    costing_kwargs = {"default": {"flowsheet_costing_block": m.fs.costing}}
    m.fs.nanofiltration.costing = UnitModelCostingBlock(**costing_kwargs)
    m.fs.P1.costing = UnitModelCostingBlock(**costing_kwargs)
    m.fs.costing.cost_process()

    m.fs.costing.dye_recovery_mass = Expression(
        expr=m.fs.costing.utilization_factor
        * pyunits.convert(
            m.fs.retentate1.flow_mass_comp[0, "dye"],
            to_units=pyunits.kg / m.fs.costing.base_period,
        ),
        doc="Mass of dye in retentate per year",
    )

    m.fs.costing.value_dye_recovery_mass = Expression(
        expr=(
            m.fs.costing.utilization_factor
            * m.fs.costing.dye_mass_cost
            * pyunits.convert(
                m.fs.retentate1.flow_mass_comp[0, "dye"],
                to_units=pyunits.kg / m.fs.costing.base_period,
            )
        ),
        doc="Dollar value of dye in the retentate priced by mass",
    )

    m.fs.costing.annual_disposal_cost = Expression(
        expr=(
            m.fs.costing.utilization_factor
            * (
                m.fs.costing.waste_disposal_cost
                * pyunits.convert(
                    m.fs.permeate1.properties[0].flow_vol,
                    to_units=pyunits.m**3 / m.fs.costing.base_period,
                )
            )
        ),
        doc="Annual cost of waste disposal",
    )

    m.fs.costing.value_dye_recovery_flow = Expression(
        expr=(
            m.fs.costing.utilization_factor
            * (
                m.fs.costing.dye_retentate_cost
                * pyunits.convert(
                    m.fs.retentate1.properties[0].flow_vol,
                    to_units=pyunits.m**3 / m.fs.costing.base_period,
                )
            )
        ),
        doc="Annual value of dye-retentate recovery",
    )
    m.fs.costing.LCOW_dye_recovered = Expression(
        expr=(
            m.fs.costing.total_capital_cost * m.fs.costing.capital_recovery_factor
            + m.fs.costing.total_operating_cost
            + m.fs.costing.utilization_factor
            * (
                m.fs.costing.waste_disposal_cost
                * pyunits.convert(
                    m.fs.permeate1.properties[0].flow_vol,
                    to_units=pyunits.m**3 / m.fs.costing.base_period,
                )
                - m.fs.costing.dye_retentate_cost
                * pyunits.convert(
                    m.fs.retentate1.properties[0].flow_vol,
                    to_units=pyunits.m**3 / m.fs.costing.base_period,
                )
            )
        )
        / (
            m.fs.costing.utilization_factor
            * pyunits.convert(
                m.fs.feed.flow_vol[0],
                to_units=pyunits.m**3 / m.fs.costing.base_period,
            )
        ),
        doc="Levelized cost of water treated with dye recovery through NF",
    )

    m.fs.costing.LCOT_dye_mass = Expression(
        expr=(
            m.fs.costing.total_capital_cost * m.fs.costing.capital_recovery_factor
            + m.fs.costing.total_operating_cost
            + m.fs.costing.utilization_factor
            * m.fs.costing.waste_disposal_cost
            * pyunits.convert(
                m.fs.permeate1.properties[0].flow_vol,
                to_units=pyunits.m**3 / m.fs.costing.base_period,
            )
            - m.fs.costing.value_dye_recovery_mass
        )
        / m.fs.costing.dye_recovery_mass,
        doc="Levelized cost of treatment on a basis of dye recovered",
    )

def display_results(m):
    unit_list = ["P1", "nanofiltration"]
    for u in unit_list:
        m.fs.component(u).report()

def display_costing(m):
    print("\nUnit Capital Costs")
    for u in m.fs.costing._registered_unit_costing:
        print(
            u.name,
            " :   ",
            value(pyunits.convert(u.capital_cost, to_units=pyunits.USD_2018)),
            "$",
        )

    print("")
    total_capital_cost = value(
        pyunits.convert(m.fs.costing.total_capital_cost, to_units=pyunits.MUSD_2018)
    )
    print(f"Total Capital Costs: {total_capital_cost:.4f} M$\n")

    total_operating_cost = value(
        pyunits.convert(
            m.fs.costing.total_operating_cost, to_units=pyunits.MUSD_2018 / pyunits.year
        )
    )
    print(f"Total Operating Costs: {total_operating_cost:.4f} M$/year\n")

    waste_disposal_cost = value(
        pyunits.convert(
            m.fs.costing.annual_disposal_cost, to_units=pyunits.MUSD_2018 / pyunits.year
        )
    )
    print(f"Total Waste Disposal Costs: {waste_disposal_cost:.4f} M$/year\n")

    dye_recovery_cost_flow = value(
        -1
        * pyunits.convert(
            m.fs.costing.value_dye_recovery_flow,
            to_units=pyunits.MUSD_2018 / pyunits.year,
        )
    )
    print(
        f"Total Dye Recovery Costs (flow basis): {dye_recovery_cost_flow:.4f} M$/year"
    )

    dye_recovery_cost_mass = value(
        -1
        * pyunits.convert(
            m.fs.costing.value_dye_recovery_mass,
            to_units=pyunits.MUSD_2018 / pyunits.year,
        )
    )
    print(
        f"Total Dye Recovery Costs (mass basis): {dye_recovery_cost_mass:.4f} M$/year\n"
    )

    levelized_cost_water = value(
        pyunits.convert(
            m.fs.costing.LCOW_dye_recovered, to_units=pyunits.USD_2018 / pyunits.m**3
        )
    )
    print(f"Levelized Cost of Water (LCOW): {levelized_cost_water:.2f} $/m3")

    levelized_cost_treatment = value(
        pyunits.convert(
            m.fs.costing.LCOT_dye_mass, to_units=pyunits.USD_2018 / pyunits.kg
        )
    )
    print(
        f"Levelized Cost of Treatment - Dye (LCOT): {levelized_cost_treatment:.2f} $/kg Dye"
    )

if __name__ == "__main__":
    m, results = main()

reason: from pyomo


text:
"""
get_paraview_slice_axis.py: calculate the axis used in slicing paraview moddels.
"""
import numpy as np
from sympy import Plane, Point3D

def sind(x):
    """
    sind: sin(x) in degree.
    """
    return np.sin(np.deg2rad(x))

def cosd(x):
    """
    cosd: cos(x) in degree.
    """
    return np.cos(np.deg2rad(x))

def convert_ll2xyz(latitude, longitude):
    """
    convert_ll2xyz: convert the latitude and longitude (degree) to x,t,z (normalized)
    """
    theta = 90 - latitude
    phi = longitude
    x = sind(theta) * cosd(phi)
    y = sind(theta) * sind(phi)
    z = cosd(theta)
    return x, y, z

if __name__ == "__main__":
    import click

    @click.command()
    @click.option('--p1', required=True, type=str, help="point1 lon1,lat1")
    @click.option('--p2', required=True, type=str, help="point2 lon2,lat2")
    def main(p1, p2):
        """
        The main function.
        """
        lon1, lat1 = map(float, p1.split(","))
        lon2, lat2 = map(float, p2.split(","))
        x1, y1, z1 = convert_ll2xyz(lat1, lon1)
        x2, y2, z2 = convert_ll2xyz(lat2, lon2)
        slice_plane = Plane(Point3D(x1, y1, z1), Point3D(
            x2, y2, z2), Point3D(0, 0, 0))
        axis = slice_plane.normal_vector
        # print out the result
        print(f"x: {float(axis[0])}")
        print(f"y: {float(axis[1])}")
        print(f"z: {float(axis[2])}")
        print(f"point1 {x1} {y1} {z1}")
        print(f"point2 {x2} {y2} {z2}")

    main()  # pylint: disable=no-value-for-parameter

ext: py

max_stars_repo_name: ziyixi/seisflow

max_issues_repo_name: ziyixi/seisflow

max_forks_repo_name: ziyixi/seisflow

reason: import numpy,from sympy


text:
from __future__ import division
from nose.tools import assert_almost_equal, assert_equal, assert_true
import numpy as np
import qubit_dst.dst_povm_sampling as samp
from itertools import combinations, product

def check_state_norms(samples):
    diffs = np.sum(np.abs(samples)**2, axis=0) - np.ones(samples.shape[1])
    assert_almost_equal(max(np.abs(diffs)), 0, 7)

def check_probabilities(probs, measurements):
    positive = [prob >= 0 for prob in probs]
    assert_true(all(positive))
    assert_almost_equal(sum(probs), 1, 7)
    assert_equal(len(probs), len(measurements))

def check_states_unique(states):

    # Verify that the states generated are unique
    for state1, state2 in combinations(states, 2):
        diff = state1 - state2
        diff_mag = diff.conj().dot(diff)
        assert_true(diff_mag > 1e-7)

def test_sampling():

    phis = [0, 1, 2]

    for phi in phis:
        dists = [samp.DSTDistribution(phi), samp.DSTxyzDistribution(phi),
                 samp.DSTxyzDistribution(phi, [1/3, 1/3, 1/3]),
                 samp.DSTxyzDistribution(phi, [4, 2, 5])]

        for dist in dists:
            check_probabilities(dist.probs, dist.measurements)
            samples = dist.sample(2**8)
            check_state_norms(samples)

def test_uniqueness():
    # Make sure all the states that are being sampled from are unique (only for
    # values of phi that will cause them to be unique), only taking one outcome
    # from the x-states when the ancilla outcome is negative, because the
    # projectors are the same for both system outcomes
    phis = [1, 2]
    pm = [1, -1]
    for phi in phis:
        x_states = [samp.x_state(ancout, sysout, phi) for ancout, sysout in
                    product(pm, pm)]
        y_states = [samp.y_state(ancout, sysout, phi) for ancout, sysout in
                    product(pm, pm)]
        z_states = [samp.z_state(ancout, phi) for ancout in pm]
        check_states_unique(x_states[:-1] + y_states + z_states)

reason: import numpy


text:
#!/usr/bin/env python
# coding: utf-8
import os
from itertools import combinations
import psutil
import multiprocessing as mp

import networkx as nx
import cell_lattices as cx

import numpy as np
import scipy.special as sp
from tqdm import tqdm
import pandas as pd

# Options for saving output(s)
save     = True
save_dir = os.path.realpath("/home/pbhamidi/git/cell-lattices/data")
# fmt      = "png"
# dpi      = 300

# Get coordinates and adjacency for a periodic square lattice
rows = cols = 5
X = cx.hex_grid(rows, cols)
A = cx.make_adjacency_periodic(rows, cols)

# Make master cell graph
G = nx.from_numpy_matrix(A)

# Get total number of cells and a subset
n = A.shape[0]
n_sub = n // 2

# Get total number of permutations this tissue can undergo
ncomb = int(sp.comb(n, n_sub))
n_cpu = psutil.cpu_count(logical=False)

# Create progress bar (approximate, since multiprocess is lazy)
pbar = tqdm(total=ncomb // n_cpu)

# Define functions
def get_graph_hashes(idx):
    subG        = G.subgraph(idx)
    graph_hash  = nx.algorithms.weisfeiler_lehman_graph_hash(subG)
    # binary_hash = cx.indices_to_bin_hash(idx, n)
    # return graph_hash, binary_hash

    # Update progress
    pbar.update(1)

    return cx.indices_to_binstr(idx, n), graph_hash

def n_connected_components(idx):
    """Computes number of connected components given cell indices"""
    subG = G.subgraph(idx)
    nc = nx.number_connected_components(subG)
    nc_inv = nx.number_connected_components(
        G.subgraph(
            [i for i in range(n) if i not in set(idx)]
        )
    )
    return nc + nc_inv

# Parallelize calculation of tissue topology (# coneccted components)
if __name__ == '__main__':

    print("Getting all cell type combinations")

#     print("Making Boolean combinations")
    # Get all cell type combinations as indices (for computing # components)
    combs_idx = [i for i in combinations(np.arange(n), n_sub)]

#     # Get combinations as Boolean data (for UMAP)
#     combs_bool = np.zeros((ncomb, n), dtype=bool)
#     for i, idx in enumerate(tqdm(combs_idx)):
#         combs_bool[i, idx] = True

#     print("Making string combinations")

#     # Get combinations as strings
#     combs_str = ["".join([str(int(c)) for c in _comb]) for _comb in combs_bool]

    print("Assembling worker pool")

    # Get worker pool
    pool = mp.Pool(psutil.cpu_count(logical=False))

    print("Computing hashes for graph")

    # Perform parallel computation
    # results = pool.imap_unordered(get_graph_hashes, combs_idx)
    results = pool.imap(get_graph_hashes, combs_idx, chunksize=5)

    # Unravel results
#    n_comp, n_edges = map(list, zip(*result_list))
    bin_strings, graph_hashes = map(list, zip(*results))

    print("Closing worker pool")

    pool.close()
    pool.join()

pbar.close()

# Combine into dataframe
df = pd.DataFrame(dict(
    binary_str = bin_strings,
    graph_hash = graph_hashes,
#    combination = combs_str,
#    n_components = n_comp,
))
df = df.sort_values("binary_str")

# Get smallest binary hash for each graph hash (~50x compression)
df = df.groupby("graph_hash").agg(min).reset_index()

if save:
    print("Saving!")
    data_fname = "cellgraph_hashes.csv"
    data_fpath = os.path.join(save_dir, data_fname)
    df.to_csv(data_fpath, index=False)
    print(f"Saved to {data_fpath}")

max_stars_repo_name: pranav-bhamidipati/cell-lattices

max_issues_repo_name: pranav-bhamidipati/cell-lattices

max_forks_repo_name: pranav-bhamidipati/cell-lattices

reason: import numpy,import scipy,import networkx


text:
from PyQt5.QtCore import QThread
from PyQt5 import QtWidgets
from pymodaq.daq_viewer.utility_classes import DAQ_Viewer_base
import numpy as np
from easydict import EasyDict as edict
from pymodaq.daq_utils.daq_utils import ThreadCommand, getLineInfo, gauss1D, linspace_step, DataFromPlugins, Axis
from pymodaq.daq_viewer.utility_classes import comon_parameters
from pymodaq.daq_utils.parameter.utils import iter_children

class DAQ_1DViewer_Mock_spectro(DAQ_Viewer_base):
    """
    1D viewer plugin simulating a photon spectrometer controlling the laser source, the exposure and the calibration axis
    Produces one data signal with two hypergaussians whose parameters are fully controllable or
    produces 2 data signals, each beeing a fully controllable hypergaussian
    Features specific methods that should be present for spectrometer features using Pymodaq_spectrometer package:

    get_laser_wl: emit the value of the currently selected laser (if available)
    set_laser_wl: try to set the selected laser wavelength (if applicable), emit the value of the new one

    get_spectro_wl: emit the value of the central frequency
    set_spectro_wl: set the newly central frequency

    get_exposure_ms: emit the value of the exposure time in ms
    set_exposure_ms: set the new exposure time in ms

    """
    params = comon_parameters + [
        {'title': 'Rolling?:', 'name': 'rolling', 'type': 'int', 'value': 0, 'min': 0},
        {'title': 'Multi Channels?:', 'name': 'multi', 'type': 'bool', 'value': False,
         'tip': 'if true, plugin produces multiple curves (2) otherwise produces one curve with 2 peaks'},
        {'title': 'Mock1:', 'name': 'Mock1', 'type': 'group', 'children': [
            {'title': 'Amp:', 'name': 'Amp', 'type': 'int', 'value': 20, 'default': 20},
            {'title': 'x0:', 'name': 'x0', 'type': 'float', 'value': 500, 'default': 500},
            {'title': 'dx:', 'name': 'dx', 'type': 'float', 'value': 0.3, 'default': 20},
            {'title': 'n:', 'name': 'n', 'type': 'int', 'value': 1, 'default': 1, 'min': 1},
            {'title': 'noise:', 'name': 'amp_noise', 'type': 'float', 'value': 0.1, 'default': 0.1, 'min': 0}
        ]},
        {'title': 'Mock2:', 'name': 'Mock2', 'type': 'group', 'children': [
            {'title': 'Amp?:', 'name': 'Amp', 'type': 'int', 'value': 10},
            {'title': 'x0:', 'name': 'x0', 'type': 'float', 'value': 520},
            {'title': 'dx:', 'name': 'dx', 'type': 'float', 'value': 0.7},
            {'title': 'n:', 'name': 'n', 'type': 'int', 'value': 2, 'default': 2, 'min': 1},
            {'title': 'noise:', 'name': 'amp_noise', 'type': 'float', 'value': 0.1, 'default': 0.1, 'min': 0}, ]},

        {'title': 'xaxis:', 'name': 'x_axis', 'type': 'group', 'children': [
            {'title': 'Npts:', 'name': 'Npts', 'type': 'int', 'value': 513, },
            {'title': 'x0:', 'name': 'x0', 'type': 'float', 'value': 515, },
            {'title': 'dx:', 'name': 'dx', 'type': 'float', 'value': 0.1, },
        ]},
        {'title': 'Laser Wavelength', 'name': 'laser_wl', 'type': 'list', 'value': 515, 'values': [405, 515, 632.8]},
        {'title': 'Exposure (ms)', 'name': 'exposure_ms', 'type': 'int', 'value': 100, 'default': 100}
    ]
    hardware_averaging = False

    def __init__(self, parent=None,
                 params_state=None):  # init_params is a list of tuple where each tuple contains info on a 1D channel (Ntps,amplitude, width, position and noise)
        super().__init__(parent, params_state)

        self.x_axis = Axis(label='photon wavelength', units='nm')
        self.ind_data = 0

    def commit_settings(self, param):
        """
            Setting the mock data

            ============== ========= =================
            **Parameters**  **Type**  **Description**
            *param*         none      not used
            ============== ========= =================

            See Also
            --------
            set_Mock_data
        """
        if param.name() in iter_children(self.settings.child(('x_axis')), []):
            if param.name() == 'x0':
                self.get_spectro_wl()

            self.set_x_axis()
            self.emit_x_axis()

        else:
            self.set_Mock_data()

    def set_Mock_data(self):
        """
            For each parameter of the settings tree :
                * compute linspace numpy distribution with local parameters values
                * shift right the current data of ind_data position
                * add computed results to the data_mock list

            Returns
            -------
            list
                The computed data_mock list.
        """
        ind = -1
        self.data_mock = []
        data = np.zeros(self.x_axis['data'].shape)
        exposure_factor = self.settings.child('exposure_ms').value() / \
            self.settings.child('exposure_ms').opts['default']

        for param in self.settings.children():  #
            if 'Mock' in param.name():
                ind += 1

                data_tmp = exposure_factor * \
                    param.child('Amp').value() * gauss1D(self.x_axis['data'], param.child('x0').value(),
                                                           param.child('dx').value(),
                                                           param.child('n').value())
                if ind == 0:
                    data_tmp = data_tmp * np.sin(self.x_axis['data'] / 4) ** 2
                data_tmp += param.child('amp_noise').value() * np.random.rand((len(self.x_axis['data'])))
                data_tmp = self.settings.child('exposure_ms').value() /\
                    1000 * np.roll(data_tmp, self.ind_data * self.settings.child('rolling').value())
                if self.settings.child('multi').value():
                    self.data_mock.append(data_tmp)
                else:
                    data += data_tmp
        if not self.settings.child('multi').value():
            self.data_mock.append(data)
        self.ind_data += 1
        return self.data_mock

    def set_x_axis(self):
        Npts = self.settings.child('x_axis', 'Npts').value()
        x0 = self.settings.child('x_axis', 'x0').value()
        dx = self.settings.child('x_axis', 'dx').value()
        self.x_axis['data'] = linspace_step(x0 - (Npts - 1) * dx / 2, x0 + (Npts - 1) * dx / 2, dx)
        self.emit_x_axis()

    def set_spectro_wl(self, spectro_wl):
        """
        Method related to spectrometer module, mandatory
        Parameters
        ----------
        spectro_wl: set the "grating" position

        """
        self.settings.child('x_axis', 'x0').setValue(spectro_wl)
        QtWidgets.QApplication.processEvents()
        self.emit_status(ThreadCommand('spectro_wl', [self.settings.child('x_axis', 'x0').value()]))
        self.set_x_axis()

    def get_spectro_wl(self):
        """
        Method related to spectrometer module, mandatory
        Get the "grating" position
        """
        self.emit_status(ThreadCommand('spectro_wl', [self.settings.child('x_axis', 'x0').value()]))

    def get_laser_wl(self):
        """
        Method related to spectrometer module, mandatory if laser can be set
        Get the "laser" position
        """
        self.emit_status(ThreadCommand('laser_wl', [self.settings.child(("laser_wl")).value()]))

    def set_laser_wl(self, laser_wl):
        """
        Method related to spectrometer module, mandatory if laser can be set
        Set the "laser" position
        """
        self.settings.child("laser_wl").setValue(laser_wl)
        QtWidgets.QApplication.processEvents()
        self.emit_status(ThreadCommand('laser_wl', [self.settings.child(('laser_wl')).value()]))

    def get_exposure_ms(self):
        self.emit_status(ThreadCommand('exposure_ms', [self.settings.child(('exposure_ms')).value()]))

    def set_exposure_ms(self, exposure):
        self.settings.child("exposure_ms").setValue(exposure)
        QtWidgets.QApplication.processEvents()
        self.emit_status(ThreadCommand('exposure_ms', [self.settings.child(('exposure_ms')).value()]))

    def ini_detector(self, controller=None):
        """
            Initialisation procedure of the detector updating the status dictionnary.

            See Also
            --------
            set_Mock_data, daq_utils.ThreadCommand
        """
        self.status.update(edict(initialized=False, info="", x_axis=None, y_axis=None, controller=None))
        try:

            if self.settings.child('controller_status').value() == "Slave":
                if controller is None:
                    raise Exception('no controller has been defined externally while this detector is a slave one')
                else:
                    self.controller = controller
            else:
                self.controller = "Mock controller"
            self.set_x_axis()
            self.set_Mock_data()

            # initialize viewers with the future type of data
            self.data_grabed_signal_temp.emit([DataFromPlugins(name='Mock1', data=self.data_mock, dim='Data1D',
                                                               x_axis=self.x_axis, labels=['Mock1', 'label2']), ])

            self.status.initialized = True
            self.status.controller = self.controller
            self.status.x_axis = self.x_axis
            return self.status

        except Exception as e:
            self.emit_status(ThreadCommand('Update_Status', [getLineInfo() + str(e), 'log']))
            self.status.info = getLineInfo() + str(e)
            self.status.initialized = False
            return self.status

    def close(self):
        """
            Not implemented.
        """
        pass

    def grab_data(self, Naverage=1, **kwargs):
        """
            | Start new acquisition

            For each integer step of naverage range:
                * set mock data
                * wait 100 ms
                * update the data_tot array

            | Send the data_grabed_signal once done.

            =============== ======== ===============================================
            **Parameters**  **Type**  **Description**
            *Naverage*      int       Number of spectrum to average.
                                      Specify the threshold of the mean calculation
            =============== ======== ===============================================

            See Also
            --------
            set_Mock_data
        """
        Naverage = 1
        data_tot = self.set_Mock_data()
        for ind in range(Naverage - 1):
            data_tmp = self.set_Mock_data()
            QThread.msleep(self.settings.child('exposure_ms').value())

            for ind, data in enumerate(data_tmp):
                data_tot[ind] += data

        data_tot = [data / Naverage for data in data_tot]
        QThread.msleep(self.settings.child('exposure_ms').value())
        self.data_grabed_signal.emit([DataFromPlugins(name='Mock1', data=data_tot, dim='Data1D')])

    def stop(self):
        """
            not implemented.
        """

        return ""


text:
import os
from PIL import Image
import numpy as np
from pathlib import Path
from sklearn.preprocessing import StandardScaler, OneHotEncoder, MinMaxScaler
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from tqdm.auto import trange
from sklearn.base import TransformerMixin
import logging
from abc import ABC, abstractmethod
import time

class DatasetLoader(ABC):
    class Reshaper(TransformerMixin):
        def transform(self, X):
            return X.reshape(X.shape[0], -1)

        def fit_transform(self, X, y=None, **fit_params):
            return self.transform(X)

    @abstractmethod
    def __iter__(self):
        pass

    @abstractmethod
    def __next__(self):
        pass

    @abstractmethod
    def __len__(self):
        pass

    @abstractmethod
    def _getindex(self, index):
        pass

    def __getitem__(self, key):
        if isinstance(key, slice):
            start, stop, step = key.indices(len(self))
            return [self[i] for i in range(start, stop, step)]
        elif isinstance(key, int):
            return self._getindex(key)
        else:
            raise TypeError('Invalid argument type: {}'.format(type(key)))

    @abstractmethod
    def batches(self, *args, **kwargs):
        pass

    @abstractmethod
    def validation_set(self, batch_size):
        pass

    @property
    @abstractmethod
    def test_split_index(self):
        pass

    @property
    @abstractmethod
    def val_split_index(self):
        pass

    @abstractmethod
    def test_set(self, batch_size):
        pass

    def update_progress(self, end=False, **kwargs):
        try:
            if end:
                self.prange.close()
                epoch = kwargs.pop("epoch")
                print(f"Epoch #{epoch} ended with " + "\t".join([f"{k}: {float(v):.3f}" for k, v in kwargs.items()]))
            else:
                if "epoch" in kwargs.keys():
                    self.prange.set_description(f"Epoch #{kwargs['epoch'] + 1}")
                    kwargs.pop("epoch")
                self.prange.set_postfix(kwargs)
                self.prange.refresh()
                time.sleep(0.01)

        except Exception as e:
            # logging.debug(e)
            print(e)
            pass

class UTKDatasetLoader(DatasetLoader):
    class Decorators:
        @classmethod
        def target_column(cls, gen_func):
            def wrapper(self, *args, **kwargs):
                for X, y in gen_func(self, *args, **kwargs):
                    if self.target_column is None:
                        pass  # Do Nothing
                    else:
                        y = y[:, self.target_column][..., np.newaxis]
                        y = self.encoders[self.target_column].transform(y)
                    yield X, y
            return wrapper

        @classmethod
        def floater(cls, gen_func):
            def wrapper(self, *args, **kwargs):
                for X, y in gen_func(self, *args, **kwargs):
                    yield X.astype(np.float64), y
            return wrapper

    def __init__(
        self,
        path,
        batch_size=128,
        validation_split=0.1,
        test_split=0.1,
        dim_reducer_size=128,
        target_column=None,
    ):
        self.files = [
            os.path.join(root, f)
            for root, __, files in os.walk(path)
            for f in files if len(f.split("_")) == 4
        ]
        self.shuffle = True
        if self.shuffle:
            np.random.shuffle(self.files)

        self.validation_split = validation_split
        self.test_split = test_split
        self.batch_size = batch_size
        self._target_column = target_column
        self.dim_reducer_size = dim_reducer_size
        self.dim_reducer = Pipeline(
            [
                ("reshaper", DatasetLoader.Reshaper()),
                ("scaler", StandardScaler()),
                ("pca", PCA(n_components=dim_reducer_size)),
            ]
        )
        self._initalize_label_encoders()

    def _initalize_label_encoders(self):
        ages, genders, races = list(
            zip(*[map(int, Path(f).stem.split("_")[:-1]) for f in self.files]))
        ages, genders, races = map(
            lambda x: np.array(x)[..., np.newaxis],
            (ages, genders, races))
        self.encoders = [MinMaxScaler(), OneHotEncoder(sparse=False), OneHotEncoder(sparse=False)]
        self.encoders[0].fit_transform(ages)
        self.encoders[1].fit_transform(genders)
        self.encoders[2].fit_transform(races)

    @property
    def target_column(self):
        return self._target_column

    @target_column.setter
    def target_column(self, c):
        self._target_column = c

    def fit_dim_reducer(self, n_samples, shuffle=True):
        logging.info(f"fitting dimension reducer(shuffle={shuffle}).")
        if shuffle:
            sample_files = np.random.choice(self.files, n_samples)
        else:
            sample_files = self.files[:n_samples]
        X = []
        for fpath in sample_files:
            img = np.array(Image.open(fpath))
            X.append(img)
        X = np.array(X)
        self.dim_reducer.fit(X)
        logging.info(f"fitting dimension reducer complete.")

    def __iter__(self):
        self.n = 0
        return self

    def __len__(self):
        return len(self.files)

    def __next__(self):
        prev_n = self.n
        self.n += 1
        return self[prev_n]

    def _getindex(self, index):
        fpath = self.files[index]
        fname = Path(fpath).name
        age, gender, race, _ = fname.split("_")
        img = np.array(Image.open(fpath))
        return img, int(age), int(gender), int(race)

    @property
    def val_split_index(self):
        return int(self.test_split_index * (1 - self.validation_split))

    @property
    def test_split_index(self):
        return int(len(self) * (1 - self.test_split))

    @Decorators.floater
    @Decorators.target_column
    def batches(self, progress=True):
        logging.info("loading training batches.")
        train_indices = np.arange(self.val_split_index)
        if self.shuffle:
            np.random.shuffle(train_indices)
        self.prange = (
            trange(0, len(train_indices), self.batch_size, leave=False)
            if progress
            else range(0, len(train_indices), self.batch_size)
        )

        for start_idx in self.prange:
            batch_indices = list(train_indices[
                start_idx: start_idx + self.batch_size
            ])
            X_batch, y_batch = [], []
            for index in batch_indices:
                img, age, gender, race = self[int(index)]
                X_batch.append(img)
                y_batch.append((age, gender, race))
            X_batch = np.array(X_batch)
            X_batch = self.dim_reducer.transform(X_batch)
            y_batch = np.array(y_batch)
            yield X_batch, y_batch

    def validation_set(self, batch_size=30):
        val_size = self.test_split_index - self.val_split_index
        X_val = np.zeros((val_size, self.dim_reducer_size))
        y_val = np.zeros((val_size, 3))
        logging.info(
            f"validation set loading started.(batch_size={batch_size}"
            f", validation size={val_size})")
        # lazily reducing dimensions of validation set
        for start_idx in range(self.val_split_index, self.test_split_index, batch_size):
            curr_idx = start_idx - self.val_split_index
            X_cache, ages, genders, races = list(
                zip(*self[start_idx:min(start_idx+batch_size, self.test_split_index)]))
            X_cache = np.array(X_cache)
            X_cache = self.dim_reducer.transform(X_cache)
            y_cache = np.array((ages, genders, races)).T
            y_val[curr_idx:curr_idx+batch_size, :] = y_cache
            X_val[curr_idx:curr_idx+batch_size, :] = X_cache
            logging.info(
                "validation set loading: "
                f"step #{1 + curr_idx//batch_size} "
                f"out of total {1 + val_size//batch_size} steps.")
        y_val = y_val[..., self.target_column][..., np.newaxis]
        y_val = self.encoders[self.target_column].transform(y_val)
        return X_val.astype(np.float64), y_val

    def test_set(self, batch_size=30):
        test_size = len(self) - self.test_split_index
        X_test = np.zeros((test_size, self.dim_reducer_size))
        y_test = np.zeros((test_size, 3))
        logging.info(
            f"test set loading started.(batch_size={batch_size})")
        # lazily reducing dimensions of validation set
        for start_idx in range(self.test_split_index, len(self), batch_size):
            curr_idx = start_idx - self.test_split_index
            X_cache, ages, genders, races = list(
                zip(*self[start_idx:start_idx+batch_size]))
            X_cache = np.array(X_cache)
            X_cache = self.dim_reducer.transform(X_cache)
            y_cache = np.array((ages, genders, races)).T
            y_test[curr_idx:curr_idx+batch_size, :] = y_cache
            X_test[curr_idx:curr_idx+batch_size, :] = X_cache
            logging.info(
                "test set loading: "
                f"step #{1 + curr_idx//batch_size} "
                f"out of total {1 + test_size//batch_size} steps.")
        y_test = y_test[..., self.target_column][..., np.newaxis]
        y_test = self.encoders[self.target_column].transform(y_test)
        return X_test.astype(np.float64), y_test

    @property
    def dimensions(self):
        dim = [self.dim_reducer_size]
        if self.target_column == 0:
            dim.append(1)
        else:
            dim.append(len(self.encoders[self.target_column].get_feature_names()))
        return dim


text:
import scipy as sp
import matplotlib.pyplot as plt

class GraphPlotter:
    NUMBER_OF_POINTS = 100

    def __init__(self):
        self.functions = []

    def add_function(self, f, desc):
        """Adds the function to graph plotter's list of functions"""
        self.functions.append((f, desc))

    def plot_together(self, x_from, x_to):
        """Display a single window with all functions plotted on one plot"""
        for i in range(len(self.functions)):
            self.__plot(i, x_from, x_to)
        plt.show()

    def plot_one_by_one(self, x_from, x_to):
        """Plots each function on a separate plot in separate window"""
        for i in range(len(self.functions)):
            self.__plot(i, x_from, x_to)
            plt.show()

    def plot_on_grid(self, x_from, x_to):
        """Plots the functions on separate plots but in one window"""
        k = len(self.functions)
        q = int(sp.sqrt(k))
        if k == q**2:
            rows = q
            cols = q
        elif (q + 1)*q >= k:
            rows = q + 1
            cols = q
        else:
            rows = cols = q + 1

        for i in range(len(self.functions)):
            plt.subplot(rows, cols, i + 1)
            self.__plot(i, x_from, x_to)
        plt.show()

    def __plot(self, index, x_from, x_to):
        xx = sp.linspace(x_from, x_to, self.NUMBER_OF_POINTS)
        yy = self.functions[index][0](xx)
        plt.plot(xx, yy, label=self.functions[index][1])
        plt.xlabel('x')
        plt.ylabel('y')
        plt.legend(loc=4)
        plt.axhline(y=0, color='black')
        plt.axvline(x=0, color='black')
        plt.grid()

def cube(x):
    return x**3

def square(x):
    return x**2

def lin(x):
    return 3*x + 5

def lin2(x):
    return -2*x

def lin3(x):
    return 0.1*x + 10

def mylog(x):
    return 1/(sp.log(abs(x) - 5))

g = GraphPlotter()
g.add_function(square, 'square')
g.add_function(lin, '3x + 5')
g.add_function(cube, 'cube')
g.add_function(lin2, '-2x')
g.add_function(lin3, '0.1x + 10')
g.add_function(mylog, '1/log(|x| - 5)')
g.plot_together(-5, 5)
g.plot_one_by_one(-10, 10)
g.plot_on_grid(-10, 10)

max_stars_repo_name: silverfield/pythonsessions

max_issues_repo_name: silverfield/pythonsessions

max_forks_repo_name: silverfield/pythonsessions

reason: import scipy


text:
import random
from collections import OrderedDict
from common.operation import Operation
from workflowgen.baseaction import BaseAction
from workflowgen.linkaction import LinkAction
import pandasql
import math
import numpy as np

class SelectionAction(BaseAction):

    def get_states(self):

        if len(LinkAction.LINKS) == 0:
            return

        rand_link = self.pick(list(LinkAction.LINKS))
        rand_link_src = rand_link[0]
        nodes_dict = self.vizgraph.get_nodes_dict()
        src_viz = nodes_dict["viz_" + str(rand_link_src)]
        computed_filter = src_viz.get_computed_filter()
        df = self.df
        sql_statement = "SELECT * FROM df "
        if len(computed_filter) > 0:
            sql_statement += "WHERE " + computed_filter

        df_result = pandasql.sqldf(sql_statement, locals())

        if df_result.empty:
            return None

        filter_per_dim = []

        for bin_dim in range(len(src_viz.binning)):
            filters = []
            dim = src_viz.binning[bin_dim]["dimension"]
            field = list(filter(lambda x: x["field"] == dim, self.sample_json["tables"]["fact"]["fields"]))[0]
            if field["type"] == "quantitative":
                bin_width = float(src_viz.binning[bin_dim]["width"])
                min_val = df_result[dim].min()
                max_val = df_result[dim].max()

                min_index = math.floor(min_val / bin_width)
                max_index = math.floor(max_val / bin_width)
                num_bins = 0
                if np.random.rand() < 0.4:
                    num_bins = 1
                else:
                    num_bins = random.randint(1, max_index-min_index) if max_index > min_index else 1
                selected_bins = np.random.choice(np.arange(min_index, max_index + 1), size=num_bins, replace=False)

                for selected_bin in selected_bins:
                    range_min = selected_bin * bin_width
                    range_max = (selected_bin + 1) * bin_width
                    filt = "(%s >= %s and %s < %s)" % (dim, '{:.1f}'.format(range_min), dim, '{:.1f}'.format(range_max))
                    filters.append(filt)
            else:
                all_bins = df_result[dim].unique().tolist()
                num_bins = random.randint(1, len(all_bins))
                selected_bins = np.random.choice(all_bins, size=num_bins, replace=False)
                for selected_bin in list(selected_bins):
                    filt =  "(%s = '%s')" % (dim, selected_bin)
                    filters.append(filt)
            filter_per_dim.append(" or ".join(filters))
        filter_per_dim = ["(%s)" % f for f in filter_per_dim]

        return Operation(OrderedDict({"name": ("viz_%s" % rand_link_src), "selection": " and ".join(filter_per_dim)}))

max_stars_repo_name: zeyuanxy/IDEBench-public

max_issues_repo_name: zeyuanxy/IDEBench-public

max_forks_repo_name: zeyuanxy/IDEBench-public


text:
# /**
#  * Parse the YtDataLink protocol
#  * author: chenliang @ Youtu Lab, Tencent
#  * @example
#     const SerialPort = require('serialport')
#     const YtMsgParser = require('@api/YtMsgParser')
#     const port = new SerialPort('/dev/ttyUSB0')
#     const parser = port.pipe(new YtMsgParser())
#     parser.on('data', console.log)
#  */
from .YtMsg_pb2 import *
from .FilePart_pb2 import *
from . import YtFaceAlignment
import numpy as np
import struct
import time

class YtDataLink:
    class YtVisionSeedResultType:
        YT_RESULT_RECT = 0
        YT_RESULT_ARRAY = 1
        YT_RESULT_CLASSIFICATION = 2
        YT_RESULT_STRING = 3
        YT_RESULT_POINTS = 4
        YT_RESULT_VARUINT32 = 5

    class AttributeDict(dict):
        def __getattr__(self, attr):
            return self[attr]
        def __setattr__(self, attr, value):
            self[attr] = value

    class YtVisionSeedResultTypeConfidence:
        def __init__ (self):
            self.type = 'YtVisionSeedResultTypeConfidence'
            self.conf = 0
        def parse (self, bufObj, count):
            conf = YtDataLink.unpackUInt16(bufObj)
            self.conf = conf / 65535
    class YtVisionSeedResultTypeClassification(YtVisionSeedResultTypeConfidence):
        def __init__ (self):
            super().__init__()
            self.type = 'YtVisionSeedResultTypeClassification'
            self.cls = 0
        def parse (self, bufObj, count):
            super().parse(bufObj, count)
            self.cls = YtDataLink.unpackUInt16(bufObj)
    class YtVisionSeedResultTypeRect(YtVisionSeedResultTypeClassification):
        def __init__ (self):
            super().__init__()
            self.type = 'YtVisionSeedResultTypeRect'
            self.x = 0
            self.y = 0
            self.w = 0
            self.h = 0
        def parse (self, bufObj, count):
            super().parse(bufObj, count)
            self.x = YtDataLink.unpackInt16(bufObj)
            self.y = YtDataLink.unpackInt16(bufObj)
            self.w = YtDataLink.unpackInt16(bufObj)
            self.h = YtDataLink.unpackInt16(bufObj)

    class YtVisionSeedResultTypeArray:
        def __init__ (self):
            self.type = 'YtVisionSeedResultTypeArray'
            self.array = []

        def parse (self, bufObj, count):
            cur_data = bufObj.p[bufObj.i : bufObj.i + count]
            dt = np.dtype(np.float32)
            dt = dt.newbyteorder('<')
            self.array = np.frombuffer(cur_data, dtype=dt).tolist()
            bufObj.i += count

    class YtVisionSeedResultTypeString(YtVisionSeedResultTypeConfidence):
        def __init__ (self):
            super().__init__()
            self.type = 'YtVisionSeedResultTypeString'
            self.str = ''

        def parse (self, bufObj, count):
            if (count < 3): #at least 1 for the null terminator and 2 for confidence
                return

            super().parse(bufObj, count)
            cur_data = bufObj.p[bufObj.i : bufObj.i + count - 1 - 2]
            bufObj.i += count - 2 # - 1 null terminator
            self.str = str(cur_data, encoding = "utf-8")

    class YtVisionSeedResultTypePoints:
        def __init__ (self):
            self.type = 'YtVisionSeedResultTypePoints'
            self.points = []

        def parse (self, bufObj, count):
            for i in range(count // 4):
                self.points.append(YtDataLink.AttributeDict({
                    'x': YtDataLink.unpackInt16(bufObj),
                    'y': YtDataLink.unpackInt16(bufObj),
                }))

    class YtVisionSeedModel:
        FACE_DETECTION = 1
        FACE_LANDMARK = 2
        FACE_POSE = 3
        FACE_QUALITY = 4
        FACE_RECOGNITION = 6
        DETECTION_TRACE = 8

    class DataV2:
        def __init__(self, buf):
            # self.array = bytearray(0)
            self.data = {}
            bufObj = YtDataLink.AttributeDict({'p': buf, 'i': 0})
            count = bufObj.p[bufObj.i]
            bufObj.i += 1
            for i in range(count):
                cur_path_len = bufObj.p[bufObj.i]
                bufObj.i += 1
                cur_path = bufObj.p[bufObj.i : bufObj.i + cur_path_len]
                bufObj.i += cur_path_len
                type = bufObj.p[bufObj.i]
                bufObj.i += 1
                cur_data_len = YtDataLink.unpackVarUInt32(bufObj)

                result = None
                if (type == YtDataLink.YtVisionSeedResultType.YT_RESULT_CLASSIFICATION and cur_data_len == 4):
                    result = YtDataLink.YtVisionSeedResultTypeClassification()
                elif (type == YtDataLink.YtVisionSeedResultType.YT_RESULT_RECT and cur_data_len == 12):
                    result = YtDataLink.YtVisionSeedResultTypeRect()
                elif (type == YtDataLink.YtVisionSeedResultType.YT_RESULT_ARRAY):
                    result = YtDataLink.YtVisionSeedResultTypeArray()
                elif (type == YtDataLink.YtVisionSeedResultType.YT_RESULT_STRING):
                    result = YtDataLink.YtVisionSeedResultTypeString()
                elif (type == YtDataLink.YtVisionSeedResultType.YT_RESULT_POINTS):
                    result = YtDataLink.YtVisionSeedResultTypePoints()
                elif (type == YtDataLink.YtVisionSeedResultType.YT_RESULT_VARUINT32):
                    result = YtDataLink.unpackVarUInt32(bufObj)

                cur_path = ','.join([str(int.from_bytes(cur_path[x:x+1], byteorder='little', signed=False)) for x in range(len(cur_path))])
                if not (result is None):
                    if hasattr(result, 'parse'):
                        result.parse(bufObj, cur_data_len)

                    if (type == YtDataLink.YtVisionSeedResultType.YT_RESULT_POINTS and len(result.points) == 90):
                        result.faceShape = YtFaceAlignment.YtFaceShape(result)

                    self.data[cur_path] = result
                else:
                    cur_data = bufObj.p[bufObj.i : bufObj.i + cur_data_len]
                    self.data[cur_path] = cur_data
                    bufObj.i += cur_data_len
            # // console.log('[result]', self.data)

        def getResult(self, path):
            path = ','.join(str(x) for x in path)
            if path in self.data:
                return self.data[path]
            return None

    class YtDataLinkStatus:
        YT_DL_IDLE = 0
        YT_DL_LEN1_PENDING = 1
        YT_DL_LEN2_PENDING = 2
        YT_DL_LEN3_PENDING = 3
        YT_DL_LEN_CRC_H = 4
        YT_DL_LEN_CRC_L = 5
        YT_DL_DATA = 6
        YT_DL_CRC_H = 7
        YT_DL_CRC_L = 8
    SOF = 0x10
    TRANS = 0x11
    ytMsgSize = 2097152
    fileBlobSize = 131072
    ccittTable = [
        0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50A5, 0x60C6, 0x70E7,
        0x8108, 0x9129, 0xA14A, 0xB16B, 0xC18C, 0xD1AD, 0xE1CE, 0xF1EF,
        0x1231, 0x0210, 0x3273, 0x2252, 0x52B5, 0x4294, 0x72F7, 0x62D6,
        0x9339, 0x8318, 0xB37B, 0xA35A, 0xD3BD, 0xC39C, 0xF3FF, 0xE3DE,
        0x2462, 0x3443, 0x0420, 0x1401, 0x64E6, 0x74C7, 0x44A4, 0x5485,
        0xA56A, 0xB54B, 0x8528, 0x9509, 0xE5EE, 0xF5CF, 0xC5AC, 0xD58D,
        0x3653, 0x2672, 0x1611, 0x0630, 0x76D7, 0x66F6, 0x5695, 0x46B4,
        0xB75B, 0xA77A, 0x9719, 0x8738, 0xF7DF, 0xE7FE, 0xD79D, 0xC7BC,
        0x48C4, 0x58E5, 0x6886, 0x78A7, 0x0840, 0x1861, 0x2802, 0x3823,
        0xC9CC, 0xD9ED, 0xE98E, 0xF9AF, 0x8948, 0x9969, 0xA90A, 0xB92B,
        0x5AF5, 0x4AD4, 0x7AB7, 0x6A96, 0x1A71, 0x0A50, 0x3A33, 0x2A12,
        0xDBFD, 0xCBDC, 0xFBBF, 0xEB9E, 0x9B79, 0x8B58, 0xBB3B, 0xAB1A,
        0x6CA6, 0x7C87, 0x4CE4, 0x5CC5, 0x2C22, 0x3C03, 0x0C60, 0x1C41,
        0xEDAE, 0xFD8F, 0xCDEC, 0xDDCD, 0xAD2A, 0xBD0B, 0x8D68, 0x9D49,
        0x7E97, 0x6EB6, 0x5ED5, 0x4EF4, 0x3E13, 0x2E32, 0x1E51, 0x0E70,
        0xFF9F, 0xEFBE, 0xDFDD, 0xCFFC, 0xBF1B, 0xAF3A, 0x9F59, 0x8F78,
        0x9188, 0x81A9, 0xB1CA, 0xA1EB, 0xD10C, 0xC12D, 0xF14E, 0xE16F,
        0x1080, 0x00A1, 0x30C2, 0x20E3, 0x5004, 0x4025, 0x7046, 0x6067,
        0x83B9, 0x9398, 0xA3FB, 0xB3DA, 0xC33D, 0xD31C, 0xE37F, 0xF35E,
        0x02B1, 0x1290, 0x22F3, 0x32D2, 0x4235, 0x5214, 0x6277, 0x7256,
        0xB5EA, 0xA5CB, 0x95A8, 0x8589, 0xF56E, 0xE54F, 0xD52C, 0xC50D,
        0x34E2, 0x24C3, 0x14A0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
        0xA7DB, 0xB7FA, 0x8799, 0x97B8, 0xE75F, 0xF77E, 0xC71D, 0xD73C,
        0x26D3, 0x36F2, 0x0691, 0x16B0, 0x6657, 0x7676, 0x4615, 0x5634,
        0xD94C, 0xC96D, 0xF90E, 0xE92F, 0x99C8, 0x89E9, 0xB98A, 0xA9AB,
        0x5844, 0x4865, 0x7806, 0x6827, 0x18C0, 0x08E1, 0x3882, 0x28A3,
        0xCB7D, 0xDB5C, 0xEB3F, 0xFB1E, 0x8BF9, 0x9BD8, 0xABBB, 0xBB9A,
        0x4A75, 0x5A54, 0x6A37, 0x7A16, 0x0AF1, 0x1AD0, 0x2AB3, 0x3A92,
        0xFD2E, 0xED0F, 0xDD6C, 0xCD4D, 0xBDAA, 0xAD8B, 0x9DE8, 0x8DC9,
        0x7C26, 0x6C07, 0x5C64, 0x4C45, 0x3CA2, 0x2C83, 0x1CE0, 0x0CC1,
        0xEF1F, 0xFF3E, 0xCF5D, 0xDF7C, 0xAF9B, 0xBFBA, 0x8FD9, 0x9FF8,
        0x6E17, 0x7E36, 0x4E55, 0x5E74, 0x2E93, 0x3EB2, 0x0ED1, 0x1EF0
    ]

    def __init__(self, port):
        self.array = bytearray(0)
        self.cursor = 0
        self.mStatus = self.YtDataLinkStatus.YT_DL_IDLE
        self.mMsgLen = 0
        self.mCrc = 0
        self.mCrcCalc = 0xffff
        self.mTrans = False
        self.mCrcSendCalc = 0xffff
        self.port = port
        self.rpcId = 0
        self.port.read(1)

    def crcUpdate (self, ch, first):
        if (first):
            self.mCrcCalc = 0xffff
        self.mCrcCalc = self.ccittTable[(self.mCrcCalc >> 8 ^ ch) & 0xff] ^ ((self.mCrcCalc << 8) & 0xffff)

    def toHex (self, d):
        return format(d, '02X')

    def printBuf (self, buf):
        print(''.join('{:02x} '.format(x) for x in buf))

    def recvRunOnce(self):
        if len(self.array) < 10:
            buf = self.port.read(16)
            self.array += buf
        while len(self.array) > 0:
            ch = self.array.pop(0)
            # print('loop=', self.toHex(ch))
            if (ch == self.SOF):
                if (self.mStatus != self.YtDataLinkStatus.YT_DL_IDLE):
                    print('[YtMsg] unfinished pkg(%d/%d)', self.mBufi, self.mMsgLen)

                self.mStatus = self.YtDataLinkStatus.YT_DL_LEN1_PENDING
            elif (ch == self.TRANS):
                self.mTrans = True
            else:
                # //转义后，不会出现SOF, TRANS
                if (self.mTrans):
                    ch = ch ^ self.TRANS
                    self.mTrans = False

                # if (self.mStatus == self.YtDataLinkStatus.YT_DL_IDLE):
                if (self.mStatus == self.YtDataLinkStatus.YT_DL_LEN1_PENDING):
                    # print('[YtMsg] begin\n')
                    self.mStatus = self.YtDataLinkStatus.YT_DL_LEN1_PENDING
                    self.mMsgLen = 0
                    self.mCrc = 0
                    # //falls through
                if (self.mStatus == self.YtDataLinkStatus.YT_DL_LEN1_PENDING or
                    self.mStatus == self.YtDataLinkStatus.YT_DL_LEN2_PENDING or
                    self.mStatus == self.YtDataLinkStatus.YT_DL_LEN3_PENDING):
                    # // console.log('[YtMsg]', self.toHex(ch))
                    self.crcUpdate(ch, self.mStatus == self.YtDataLinkStatus.YT_DL_LEN1_PENDING)
                    self.mMsgLen = (self.mMsgLen << 8) | ch
                    if (self.mStatus == self.YtDataLinkStatus.YT_DL_LEN3_PENDING):
                        # print('len=', self.mMsgLen)
                        if (self.mMsgLen > self.ytMsgSize):
                            # console.log('[YtMsg] Error: msg len %d > %d\n', self.mMsgLen, ytMsgSize)
                            self.mStatus = self.YtDataLinkStatus.YT_DL_IDLE
                            continue

                    self.mStatus = self.mStatus + 1
                    continue

                if (self.mStatus == self.YtDataLinkStatus.YT_DL_LEN_CRC_H):
                    self.mCrc = (self.mCrc << 8) | ch
                    self.mStatus = self.mStatus + 1
                    continue

                if (self.mStatus == self.YtDataLinkStatus.YT_DL_LEN_CRC_L):
                    self.mCrc = (self.mCrc << 8) | ch
                    if ((self.mCrcCalc) != self.mCrc):
                        print('[YtMsg] Error: msg len crc 0x%04x != 0x%04x\n', self.toHex(self.mCrcCalc), self.toHex(self.mCrc))
                        self.mStatus = self.YtDataLinkStatus.YT_DL_IDLE
                        continue

                    self.mStatus = self.mStatus + 1
                    self.mBuf = bytearray(self.mMsgLen)
                    self.mBufi = 0

                    # Brust read
                    # print('[YtMsg] Brust read', self.mMsgLen)
                    self.array += self.port.read(self.mMsgLen)
                    continue

                if (self.mStatus == self.YtDataLinkStatus.YT_DL_DATA):
                    self.crcUpdate(ch, self.mBufi == 0)
                    self.mBuf[self.mBufi] = ch
                    self.mBufi += 1
                    if (self.mBufi == self.mMsgLen):
                        self.mStatus = self.mStatus + 1
                    continue

                if (self.mStatus == self.YtDataLinkStatus.YT_DL_CRC_H):
                    self.mCrc = 0
                    # //falls through
                if (self.mStatus == self.YtDataLinkStatus.YT_DL_CRC_H or
                    self.mStatus == self.YtDataLinkStatus.YT_DL_CRC_L):
                    self.mCrc = (self.mCrc << 8) | ch
                    if (self.mStatus == self.YtDataLinkStatus.YT_DL_CRC_L):
                        if ((self.mCrcCalc) != self.mCrc):
                            print('[YtMsg] Error: msg crc 0x%04x != 0x%04x\n', self.toHex(self.mCrcCalc), self.toHex(self.mCrc))
                            self.mStatus = self.YtDataLinkStatus.YT_DL_IDLE
                            continue

                        self.mStatus = self.YtDataLinkStatus.YT_DL_IDLE

                        target = YtMsg()
                        dataV2 = None
                        target.ParseFromString(self.mBuf)
                        if target.result.HasField('dataV2'):
                            dataV2 = YtDataLink.DataV2(target.result.dataV2)
                        return dataV2, target

                    self.mStatus = self.mStatus + 1
                    continue

        return None, None

    def _sendFilePackage (self, remoteFile, totalLength, buf, offset, auth = ''):
        rpc = YtRpc()
        rpc.func = YtRpc.Function.uploadFile
        rpc.auth = auth

        params = FilePart()
        params.path = (remoteFile)
        params.totalLength = (totalLength)
        params.offset = (offset)
        params.data = (buf)
        rpc.filePart.CopyFrom(params)
        result = self.sendRpcMsg(rpc, 10000)
        # print(result)
        if (not result.HasField('response')):
            raise Exception('未知错误')

    def sendFile (self, localFile, remoteFile, auth = '', progressCb = None):
        file = open(localFile, mode='rb')
        data = file.read()
        # print(len(data))

        totalLength = len(data)
        currentOffset = 0
        remaining = totalLength
        sizeToTransmit = 0
        while (remaining > 0):
            sizeToTransmit = self.fileBlobSize if remaining > self.fileBlobSize else remaining
            buf = data[currentOffset : currentOffset + sizeToTransmit]
            for i in range(100):
                try:
                    self._sendFilePackage(remoteFile, totalLength, buf, currentOffset, auth)
                    break
                except Exception as e:
                    if (i > 10):
                        raise e
                    # console.log('[sendFile]', e, 'retry', i)
            currentOffset += sizeToTransmit
            remaining = totalLength - currentOffset
            if not (progressCb is None):
                progressCb(parseInt(Math.min(100 if remaining == 0 else 99, (currentOffset + sizeToTransmit) / totalLength * 100)))

    def genRpcId (self):
        self.rpcId += 1
        return self.rpcId

    def sendRpcMsg (self, rpc, timeoutMs = 10000):
        rpcid = self.genRpcId()
        rpc.sequenceId = rpcid

        msg = YtMsg()
        msg.rpc.CopyFrom(rpc)

        self.sendYtMsg(msg)
        ts = time.time()
        while (time.time() - ts < timeoutMs/1000):
            _, resp = self.recvRunOnce()
            if (resp and resp.HasField('response')):
                if (resp.response.sequenceId == rpcid):
                    if (resp.response.code != YtRpcResponse.ReturnCode.SUCC and
                        resp.response.code != YtRpcResponse.ReturnCode.CONTINUE):
                        err = self.getErrMsg(resp.response.code)
                        raise Exception(err)
                    return resp
        raise Exception('Timeout')

    def sendYtMsg (self, msg):
        data = msg.SerializeToString()
        self.write(data)

    def getErrMsg (self, code):
        errMsgs = {}
        c = YtRpcResponse.ReturnCode
        errMsgs[c.ERROR_REGISTER_FACEID_TIMEOUT] = '在指定时间内，没有检测到合格人脸'
        errMsgs[c.ERROR_FILE_EXCEED_LIMITS] = '文件超过大小/尺寸限制'
        errMsgs[c.ERROR_REGISTER_FACEID_NO_FACE_DETECTED] = '没有检测到人脸'
        errMsgs[c.ERROR_REGISTER_FACEID_FACE_QUALITY_TOO_LOW] = '人脸侧偏过大'
        errMsgs[c.ERROR_REGISTER_FACEID_TOO_MANY_FACES] = '人脸太多'
        errMsgs[c.ERROR_REGISTER_FACEID_FILE_NOT_READABLE] = '无效文件'
        errMsgs[c.ERROR_REGISTER_FACEID_LIB_FULL] = '库满了'
        errMsgs[c.ERROR_FACEID_NOT_EXIST] = 'FaceID不存在'

        if (code in errMsgs):
            return errMsgs[code]
        else:
            return '未知错误(' + str(code) + ')'

    def crcSendUpdate (self, ch, first = False):
        if (first):
            self.mCrcSendCalc = 0xffff

        self.mCrcSendCalc = self.ccittTable[(self.mCrcSendCalc >> 8 ^ ch) & 0xff] ^ ((self.mCrcSendCalc << 8) & 0xffff)

    def write (self, data):
        buf = bytearray(len(data) + 8)
        buf[0] = 0x10
        buf[1] = (len(data) >> 16) & 0xff
        buf[2] = (len(data) >> 8) & 0xff
        buf[3] = (len(data) >> 0) & 0xff
        self.crcSendUpdate(buf[1], True)
        self.crcSendUpdate(buf[2])
        self.crcSendUpdate(buf[3])
        buf[4] = (self.mCrcSendCalc >> 8) & 0xff
        buf[5] = (self.mCrcSendCalc >> 0) & 0xff
        buf[6:] = data
        buf.append(0)
        buf.append(0)

        transLen = 0
        # self.printBuf(buf)
        for i in range(1, len(buf)):
            # print(i, self.toHex(buf[i]))
            if (i >= 6 and i < len(buf) - 2):
                self.crcSendUpdate(buf[i], i == 6)
            if (buf[i] == 0x10 or buf[i] == 0x11):
                transLen += 1

        buf[len(buf) - 2] = (self.mCrcSendCalc >> 8) & 0xff
        buf[len(buf) - 1] = (self.mCrcSendCalc >> 0) & 0xff

        for i in range(len(buf)-2, len(buf)):
            # print(i, self.toHex(buf[i]))
            if (buf[i] == 0x10 or buf[i] == 0x11):
                transLen += 1

        idx = 0
        transedBuffer = bytearray(len(buf) + transLen)

        for i in range(0, len(buf)):
            if ((buf[i] == 0x10 or buf[i] == 0x11) and i > 0):
                transedBuffer[idx] = 0x11
                idx += 1
                transedBuffer[idx] = buf[i] ^ 0x11
                idx += 1
            else:
                transedBuffer[idx] = buf[i]
                idx += 1

        # self.printBuf(transedBuffer)
        self.port.write(transedBuffer)

    # 结果包解析
    def unpackInt16 (bufObj):
        buf = bytearray(2)
        buf[0] = bufObj.p[bufObj.i]
        buf[1] = bufObj.p[bufObj.i + 1]
        ret = struct.unpack("<h", buf)[0]
        # ret = 0
        # ret = bufObj.p[bufObj.i + 1]
        # ret <<= 8
        # ret |= bufObj.p[bufObj.i]
        bufObj.i += 2
        return ret

    def unpackUInt16 (bufObj):
        ret = 0
        ret = bufObj.p[bufObj.i + 1]
        ret <<= 8
        ret |= bufObj.p[bufObj.i]
        bufObj.i += 2
        return ret

    def unpackVarUInt32 (bufObj):
        ret = 0
        shift = 0
        while (True):
            byte = bufObj.p[bufObj.i]
            bufObj.i += 1
            ret |= (byte & 0x7f) << shift
            if ((byte & 0x80) == 0):
                break
            shift += 7
        return ret


text:
# %% [markdown]
# Problem statement: To build a CNN based model which can accurately detect melanoma. Melanoma is a type of cancer that can be deadly if not detected early. It accounts for 75% of skin cancer deaths. A solution which can evaluate images and alert the dermatologists about the presence of melanoma has the potential to reduce a lot of manual effort needed in diagnosis.
# %% [markdown]
# ### Importing Skin Cancer Data
# #### To do: Take necessary actions to read the data
# %% [markdown]
# ### Importing all the important libraries

# %%
import Augmentor
from IPython import get_ipython
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import os
import PIL
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.preprocessing import image_dataset_from_directory
import sys
from pathlib2 import Path
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, Conv2D, MaxPooling2D
from tensorflow.keras.regularizers import l2
import kerastuner as kt
from kerastuner import HyperModel, RandomSearch, Hyperband, BayesianOptimization
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.layers.experimental.preprocessing import Rescaling
# %%
# Ref: https://www.tensorflow.org/tutorials/images/classification

# %% [markdown]
# This assignment uses a dataset of about 2357 images of skin cancer types. The dataset contains 9 sub-directories in each train and test subdirectories. The 9 sub-directories contains the images of 9 skin cancer types respectively.

# %%
root_folder = Path().resolve()
images_path = root_folder / "Skin cancer ISIC The International Skin Imaging Collaboration"
images_path

# %%
# Defining the path for train and test images
data_dir_train = images_path / "Train"
data_dir_test = images_path / "Test"

# %%
train_image_file_names = list(data_dir_train.glob('*/*.jpg'))
image_count_train = len(train_image_file_names)
print(image_count_train)
test_image_file_names = list(data_dir_test.glob('*/*.jpg'))
image_count_test = len(test_image_file_names)
print(image_count_test)

# %% [markdown]
# ### Load using keras.preprocessing
#
# Let's load these images off disk using the helpful image_dataset_from_directory utility.
# %% [markdown]
# ### Create a dataset
#
# Define some parameters for the loader:

# %%
batch_size = 32
img_height = 180
img_width = 180
SEED = 123
image_size = (img_height, img_width)

# %% [markdown]
# Use 80% of the images for training, and 20% for validation.

# %%
# Write your train dataset here
# Note use seed=123 while creating your dataset using tf.keras.preprocessing.image_dataset_from_directory
# Note, make sure your resize your images to the size img_height*img_width, while writting the dataset
train_ds = image_dataset_from_directory(data_dir_train, seed=SEED, image_size=image_size, batch_size=batch_size, validation_split=0.2, subset="training")
train_ds

# %%
# Write your validation dataset here
# Note use seed=123 while creating your dataset using tf.keras.preprocessing.image_dataset_from_directory
# Note, make sure your resize your images to the size img_height*img_width, while writting the dataset
val_ds = image_dataset_from_directory(data_dir_test, seed=SEED, image_size=image_size, batch_size=batch_size, subset="validation", validation_split=0.2)

# %%
# List out all the classes of skin cancer and store them in a list.
# You can find the class names in the class_names attribute on these datasets.
# These correspond to the directory names in alphabetical order.
class_names = train_ds.class_names
print(class_names)

# %% [markdown]
# ### Visualize the data
# %%
fig = plt.figure(figsize=(10, 10))
for i in range(len(class_names)):
    ax = plt.subplot(3, 3, i + 1)
    class_name = class_names[i]
    img_dir = data_dir_train / class_name
    img_num = np.random.randint(image_count_train)
    im = plt.imread(str(train_image_file_names[img_num]))
    ax.set_title(class_names[i])
    plt.imshow(im)
plt.show()

# %%
# train_ds_np =
# train_images = train_ds.reshape(len(train_ds), img_height, img_width, 3)
# test_images = val_ds.reshape(len(val_ds), img_height, img_width, 3)

# %% [markdown]
# The `image_batch` is a tensor of the shape `(32, 180, 180, 3)`. This is a batch of 32 images of shape `180x180x3` (the last dimension refers to color channels RGB). The `label_batch` is a tensor of the shape `(32,)`, these are corresponding labels to the 32 images.
# %% [markdown]
# `Dataset.cache()` keeps the images in memory after they're loaded off disk during the first epoch.
#
# `Dataset.prefetch()` overlaps data preprocessing and model execution while training.
# %%
# for image_batch, labels_batch in train_ds:
#     print(image_batch.shape)
#     print(labels_batch.shape)
#     break
# %%
AUTOTUNE = tf.data.experimental.AUTOTUNE
train_ds = train_ds.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE)
val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)

# %% [markdown]
# ### Create the model
# #### Todo: Create a CNN model, which can accurately detect 9 classes present in the dataset. Use ```layers.experimental.preprocessing.Rescaling``` to normalize pixel values between (0,1). The RGB channel values are in the `[0, 255]` range. This is not ideal for a neural network. Here, it is good to standardize values to be in the `[0, 1]`
# %%
data_augmentation = keras.Sequential(
    [
        layers.experimental.preprocessing.RandomFlip("horizontal"),
        layers.experimental.preprocessing.RandomRotation(0.1),
    ]
)
augmented_train_ds = train_ds.map(lambda x, y: (data_augmentation(x, training=True), y))

# %%
# Building Hyperparameter tuner
# Note: Scaling layer and model compilation is part of ClassificationHyperModel

# %%

# %% [markdown]
# ### Compile the model
# Choose an appropriate optimizer and loss function for model training

# %%
# Todo, choose an appropriate optimizer and loss function
#  Optimizer and loss function is part of ClassificationHyperModel
# %% [markdown]
'''
#### Build, Run and Evaluate Random Search Tuner
'''
# %%

# # %%
# for image_batch, labels_batch in train_ds:
#     print(image_batch.shape)
#     print(labels_batch.shape)
#     break

# %%
#  Extract X and Y
# images, labels = tuple(zip(*train_ds))
# %%

# %%
# View the summary of all layers
model.summary()

# %% [markdown]
# ### Train the model

# %%
epochs = 20
history = model.fit(
    train_ds,
    validation_data=val_ds,
    epochs=epochs
)

# %% [markdown]
# ### Visualizing training results

# %%
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs_range = range(epochs)

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()

# %% [markdown]
# #### Todo: Write your findings after the model fit, see if there is an evidence of model overfit or underfit
# %% [markdown]
# ### Write your findings here

# %%
# Todo, after you have analysed the model fit history for presence of underfit or overfit, choose an appropriate data augumentation strategy.
# Your code goes here

# %%
# Todo, visualize how your augmentation strategy works for one instance of training image.
# Your code goes here

# %% [markdown]
# ### Todo:
# ### Create the model, compile and train the model
#

# %%
# You can use Dropout layer if there is an evidence of overfitting in your findings

# Your code goes here

# %% [markdown]
# ### Compiling the model

# %%
# Your code goes here

# %% [markdown]
# ### Training the model

# %%
# Your code goes here, note: train your model for 20 epochs
history =  # your training code

# %% [markdown]
# ### Visualizing the results

# %%
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs_range = range(epochs)

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()

# %% [markdown]
# #### Todo: Write your findings after the model fit, see if there is an evidence of model overfit or underfit. Do you think there is some improvement now as compared to the previous model run?
# %% [markdown]
# #### **Todo:** Find the distribution of classes in the training dataset.
# #### **Context:** Many times real life datasets can have class imbalance, one class can have proportionately higher number of samples compared to the others. Class imbalance can have a detrimental effect on the final model quality. Hence as a sanity check it becomes important to check what is the distribution of classes in the data.

# %%
# Your code goes here.

# %% [markdown]
# #### **Todo:** Write your findings here:
# #### - Which class has the least number of samples?
# #### - Which classes dominate the data in terms proportionate number of samples?
#
# %% [markdown]
# #### **Todo:** Rectify the class imbalance
# #### **Context:** You can use a python package known as `Augmentor` (https://augmentor.readthedocs.io/en/master/) to add more samples across all classes so that none of the classes have very few samples.

# %%
get_ipython().system('pip install Augmentor')

# %% [markdown]
# To use `Augmentor`, the following general procedure is followed:
#
# 1. Instantiate a `Pipeline` object pointing to a directory containing your initial image data set.

# 2. Define a number of operations to perform on this data set using your `Pipeline` object.

# 3. Execute these operations by calling the `Pipeline’s` `sample()` method.
#

# %%
path_to_training_dataset = "To do"
for i in class_names:
    p = Augmentor.Pipeline(path_to_training_dataset + i)
    p.rotate(probability=0.7, max_left_rotation=10, max_right_rotation=10)
    p.sample(500)  # We are adding 500 samples per class to make sure that none of the classes are sparse.

# %% [markdown]
# Augmentor has stored the augmented images in the output sub-directory of each of the sub-directories of skin cancer types.. Lets take a look at total count of augmented images.

# %%
image_count_train = len(list(data_dir_train.glob('*/output/*.jpg')))
print(image_count_train)

# %% [markdown]
# ### Lets see the distribution of augmented data after adding new images to the original training data.

# %%
path_list = [x for x in glob(os.path.join(data_dir_train, '*', 'output', '*.jpg'))]
path_list

# %%
lesion_list_new = [os.path.basename(os.path.dirname(os.path.dirname(y))) for y in glob(os.path.join(data_dir_train, '*', 'output', '*.jpg'))]
lesion_list_new

# %%
dataframe_dict_new = dict(zip(path_list_new, lesion_list_new))

# %%
df2 = pd.DataFrame(list(dataframe_dict_new.items()), columns=['Path', 'Label'])
new_df = original_df.append(df2)

# %%
new_df['Label'].value_counts()

# %% [markdown]
# So, now we have added 500 images to all the classes to maintain some class balance. We can add more images as we want to improve training process.
# %% [markdown]
# #### **Todo**: Train the model on the data created using Augmentor

# %%
batch_size = 32
img_height = 180
img_width = 180

# %% [markdown]
# #### **Todo:** Create a training dataset

# %%
data_dir_train = "path to directory with training data + data created using augmentor"
train_ds = tf.keras.preprocessing.image_dataset_from_directory(
    data_dir_train,
    seed=123,
    validation_split=0.2,
    subset=# Todo choose the correct parameter value, so that only training data is refered to,,
    image_size=(img_height, img_width),
    batch_size=batch_size)

# %% [markdown]
# #### **Todo:** Create a validation dataset

# %%
val_ds = tf.keras.preprocessing.image_dataset_from_directory(
    data_dir_train,
    seed=123,
    validation_split=0.2,
    subset=# Todo choose the correct parameter value, so that only validation data is refered to,
    image_size=(img_height, img_width),
    batch_size=batch_size)

# %% [markdown]
# #### **Todo:** Create your model (make sure to include normalization)

# %%
# your code goes here

# %% [markdown]
# #### **Todo:** Compile your model (Choose optimizer and loss function appropriately)

# %%
# your code goes here

# %% [markdown]
# #### **Todo:**  Train your model

# %%
epochs = 30
# Your code goes here, use 50 epochs.
history =  # your model fit code

# %% [markdown]
# #### **Todo:**  Visualize the model results

# %%
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs_range = range(epochs)

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()

# %% [markdown]
# #### **Todo:**  Analyze your results here. Did you get rid of underfitting/overfitting? Did class rebalance help?
#
#

# %%


text:
# -*- coding: utf-8 -*-
''' Data Transforms Module

This module contains functions for transforming PV power data, including time-axis standardization and
2D-array generation

'''

from datetime import timedelta
import numpy as np
import pandas as pd
from scipy.signal import argrelextrema
from sklearn.neighbors.kde import KernelDensity

from solardatatools.clear_day_detection import find_clear_days
from solardatatools.utilities import total_variation_filter, total_variation_plus_seasonal_filter

def standardize_time_axis(df, datetimekey='Date-Time'):
    '''
    This function takes in a pandas data frame containing tabular time series data, likely generated with a call to
    pandas.read_csv(). It is assumed that each row of the data frame corresponds to a unique date-time, though not
    necessarily on standard intervals. This function will attempt to convert a user-specified column containing time
    stamps to python datetime objects, assign this column to the index of the data frame, and then standardize the
    index over time. By standardize, we mean reconstruct the index to be at regular intervals, starting at midnight of
    the first day of the data set. This solves a couple common data errors when working with raw data. (1) Missing data
    points from skipped scans in the data acquisition system. (2) Time stamps that are at irregular exact times,
    including fractional seconds.

    :param df: A pandas data frame containing the tabular time series data
    :param datetimekey: An optional key corresponding to the name of the column that contains the time stamps
    :return: A new data frame with a standardized time axis
    '''
    # convert index to timeseries
    try:
        df[datetimekey] = pd.to_datetime(df[datetimekey])
        df.set_index('Date-Time', inplace=True)
    except KeyError:
        time_cols = [col for col in df.columns if np.logical_or('Time' in col, 'time' in col)]
        key = time_cols[0]
        df[datetimekey] = pd.to_datetime(df[key])
        df.set_index(datetimekey, inplace=True)
    # standardize the timeseries axis to a regular frequency over a full set of days
    diff = (df.index[1:] - df.index[:-1]).seconds
    freq = int(np.median(diff))  # the number of seconds between each measurement
    start = df.index[0]
    end = df.index[-1]
    time_index = pd.date_range(start=start.date(), end=end.date() + timedelta(days=1), freq='{}s'.format(freq))[:-1]
    df = df.reindex(index=time_index, method='nearest')
    return df.fillna(value=0)

def make_2d(df, key='dc_power'):
    '''
    This function constructs a 2D array (or matrix) from a time series signal with a standardized time axis. The data is
    chunked into days, and each consecutive day becomes a column of the matrix.

    :param df: A pandas data frame contained tabular data with a standardized time axis.
    :param key: The key corresponding to the column in the data frame contained the signal to make into a matrix
    :return: A 2D numpy array with shape (measurements per day, days in data set)
    '''
    if df is not None:
        days = df.resample('D').max().index[1:-1]
        start = days[0]
        end = days[-1]
        n_steps = int(24 * 60 * 60 / df.index.freq.delta.seconds)
        D = df[key].loc[start:end].iloc[:-1].values.reshape(n_steps, -1, order='F')
        return D
    else:
        return

def fix_time_shifts(data, verbose=False, return_ixs=False, clear_day_filter=True,
                    c1=10., c2=500., c3=5.):
    '''
    This is an algorithm to detect and fix time stamping shifts in a PV power database. This is a common data error
    that can have a number of causes: improper handling of DST, resetting of a data logger clock, or issues with
    storing the data in the database. The algorithm performs as follows:
    Part 1:
        a) Estimate solar noon for each day relative to the provided time axis. This is estimated as the "center of
           mass" in time of the energy production each day.
        b) Filter this signal for clear days
        c) Fit a total variation filter with seasonal baseline to the output of (b)
        d) Perform KDE-based clustering on the output of (c)
        e) Extract the days on which the transitions between clusters occur
    Part 2:
        a) Find the average solar noon value for each cluster
        b) Taking the first cluster as a reference point, find the offsets in average values between the first cluster
           and all others
        c) Adjust the time axis for all clusters after the first by the amount calculated in (b)

    :param data: A 2D numpy array containing a solar power time series signal (see `data_transforms.make_2d`)
    :param verbose: An option to print information about what clusters are found
    :param return_ixs: An option to return the indices of the boundary days for the clusters
    :return:
    '''
    D = data
    #################################################################################################################
    # Part 1: Detecting the days on which shifts occurs. If no shifts are detected, the algorithm exits, returning
    # the original data array. Otherwise, the algorithm proceeds to Part 2.
    #################################################################################################################
    # Find "center of mass" of each day's energy content. This generates a 1D signal from the 2D input signal.
    div1 = np.dot(np.linspace(0, 24, D.shape[0]), D)
    div2 = np.sum(D, axis=0)
    s1 = np.empty_like(div1)
    s1[:] = np.nan
    msk = div2 != 0
    s1[msk] = np.divide(div1[msk], div2[msk])
    # Apply a clear day filter
    if clear_day_filter:
        m = find_clear_days(D)
        s1_f = np.empty_like(s1)
        s1_f[:] = np.nan
        s1_f[m] = s1[m]
    else:
        s1_f = s1
    # Apply total variation filter (with seasonal baseline if >1yr of data)
    if len(s1) > 365:
        s2, s_seas = total_variation_plus_seasonal_filter(s1_f, c1=c1, c2=c2)
    else:
        s2 = total_variation_filter(s1_f, C=c3)
    # Perform clustering with KDE
    kde = KernelDensity(kernel='gaussian', bandwidth=0.05).fit(s2[:, np.newaxis])
    X_plot = np.linspace(0.95 * np.min(s2), 1.05 * np.max(s2))[:, np.newaxis]
    log_dens = kde.score_samples(X_plot)
    mins = argrelextrema(log_dens, np.less)[0]      # potential cut points to make clusters
    maxs = argrelextrema(log_dens, np.greater)[0]   # locations of the max point in each cluster
    # Drop clusters with too few members
    keep = np.ones_like(maxs, dtype=np.bool)
    for ix, mx in enumerate(maxs):
        if np.exp(log_dens)[mx] < 1e-1:
            keep[ix] = 0
    mx_keep = maxs[keep]
    mx_drop = maxs[~keep]
    mn_drop = []
    # Determine closest clusters in keep set to each cluster that should be dropped
    for md in mx_drop:
        dists = np.abs(X_plot[:, 0][md] - X_plot[:, 0][mx_keep])
        max_merge = mx_keep[np.argmin(dists)]
        # Determine which minimum index to remove to correctly merge clusters
        for mn in mins:
            cond1 = np.logical_and(mn < max_merge, mn > md)
            cond2 = np.logical_and(mn > max_merge, mn < md)
            if np.logical_or(cond1, cond2):
                if verbose:
                    print('merge', md, 'with', max_merge, 'by dropping', mn)
                mn_drop.append(mn)
    mins_new = np.array([i for i in mins if i not in mn_drop])
    # Assign cluster labels to days in data set
    clusters = np.zeros_like(s1)
    if len(mins_new) > 0:
        for it, ex in enumerate(X_plot[:, 0][mins_new]):
            m = s2 >= ex
            clusters[m] = it + 1
    # Identify indices corresponding to days when time shifts occurred
    index_set = np.arange(D.shape[1]-1)[clusters[1:] != clusters[:-1]] + 1
    # Exit if no time shifts detected
    if len(index_set) == 0:
        if verbose:
            print('No time shifts found')
        if return_ixs:
            return D, []
        else:
            return D
    #################################################################################################################
    # Part 2: Fixing the time shifts.
    #################################################################################################################
    if verbose:
        print('Time shifts found at: ', index_set)
    ixs = np.r_[[None], index_set, [None]]
    # Take averages of solar noon estimates over the segments of the data set defined by the shift points
    A = []
    for i in range(len(ixs) - 1):
        avg = np.average(np.ma.masked_invalid(s1_f[ixs[i]:ixs[i + 1]]))
        A.append(np.round(avg * D.shape[0] / 24))
    A = np.array(A)
    # Considering the first segment as the reference point, determine how much to shift the remaining segments
    rolls = A[0] - A[1:]
    # Apply the corresponding shift to each segment
    Dout = np.copy(D)
    for ind, roll in enumerate(rolls):
        D_rolled = np.roll(D, int(roll), axis=0)
        Dout[:, ixs[ind + 1]:] = D_rolled[:, ixs[ind + 1]:]
    # We find that a second pass with halved weights catches some transition points
    # that might have been missed for data with many small transitions
    Dout = fix_time_shifts(Dout, return_ixs=False, c1=c1/2, c2=c2/2, c3=c3/2)
    if return_ixs:
        return Dout, index_set
    else:
        return Dout

max_stars_repo_name: tadatoshi/solar-data-tools

max_issues_repo_name: tadatoshi/solar-data-tools

max_forks_repo_name: tadatoshi/solar-data-tools

reason: import numpy,from scipy


text:
# https://deeplearningcourses.com/c/data-science-supervised-machine-learning-in-python
# https://www.udemy.com/data-science-supervised-machine-learning-in-python
# This is an example of a K-Nearest Neighbors classifier on MNIST data.
# We try k=1...5 to show how we might choose the best k.
# sudo pip install sortedcontainers (if you don't have it)

import numpy as np
from sortedcontainers import SortedList
# Note: You can't use SortedDict because the key is distance
# if 2 close points are the same distance away, one will be overwritten
from util import get_data
from datetime import datetime

class KNN(object):
    def __init__(self, k):
        self.k = k

    def fit(self, X, y):
        self.X = X
        self.y = y

    def predict(self, X):
        y = np.zeros(len(X))
        for i,x in enumerate(X): # test points
            sl = SortedList(load=self.k) # stores (distance, class) tuples
            for j,xt in enumerate(self.X): # training points
                diff = x - xt
                d = diff.dot(diff)
                if len(sl) < self.k:
                    # don't need to check, just add
                    sl.add( (d, self.y[j]) )
                else:
                    if d < sl[-1][0]:
                        del sl[-1]
                        sl.add( (d, self.y[j]) )
            # print "input:", x
            # print "sl:", sl

            # vote
            votes = {}
            for _, v in sl:
                # print "v:", v
                votes[v] = votes.get(v,0) + 1
            # print "votes:", votes, "true:", Ytest[i]
            max_votes = 0
            max_votes_class = -1
            for v,count in votes.iteritems():
                if count > max_votes:
                    max_votes = count
                    max_votes_class = v
            y[i] = max_votes_class
        return y

    def score(self, X, Y):
        P = self.predict(X)
        return np.mean(P == Y)

if __name__ == '__main__':
    X, Y = get_data(2000)
    Ntrain = 1000
    Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
    Xtest, Ytest = X[Ntrain:], Y[Ntrain:]
    for k in (1,2,3,4,5):
        knn = KNN(k)
        t0 = datetime.now()
        knn.fit(Xtrain, Ytrain)
        print "Training time:", (datetime.now() - t0)

        t0 = datetime.now()
        print "Train accuracy:", knn.score(Xtrain, Ytrain)
        print "Time to compute train accuracy:", (datetime.now() - t0), "Train size:", len(Ytrain)

        t0 = datetime.now()
        print "Test accuracy:", knn.score(Xtest, Ytest)
        print "Time to compute test accuracy:", (datetime.now() - t0), "Test size:", len(Ytest)

max_stars_repo_name: austinburks/nlp-udemy

max_issues_repo_name: austinburks/nlp-udemy

max_forks_repo_name: austinburks/nlp-udemy


text:
import sys
import numpy as np
from pathlib import Path
from TopasOpt import Optimisers as to

BaseDirectory = 'C:\\Users\\Brendan\\Documents\\temp'
SimulationName = 'development_test'
OptimisationDirectory = Path(__file__).parent

# set up optimisation params for rosen function:
optimisation_params = {}
optimisation_params['ParameterNames'] = ['x', 'y']
optimisation_params['UpperBounds'] = np.array([1, 1])
optimisation_params['LowerBounds'] = np.array([-1, -1])
optimisation_params['start_point'] = np.array([.9, .9])
# Remember true values are  [1.82, 2.5, 27]
optimisation_params['Nitterations'] = 100
# optimisation_params['Suggestions'] # you can suggest points to test if you want - we won't here.
ReadMeText = 'This is an example in which the rosen function is minimised, and demonstrates how this library' \
             'can be tested without actually calling topas'

Optimiser = to.NelderMeadOptimiser(optimisation_params, BaseDirectory, SimulationName, OptimisationDirectory,
                                TopasLocation='testing_mode', ReadMeText=ReadMeText, Overwrite=True, KeepAllResults=False,
                                   NM_StartingSimplexRelativeVal=.4)

Optimiser.RunOptimisation()

max_stars_repo_name: ACRF-Image-X-Institute/TopasOpt

max_issues_repo_name: ACRF-Image-X-Institute/TopasOpt

max_forks_repo_name: ACRF-Image-X-Institute/TopasOpt