from nemo.collections.asr.models.ctc_bpe_models import EncDecCTCModelBPE
from nemo.core.config import hydra_runner
from nemo.utils import logging
from nemo.collections.asr.losses.ctc import CTCLoss
import omegaconf
from omegaconf import OmegaConf
from omegaconf import DictConfig
import pytorch_lightning as pl
from pytorch_lightning.callbacks import ModelCheckpoint
from pytorch_lightning.plugins import DDPPlugin
from pytorch_lightning.callbacks.early_stopping import EarlyStopping
import sys
import logging
from modified_model import ModifiedModel, ModifiedTeacherModel
import os
import pickle
import numpy as np

LANGUAGE = "splits"
path = '/home/DATA2/apoorvaaggarwal'
manifest_dir = os.path.join(path, LANGUAGE)
train_manifest = f"{manifest_dir}/train/train.json"
dev_manifest = f"{manifest_dir}/val/val.json"
test_manifest = f"{manifest_dir}/test/test.json"

@hydra_runner(config_path=r"/home/DATA2/apoorvaaggarwal/Paper_1_Implementation/conformer/", config_name="conformer_ctc_bpe")
def main(cfg):
    logging.debug(cfg)
    cfg['model']['train_ds']['manifest_filepath'] = train_manifest
    cfg['model']['validation_ds']['manifest_filepath'] = dev_manifest
    cfg['model']['test_ds']['manifest_filepath'] = test_manifest
    logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}')
    logging.info("trainer: {}".format(cfg.trainer))
    checkpoint_callback = ModelCheckpoint(dirpath='/home/DATA2/apoorvaaggarwal/training/exp_2',
                                          save_last=True, save_top_k=20,
                                          filename='{epoch}-{val_wer:.2f}-{val_loss:.2f}',monitor="val_wer", every_n_epochs=1)

    checkpoint_path = None
    trainer = pl.Trainer(gpus=[0], accelerator='ddp', max_epochs=50, callbacks=[checkpoint_callback, EarlyStopping(monitor="val_wer", mode="min")], plugins=DDPPlugin(find_unused_parameters=False))

    teacher_model = ModifiedTeacherModel.from_pretrained("stt_en_conformer_ctc_large")
    student_model = ModifiedModel.from_pretrained("stt_en_conformer_ctc_large")

    os.chdir('/home/DATA2/apoorvaaggarwal/splits/train')
    directory = 'wav'
    manifest_dir_url = '/home/DATA2/apoorvaaggarwal/splits/train/wav'
    list_of_train_paths = []

    for filename in os.listdir(directory):
        list_of_train_paths.append(f"{manifest_dir_url}/{filename}")

#     logging.error(f"\nCURRENT DIRECTORY HAS:\n{os.listdir(directory)}")

    os.chdir('/home/DATA2/apoorvaaggarwal/splits/val')
    directory = 'wav'
    manifest_dir_url = '/home/DATA2/apoorvaaggarwal/splits/val/wav'
    list_of_dev_paths = []

    for filename in os.listdir(directory):
        list_of_dev_paths.append(f"{manifest_dir_url}/{filename}")

    list_of_all_paths = list_of_train_paths + list_of_dev_paths

#     logging.error(f"\nlist_of_all_paths: {list_of_all_paths}\n")
#     logging.error(f"\nCURRENT DIRECTORY HAS:\n{os.listdir(directory)}")

    os.chdir('/home/DATA2/apoorvaaggarwal/training')

    # Getting Teacher Model's Softmax Outputs
    teacher_logits = teacher_model.transcribe(paths2audio_files=list_of_all_paths, batch_size=2, logprobs=True)
    # Getting Teacher Model's SAB Layer Outputs as Feature Maps
    teacher_feature_map = teacher_model.transcribe(paths2audio_files=list_of_all_paths, batch_size=2,
                                                   return_self_attention_outputs=True)
    # Getting Teacher Importance Map Outputs
    teacher_importance_map = teacher_model.transcribe(paths2audio_files=list_of_all_paths, batch_size=2,
                                                   return_importance_map=True)

    # Writing objects to files to persist them
    # Writing teacher_logits object to a file
    DIR = "/home/DATA2/apoorvaaggarwal/training"
    file = f"{DIR}/teacher_logits.pkl"
    file_obj = open(file, "wb")
    # teacher_logits = list(map(lambda pred: pred.cpu().detach().numpy(), teacher_logits))
    pickle.dump({
        list_of_all_paths[idx]: teacher_logit for idx, teacher_logit in enumerate(teacher_logits)
    }, file_obj)
    file_obj.close()

    # Writing teacher_feature_map object to a file
    file = f"{DIR}/teacher_feature_map.pkl"
    file_obj = open(file, "wb")  # write binary
    # teacher_feature_map = list(map(lambda pred: pred.cpu().detach().numpy(), teacher_feature_map))
    pickle.dump({
        list_of_all_paths[idx]: teacher_feature for idx, teacher_feature in enumerate(teacher_feature_map)
    }, file_obj)
    file_obj.close()

    # Writing teacher_importance_map object to a file
    file = f"{DIR}/teacher_importance_map.pkl"
    file_obj = open(file, "wb")  # write binary
    # teacher_importance_map = list(map(lambda pred: pred.cpu().detach().numpy(), teacher_importance_map))
    for idx, importance_map in enumerate(teacher_importance_map):
        print(f"index when dumping: {idx} file path: {list_of_all_paths[idx]}")

    pickle.dump({
        list_of_all_paths[idx]: importance_map for idx, importance_map in enumerate(teacher_importance_map)
    }, file_obj)
    file_obj.close()

    del teacher_model

    # Random sample should be printed in the output at each step, along with its predicted transcript.
    student_model._wer.log_prediction = True

    # Setting the trainer
    student_model.set_trainer(trainer)

    param_config = DictConfig(cfg['model'])
    student_model.setup_training_data(param_config.train_ds)
    student_model.setup_multiple_validation_data(val_data_config=param_config.validation_ds)
    student_model.setup_multiple_test_data(test_data_config=param_config.test_ds)
    student_model.spec_augmentation = student_model.from_config_dict(student_model.cfg.spec_augment)
    student_model.setup_optimization(DictConfig(cfg['model']['optim']))
    student_model.encoder.unfreeze()
    student_model.decoder.unfreeze()

    trainer.fit(student_model, ckpt_path=checkpoint_path)
    checkpoint_callback.best_model_path
    checkpoint_callback.best_model_score
    trainer.save_checkpoint

    student_model.save_to("/home/DATA2/apoorvaaggarwal/student_model.nemo")

    if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None and False:
        gpu = 1 if cfg.trainer.gpus != 0 else 0
        test_trainer = pl.Trainer(
            gpus=gpu,
            precision=trainer.precision,
            amp_level=trainer.accelerator_connector.amp_level,
            amp_backend=cfg.trainer.get("amp_backend", "native"),
        )
        if student_model.prepare_test(test_trainer):
            test_trainer.test(student_model)

if __name__ == '__main__':
    main()


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Wed Jun  6 08:02:06 2018

@author: eilers
"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import scipy.optimize as op
import time
import pickle
from astropy.table import Column, Table, join, vstack, hstack
import sys
from astropy.io import fits
from sklearn.decomposition import PCA
from astropy import units as u
from astropy.coordinates import SkyCoord
import astropy.coordinates as coord
from mpl_toolkits.mplot3d import Axes3D
import corner
from scipy.stats import binned_statistic_2d
import astropy

# -------------------------------------------------------------------------------
# plotting settings
# -------------------------------------------------------------------------------

matplotlib.rcParams['ytick.labelsize'] = 14
matplotlib.rcParams['xtick.labelsize'] = 14
fsize = 14

# -------------------------------------------------------------------------------
# open inferred labels
# -------------------------------------------------------------------------------

N = 45787
Kfold = 2
lam = 30
#name = 'N{0}_lam{1}_K{2}'.format(N, lam, Kfold)
name = 'N{0}_lam{1}_K{2}_mag_allcolors_offset'.format(N, lam, Kfold)

print('loading new labels...')
labels = Table.read('data/training_labels_new_{}_2.fits'.format(name), format = 'fits')
labels.rename_column('ra_1', 'ra')
labels.rename_column('dec_1', 'dec')

# -------------------------------------------------------------------------------
# calculate cartesian coordinates
# -------------------------------------------------------------------------------

spec_par = labels['spec_parallax'] * u.mas
distance = spec_par.to(u.parsec, equivalencies = u.parallax())

cs = coord.ICRS(ra = labels['ra'] * u.degree,
                dec = labels['dec'] * u.degree,
                distance = distance,
                pm_ra_cosdec = labels['pmra'] * u.mas/u.yr,
                pm_dec = labels['pmdec'] * u.mas/u.yr,
                radial_velocity = labels['VHELIO_AVG'] *u.km/u.s)

#Galactocentric position of the Sun:
X_GC_sun_kpc = 8.   #[kpc]
Z_GC_sun_kpc = 0.025 #[kpc] (e.g. Juric et al. 2008)

#circular velocity of the Galactic potential at the radius of the Sun:
vcirc_kms = 220. #[km/s] (e.g. Bovy 2015)

#Velocity of the Sun w.r.t. the Local Standard of Rest (e.g. Schoenrich et al. 2009):
U_LSR_kms = 11.1  # [km/s]
V_LSR_kms = 12.24 # [km/s]
W_LSR_kms = 7.25  # [km/s]

#Galactocentric velocity of the Sun:
vX_GC_sun_kms = -U_LSR_kms           # = -U              [km/s]
vY_GC_sun_kms =  V_LSR_kms+vcirc_kms # = V+v_circ(R_Sun) [km/s]
vZ_GC_sun_kms =  W_LSR_kms           # = W               [km/s]

# keep proper motion of Sgr A* constant!
vY_GC_sun_kms = X_GC_sun_kpc * vY_GC_sun_kms / 8.

gc = coord.Galactocentric(galcen_distance = X_GC_sun_kpc*u.kpc,
                          galcen_v_sun = coord.CartesianDifferential([-vX_GC_sun_kms, vY_GC_sun_kms, vZ_GC_sun_kms] * u.km/u.s),
                          z_sun = Z_GC_sun_kpc*u.kpc)

galcen = cs.transform_to(gc)
xs, ys, zs = galcen.x.to(u.kpc), galcen.y.to(u.kpc), galcen.z.to(u.kpc)
vxs, vys, vzs = galcen.v_x, galcen.v_y, galcen.v_z

XS = np.vstack([xs, ys, zs, vxs, vys, vzs]).T.value
Xlimits = [[-30, 10], [-10, 30], [-20, 20],
           [-200, 200], [-200, 200], [-200, 200]]
Xlabels = ['$x$', '$y$', '$z$', r'$v_x$', r'$v_y$', r'$v_z$']

d2d = np.sqrt(XS[:, 0] ** 2 + XS[:, 1] ** 2)
units = XS[:, 0:2] / d2d[:, None]
perps = np.zeros_like(units)
perps[:, 0] = units[:, 1]
perps[:, 1] = -units[:, 0]
vtans = np.sum(perps * XS[:, 3:5], axis=1)
R = np.sqrt(XS[:, 0] ** 2 + XS[:, 1] ** 2) # in cylindrical coordinates! # + XS[:, 2] ** 2)

# -------------------------------------------------------------------------------
# corner plot
# -------------------------------------------------------------------------------

fig = corner.corner(XS, range = Xlimits, labels = Xlabels)
fig.savefig('plots/corner.pdf')
plt.close()

# -------------------------------------------------------------------------------
# rings
# -------------------------------------------------------------------------------

def overplot_ring(r):
    tiny = 1e-4
    thetas = np.arange(0., 2*np.pi + tiny, 0.001 * np.pi)
    xs = r * np.cos(thetas)
    ys = r * np.sin(thetas)
    plt.plot(xs, ys, "k-", alpha=0.2, lw=1, zorder = -np.inf)
    return

def overplot_rings():
    for r in [5, 10, 15, 20, 25, 30]:
        overplot_ring(r)
    return

# -------------------------------------------------------------------------------
# theoretical rotation curves
# -------------------------------------------------------------------------------

def KeplerianRotation(R):
    G = astropy.constants.G
    M = 5.8e11 * astropy.constants.M_sun
    v = np.sqrt(G * M/(R * u.kpc))
    return v.to(u.km / u.s).value

# -------------------------------------------------------------------------------
# rotation curve
# -------------------------------------------------------------------------------

# take only stars in mid-plane
vz_cut = (abs(labels['b']) < 2) # * (abs(XS[:, 5]) < 20)

# -------------------------------------------------------------------------------
# other plots
# -------------------------------------------------------------------------------

## test to rotate system
#x = (np.random.random(size = 10000) -0.5) * 100
#y = (np.random.random(size = 10000) -0.5) * 100
#phi = np.arctan2(y, -x) - 60./360 * 2 * np.pi
#phi[phi < -np.pi] += 2. * np.pi
#fig, ax = plt.subplots(1, 1, figsize = (8, 8))
#sc = plt.scatter(x, y, c = phi, cmap = 'inferno', vmin = -np.pi, vmax = np.pi, s = 10)
#cb = plt.colorbar(sc, shrink = 0.82)
#cb.set_label(label = r'$\arctan2(y, -x)$', fontsize = fsize)
#plt.xlim(-50, 50)
#plt.ylim(-50, 50)
#overplot_rings()
#plt.tick_params(axis=u'both', direction='in', which='both')
#plt.xlabel('$x$', fontsize = fsize)
#plt.ylabel(r'$y$', fontsize = fsize)
#ax.set_aspect('equal')

# cylindrical coordinates
phi = np.arctan2(XS[vz_cut, 1], -XS[vz_cut, 0]) - 60./360 * 2.*np.pi # rotate by 60 degrees
phi[phi < -np.pi] += 2. * np.pi

fig, ax = plt.subplots(1, 1, figsize = (8, 8))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = phi, cmap = 'inferno', vmin = -2, vmax = 2, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$\varphi$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
overplot_rings()
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel(r'$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/rotation/xy_azimuth.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (10, 8))
sc = plt.scatter(R[vz_cut], vtans[vz_cut], c = phi, cmap = 'inferno', vmin = -2, vmax = 2, s = 10, rasterized = True)
cb = plt.colorbar(sc)
cb.set_label(label = r'$\varphi$', fontsize = fsize)
plt.xlim(0, 30)
plt.ylim(-200, 500)
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$R$', fontsize = fsize)
plt.ylabel(r'$v_{\rm tan}$', fontsize = fsize)
plt.savefig('plots/rotation/vtanR_sun{}kpc.pdf'.format(X_GC_sun_kpc), bbox_inches = 'tight', dpi = 120)
plt.close()

# cut in phi
cut_phi  = phi > -0.7
fig, ax = plt.subplots(1, 1, figsize = (10, 8))
sc = plt.scatter(R[vz_cut][cut_phi], vtans[vz_cut][cut_phi], c = phi[cut_phi], cmap = 'inferno', vmin = -2, vmax = 2, s = 10, rasterized = True)
cb = plt.colorbar(sc)
cb.set_label(label = r'$\varphi$', fontsize = fsize)
plt.xlim(0, 30)
plt.ylim(-200, 500)
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$R$', fontsize = fsize)
plt.ylabel(r'$v_{\rm tan}$', fontsize = fsize)
plt.title(r'$\langle \varphi \rangle = {}$'.format(round(np.median(phi[cut_phi]), 2)), fontsize = fsize)
plt.savefig('plots/rotation/vtanR_sun{}kpc_phicutbig.pdf'.format(X_GC_sun_kpc), bbox_inches = 'tight', dpi = 120)
plt.close()

cut_phi  = phi < -0.7
fig, ax = plt.subplots(1, 1, figsize = (10, 8))
sc = plt.scatter(R[vz_cut][cut_phi], vtans[vz_cut][cut_phi], c = phi[cut_phi], cmap = 'inferno', vmin = -2, vmax = 2, s = 10, rasterized = True)
cb = plt.colorbar(sc)
cb.set_label(label = r'$\varphi$', fontsize = fsize)
plt.xlim(0, 30)
plt.ylim(-200, 500)
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$R$', fontsize = fsize)
plt.ylabel(r'$v_{\rm tan}$', fontsize = fsize)
plt.title(r'$\langle \varphi \rangle = {}$'.format(round(np.median(phi[cut_phi]), 2)), fontsize = fsize)
plt.savefig('plots/rotation/vtanR_sun{}kpc_phicutsmall.pdf'.format(X_GC_sun_kpc), bbox_inches = 'tight', dpi = 120)
plt.close()

plt.hist(R[vz_cut], bins = np.linspace(0, 40, 100))
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('R', fontsize = 14)
plt.savefig('plots/rotation/hist_R.pdf', bbox_inches = 'tight')
plt.close()

stats, x_edge, y_edge, bins = binned_statistic_2d(R, vtans, values = np.ones_like(R), statistic = 'count', bins = 100, range = [[0, 30], [-100, 500]])
stats[stats < 5] = np.nan
sc = plt.imshow(stats.T, cmap = 'viridis_r', origin = 'lower', extent = (0, 30, -100, 500), aspect = 'auto')
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel(r'$R\,\rm [kpc]$', fontsize = fsize)
plt.ylabel(r'$v_{\rm tan}\,\rm [km\,s^{-1}]$', fontsize = fsize)
r_kep = np.linspace(0, 27, 100)
plt.plot(r_kep, 0.1 * KeplerianRotation(r_kep) , color = 'k')
plt.savefig('plots/rotation/vtanR_density_sun{0}kpc.pdf'.format(X_GC_sun_kpc), bbox_inches = 'tight')
plt.close()

# -------------------------------------------------------------------------------
# maps
# -------------------------------------------------------------------------------

fig, ax = plt.subplots(1, 1, figsize = (10, 10))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = XS[vz_cut, 3], cmap = 'RdBu', vmin = -200, vmax = 200, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$v_x$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_vx.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (10, 10))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = XS[vz_cut, 4], cmap = 'RdBu', vmin = -200, vmax = 200, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$v_y$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_vy.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (10, 10))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = XS[vz_cut, 5], cmap = 'RdBu', vmin = -100, vmax = 100, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$v_z$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_vz.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (10, 10))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = labels['parallax_error'][vz_cut]**2 / (0.1)**2, cmap = 'RdBu', vmin = 0, vmax = 2, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$\sigma^2_{\varpi,\,\rm Gaia}/\sigma^2_{\varpi,\,\rm inferred}$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_parallax.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (10, 10))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = labels['parallax_over_error'][vz_cut], cmap = 'RdBu', vmin = 0, vmax = 20, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$\varpi_{\rm Gaia}/\sigma_{\varpi, \rm Gaia}$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_parallax_over_error.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (12, 12))
plt.quiver(XS[vz_cut, 0], XS[vz_cut, 1], XS[vz_cut, 3], XS[vz_cut, 4],
           np.clip(XS[vz_cut, 5], -10, 10), cmap = 'RdBu', scale_units='xy',
           scale=200, alpha =.8, headwidth = 3, headlength = 4, width = 0.002)
cb = plt.colorbar(shrink = .85)
cb.set_label(r'$v_z$', fontsize = 15)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
overplot_rings()
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_arrow.pdf', bbox_inches = 'tight')
plt.close()

AKs = 0.918 * (labels['H'] - labels['w2mpro'] - 0.08)
HW2 = labels['H'] - labels['w2mpro']
fig, ax = plt.subplots(1, 1, figsize = (10, 10))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = AKs[vz_cut], cmap = 'RdBu', vmin = 0, vmax = 1, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$A(K_S) = 0.981(\rm H-W2 - 0.08)$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_ext.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (10, 10))
sc = plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = vtans[vz_cut], cmap = 'RdBu', vmin = -200, vmax = 200, s = 10, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$v_{\rm tan}$', fontsize = fsize)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$y$', fontsize = fsize)
overplot_rings()
ax.set_aspect('equal')
plt.savefig('plots/maps/xy_vtan.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

fig, ax = plt.subplots(1, 1, figsize = (8, 8))
plt.scatter(XS[:, 0], XS[:, 2], c = vtans, cmap = 'RdBu', vmin = -200, vmax = 200, s = 10, alpha = .2, rasterized = True)
cb = plt.colorbar(sc, shrink = 0.82)
cb.set_label(label = r'$v_{\rm tan}$', fontsize = fsize)
plt.xlim(-30, 30)
plt.ylim(-30, 30)
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = fsize)
plt.ylabel('$z$', fontsize = fsize)
ax.set_aspect('equal')
plt.savefig('plots/maps/xz_vtan.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

# radial velocity!
fig, ax = plt.subplots(1, 1, figsize = (10, 10))
plt.scatter(XS[vz_cut, 0], XS[vz_cut, 1], c = labels['VHELIO_AVG'][vz_cut], cmap = 'RdBu_r', vmin = -150, vmax = 150, s = 10, rasterized = True)
cb = plt.colorbar(shrink = .82)
cb.set_label(r'$v_{\rm rad}$', fontsize = 15)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
overplot_rings()
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = 14)
plt.ylabel('$y$', fontsize = 14)
ax.set_aspect('equal')
plt.savefig('plots/maps/xz_vrad.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

vz_cut_feh = vz_cut * (labels['FE_H'] > -10)
fig, ax = plt.subplots(1, 1, figsize = (10, 10))
plt.scatter(XS[vz_cut_feh, 0], XS[vz_cut_feh, 1], c = labels['FE_H'][vz_cut_feh], cmap = 'RdBu_r', vmin = -.5, vmax = .5, s = 10, rasterized = True)
cb = plt.colorbar(shrink = .82)
cb.set_label(r'$\rm [Fe/H]$', fontsize = 15)
plt.xlim(Xlimits[0])
plt.ylim(Xlimits[1])
overplot_rings()
plt.tick_params(axis=u'both', direction='in', which='both')
plt.xlabel('$x$', fontsize = 14)
plt.ylabel('$y$', fontsize = 14)
ax.set_aspect('equal')
plt.savefig('plots/maps/xz_feh.pdf', bbox_inches = 'tight', dpi = 120)
plt.close()

# -------------------------------------------------------------------------------
# 3D map
# -------------------------------------------------------------------------------

#fig = plt.figure(figsize = (10, 10))
#ax = fig.add_subplot(111, projection = '3d')
#ax.scatter(xs, ys, zs, c = labels['FE_H'], cmap = 'viridis_r', vmin = -3, vmax = 1)
#ax.set_xlabel('x', fontsize = 14)
#ax.set_ylabel('y', fontsize = 14)
#ax.set_zlabel('z', fontsize = 14)
#ax.set_ylim(-20000, 20000)
#ax.set_xlim(-20000, 20000)

# ------------------------------------------------------------------------------- '''


import pygame
import sys
from math import inf
from utils.parameters import WIDTH, HEIGHT, SQUARE_SIZE, RED, WHITE, GREEN, FPS
from utils.game import Game
from minimax.algorithm import alpha_beta_ending

# Create the main window
screen_one = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("Checkers")

# Create the Second Window
screen_two = pygame.display.set_mode((WIDTH, HEIGHT))

pygame.init()
pygame.font.init()

# Fonts
title_font = pygame.font.Font("assets/Cheri400.ttf", 100)
button_font = pygame.font.Font("assets/Cheri400.ttf", 28)

# Drop-down menu options
options = ["Basic Level", "Intermediate Level", "Advance Level"]
selected_option = None  # Initialize to None

def get_row_col_from_mouse(pos):
    """Convert mouse position to row and column on the board."""
    x, y = pos
    row = y // SQUARE_SIZE
    col = x // SQUARE_SIZE
    return row, col

def main():
    """Main function to run the game."""
    global selected_option
    run = True
    clock = pygame.time.Clock()
    game = Game(screen_two)
    depth = 2

    # Main loop
    running = True
    while running:
        screen_one.fill(GREEN)

        # Draw title
        title_text = title_font.render("Checker", True, RED)
        screen_one.blit(title_text, (WIDTH // 2 + 40 - title_text.get_width() // 2, 50))

        # Draw  three buttons for game levels
        pygame.draw.rect(screen_one, WHITE, (285, 250, 300, 70), 2)
        option_button1 = button_font.render(options[0], True, WHITE)
        screen_one.blit(option_button1, (360, 270))

        pygame.draw.rect(screen_one, WHITE, (285, 380, 300, 70), 2)
        option_button2 = button_font.render(options[1], True, WHITE)
        screen_one.blit(option_button2, (300, 400))

        pygame.draw.rect(screen_one, WHITE, (285, 500, 300, 70), 2)
        option_button3 = button_font.render(options[2], True, WHITE)
        screen_one.blit(option_button3, (330, 510))

        # Event handling for buttons
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()

            elif event.type == pygame.MOUSEBUTTONDOWN:
                if option_button1.get_rect(x=360, y=270).collidepoint(pygame.mouse.get_pos()):
                    print("Selected the Basic level")
                    selected_option = options[0]
                    running = False  # Exit the button handling loop
                elif option_button2.get_rect(x=300, y=400).collidepoint(pygame.mouse.get_pos()):
                    print("Selected the Intermediate level")
                    selected_option = options[1]
                    running = False  # Exit the button handling loop
                elif option_button3.get_rect(x=330, y=510).collidepoint(pygame.mouse.get_pos()):
                    print("Selected the Advances level")
                    selected_option = options[2]
                    running = False  # Exit the button handling loop

        # Update the display
        pygame.display.flip()
        # Cap the frame rate
        pygame.time.Clock().tick(FPS)

    # Game loop
    while run:
        clock.tick(FPS)
        if game.winner() is not None:
            print(game.winner())
            break

        if game.turn == WHITE:
            value, new_board = alpha_beta_ending(game.get_board(), depth, -inf, inf, WHITE, game, selected_option)
            game.ai_move(new_board)

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()

            if event.type == pygame.MOUSEBUTTONDOWN:
                pos = pygame.mouse.get_pos()
                row, col = get_row_col_from_mouse(pos)
                game.select(row, col)

        game.update()

    pygame.quit()

if __name__ == '__main__':
    main()


from django.urls import path, include
from wiki import views
from rest_framework.routers import SimpleRouter, DefaultRouter

app_name = 'wiki'

router = SimpleRouter()
# router = DefaultRouter()
router.register(r'posts', views.PostViewSet, basename='posts')
router.register(r'tags', views.TagViewSet, basename='tags')
router.register(r'categories', views.CategoryViewSet, basename='categories')
router.register(r'comments', views.CommentViewSet, basename='comments')

urlpatterns = [
    path('', include(router.urls)),
]


import re
from random import shuffle
import pickle
import os
import discord
#permissions 201329664

bot = discord.Client()

names = ["somthing went wrong"]

optIN = {}
optOut = {}

@bot.event
async def on_ready():
    global names
    global optIN
    print('Bot ready!')
    print(bot.user.name)
    print('-------')
    if os.path.exists('names.p'):
        names = pickle.load(open("names.p", "rb"))
    else:
        names = ["DadBot"]*20

    if os.path.exists('optIN.p'):
        optIN = pickle.load(open("optIN.p", "rb"))
    else:
        optIN = {}

    if os.path.exists('optOut.p'):
        optOut = pickle.load(open("optOut.p", "rb"))
    else:
        optOut = {}

@bot.event
async def on_guild_join(guild):
    await guild.system_channel.send("Hi "+guild.name+", I'm DadBot!")

@bot.event
async def on_message(message):
    global names
    global optIN
    global optOut
    try:
        if message.author == message.guild.me:
            return

        if message.content.lower() == "optin":
            if not optIN.get(message.author.id, False):
                if not message.guild.me.top_role.permissions.manage_nicknames:
                    await message.channel.send("I cannot change nicknames, but I'll put you on the list.")
                elif message.author.top_role > message.guild.me.top_role:
                    await message.channel.send("You outrank me, but I'll put you on the list.")
                optIN[message.author.id] = True
                pickle.dump(optIN, open("optIN.p", "wb"))

                optOut[message.author.id] = message.author.display_name
                pickle.dump(optOut, open("optOut.p", "wb"))
            return

        if message.content.lower() == "optout":
            if optIN.get(message.author.id, False):
                optIN.pop(message.author.id)
                pickle.dump(optIN, open("optIN.p", "wb"))
                if optOut[message.author.id] != message.author.display_name:
                    await message.author.edit(nick=optOut[message.author.id], reason="DadBot")
            return

        word = re.search(r'\bi.?m\s+(.*)', message.content, re.IGNORECASE)
        if word is None:
            return
        if len(word.group(1)) > 32:
            word = re.search(r'\bi.?m\s+(\w+)', message.content, re.IGNORECASE)
        word = word.group(1)

        if word.lower() == "DadBot".lower():
            await message.channel.send("Wait, you're me?")
            return

        if word.lower() == "Dad".lower():
            await message.channel.send("Wait, you're me?")
            return

        shuffle(names)
        await message.channel.send("Hi "+word+", I'm " + names.pop() + "!")
        names.append(word)
        pickle.dump(names, open("names.p", "wb"))

        if optIN.get(message.author.id, True):
            if not message.guild.me.top_role.permissions.manage_nicknames:
                #await message.channel.send(
                #    "Somebody named "+message.guild.owner.display_name+" won't let me change names.")
                return
            if message.author.top_role > message.guild.me.top_role:
                #if message.author == message.guild.owner:
                #    await message.channel.send(
                #        "Somebody named "+message.author.display_name+
                #        " won't let me change their name.")
                #else:
                #    await message.channel.send(
                #        "Somebody named "+message.author.display_name+
                #        " won't let me change their name."
                #        " (It might be "+message.guild.owner.display_name+"'s fault)")
                return
            if message.author == message.guild.owner:
                return
            if len(word) > 32:
                word = word[:27] + "..."
            await message.author.edit(nick=word, reason="DadBot")

    except Exception as exc:
        print(repr(exc))
        await message.channel.send(
            "Something went wrong. It is probably "+message.author.display_name+"'s fault.")

bot.run(open("TOKEN", "r").read().rstrip())


import sqlite3

base_url = "/home/shiyanlou/Desktop/shiyanlou_spider.sqlite3.db"
table_name = "course_info"

def create_table():
    sql3_db = sqlite3.connect(base_url)
    create_sql = "create table {} (url varchar(1024), title varchar(256), teacher varchar(128), study_num int, tag varchar(256), types varchar(256), info varchar(1024), tests_name varchar(1024))".format(table_name)
    try:
        sql3_db.execute(create_sql)
    except:
        return False
    sql3_db.close()
    return True

def insert_or_update_data(url, title, teacher, study_num, tag, types, info, tests_name):
    create_table() # 创建一次数据库，没有表格时创建，表格存在时不作操作
    if query(title): # 如果查询到title存在，则是更新
        sql3_db = sqlite3.connect(base_url)
        update_sql = "update {} set study_num={}, info='{}', tests_name='{}', url='{}' where title='{}'".format(table_name, study_num, info, tests_name, url, title)
        try:
            sql3_db.execute(update_sql)
            sql3_db.commit()
        except:
            return False
        sql3_db.close()
        return True
    else: # 查询不到title，则插入
        sql3_db = sqlite3.connect(base_url)
        insert_sql = "insert into {} (url, title, teacher, study_num, tag, types, info, tests_name) values('{}', '{}', '{}', {}, '{}', '{}', '{}', '{}')".format(table_name,url, title, teacher, study_num, tag, types, info, tests_name)
        try:
            sql3_db.execute(insert_sql)
            sql3_db.commit()
        except:
            return False
        sql3_db.close()
        return True
    return False

def query(title):
    sql3_db = sqlite3.connect(base_url)
    query_sql = "select * from {} where title = '{}'".format(table_name, title)
    cu = sql3_db.cursor()
    cu.execute(query_sql)
    record_list = cu.fetchall()
    if len(record_list)>0:
        return True
    else:
        return False
    return False

if __name__=="__main__":
    insert_or_update_data('url','title','teacher',12138,'vip','web','infos_infos','课程一 课程二')


from django.shortcuts import render
from django.views.generic import FormView, DetailView
from django.http import JsonResponse
from conDE.settings import MEDIA_ROOT, MEDIA_URL, RSCRIPT_PATH, LOCAL_TEST, RPLOTS_PATH, PYTHON_PATH, PATH_TO_CONSENSUS_SCRIPT
import pandas as pd
import os
import subprocess
import plotly.plotly as py
import plotly.graph_objs as go
from plotly.offline import plot
import json
from random import randrange
import random
import string
import shutil

# Create your views here.

# def DEresult(request):
#     results = {}
#     if 'id' in request.GET:
#         job_id = request.GET['id']
#         results["job_id"] = job_id
#
#     return render(request, "DE_result.html", results)

def update_json(folder_path, FC=None,pval=None,iset=None,methods=None):
    path_to_config = os.path.join(folder_path,"config.json")
    with open(path_to_config) as f:
        config = json.load(f)
    changed = False
    if FC != config["FC"]:
        config["FC"] = FC
        changed = True
    if pval != config["pval"]:
        config["pval"] = pval
        changed = True

    config["set"] = iset

    if set(methods) != set(config["methods"]):
        config["methods"] = methods
        changed = True

    with open(path_to_config, 'w') as f:
        json.dump(config, f)
    return changed

def select_DE(path,pval,FC,out):
    with open(path,"r") as in_path:
        df = pd.read_csv(in_path,sep="\t")

    FC = float(FC)
    pval = float(pval)
    if FC < 1:
        FC= 1/float(FC)
    k1 = df.loc[(df.pvalue < pval) & ((df.FoldChange > FC) | (df.FoldChange < 1/float(FC)))]
    k1.to_csv(out, sep='\t', index=False)
    return k1['name'].tolist()

def calculate_consensus(directory,methods,pval,FC):
    import itertools
    import pandas
    con_list=[]
    shutil.rmtree(os.path.join(directory, "consensus"))
    os.mkdir(os.path.join(directory, "consensus"))
    for method in methods:
        out_dir = os.path.join(directory,"consensus",method+".txt")
        de_dir = os.path.join(directory,"de",method)
        in_file = [f for f in os.listdir(de_dir) if f.endswith("allGenes.csv")][0]
        in_path = os.path.join(directory,"de",method,in_file)
        con_list.append(select_DE(in_path,pval,FC,out_dir))
    merged = list(itertools.chain.from_iterable(con_list))
    ps = pandas.Series(merged)
    counts = ps.value_counts()
    with open(os.path.join(directory,"consensus.txt"),"w") as con:
        for i, val in counts.iteritems():
            if val >= len(methods):
                con.write(i+"\n")

def draw_barplot(path_to_config):
    with open(path_to_config) as f:
        config = json.load(f)
    n_over = {}
    n_under = {}
    random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
    for method in config["methods"]:
        in_file = os.path.join(config["folder"],"consensus", method+".txt")
        with open(in_file, "r") as in_path:
            df = pd.read_csv(in_path, sep="\t")
        n_under[method] = len(df.loc[(df.FoldChange < 1 )]["name"].tolist())
        n_over[method] = len(df.loc[(df.FoldChange > 1 )]["name"].tolist())

    data = []
    if (config["set"] =="All") or (config["set"] =="Over"):
        over = go.Bar(
            x= list(n_over.keys()) ,
            y= list(n_over.values()),
            marker_color= '#2ca02c',
            marker=dict(line=dict(width = 1 ,color="black")),

            name='Number of overexpressed')
        data.append(over)
    if (config["set"] =="All") or (config["set"] =="Under"):
        under = go.Bar(
            x= list(n_under.keys()) ,
            y= list(n_under.values()),
            marker_color='crimson',
            marker=dict(line=dict(width=1, color="black")),
            name='Number of underexpressed')
        data.append(under)
    layout = go.Layout(
            barmode='stack'
        )
    fig = go.Figure(data=data, layout=layout)
    bar_files = [os.path.join(config["folder"], "plots",f) for f in os.listdir(os.path.join(config["folder"], "plots")) if f.endswith("BarPlot.html")]
    if bar_files:
        os.remove(bar_files[0])
    # bar_plot = [os.path.join(config["folder"], "plots",f) for f in os.listdir(os.path.join(config["folder"], "plots")) if f.endswith("BarPlot.html")][0]

    out_path = os.path.join(config["folder"],"plots",random_string+"_BarPlot.html")
    plot(fig, filename=out_path,show_link=False, auto_open=False, include_plotlyjs=True)
    return out_path.replace(MEDIA_ROOT,MEDIA_URL)
    # return out_path
    # return plot(fig, show_link=False, auto_open=False, output_type='div', include_plotlyjs=True)

def consensusToJson(jobID):
    input_table = os.path.join(MEDIA_ROOT,jobID,"consensus.tsv")
    with open(input_table,"r") as table_file:
        lines = table_file.readlines()
        headers = lines[0].rstrip().split("\t")

        body_list = []
        for line in lines[1:]:
            body_list.append(line.rstrip().split("\t"))

        heads = []
        for header in headers:
            heads.append({"title": header})
    return heads,body_list
    # return json.dumps(headers),json.dumps(body_list)

def methodToJson(jobID,method):
    input_table = os.path.join(MEDIA_ROOT,jobID,"consensus",method+".txt")
    with open(input_table,"r") as table_file:
        lines = table_file.readlines()
        headers = lines[0].rstrip().split("\t")

        body_list = []
        for line in lines[1:]:
            body_list.append(line.rstrip().split("\t"))

        heads = []
        for header in headers:
            heads.append({"title": header})
    return heads,body_list

def draw_plots(path_to_config):
    wait_list = []
    call_list = [RSCRIPT_PATH, os.path.join(RPLOTS_PATH,"upset.R"),  path_to_config]

    if not LOCAL_TEST:
        wait_list.append(subprocess.Popen(call_list,
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE))
        call_list = [RSCRIPT_PATH, os.path.join(RPLOTS_PATH,"VennD.R"), path_to_config]
        wait_list.append(subprocess.Popen(call_list,
                                          stdout=subprocess.PIPE,
                                          stderr=subprocess.PIPE))
    call_list = [PYTHON_PATH, PATH_TO_CONSENSUS_SCRIPT, path_to_config ]
    wait_list.append(subprocess.Popen(call_list,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE))
    barplot_url = draw_barplot(path_to_config)
    exit_codes = [p.wait() for p in wait_list]
    folders = path_to_config.split("/")
    temp = "/".join(folders[:-1])
    with open(os.path.join(temp,"temp"),"w") as tmp:
        tmp.write(" ".join(call_list))
    return barplot_url

def get_plot_content(request):
    data = {}
    plot = request.GET.get('plot', None).replace(" ","_")
    id = request.GET.get('id', None)
    plot_path = os.path.join(MEDIA_ROOT,id,"plots",plot+".jpg")
    if os.path.exists(plot_path):
        media_plot = plot_path.replace(MEDIA_ROOT,MEDIA_URL)
        div_content = '

 '
    else:
        bar_plot = [f for f in os.listdir(os.path.join(MEDIA_ROOT,id,"plots")) if f.endswith("BarPlot.html")][0]
        div_content = ''

    data["div_content"] = div_content
    return JsonResponse(data)

class DEresult(FormView):
    #template_name = 'bench.html'
    #form_class = sRNABenchForm
    #success_url = reverse('photos:multi_start')

    def get_form_kwargs(self):
        '''This goes in the Update view'''
        kwargs = super(DEresult, self).get_form_kwargs()  # put your view name in the super

        #kwargs["folder"] = self.request.path
        return kwargs

    def get(self, request,**kwargs):
        path = request.path
        folder = path.split("/")[-1]
        folder_path = os.path.join(MEDIA_ROOT,folder)
        if not os.path.exists(folder_path):
            return render(self.request, "result_not_found.html", {"jobId": folder})
        for p in ["consensus","plots"]:
            to_make = os.path.join(folder_path,p)
            if not os.path.exists(to_make):
                os.mkdir(to_make)

        de_path = os.path.join(folder_path,"de")
        method_list = [f for f in os.listdir(de_path) if os.path.isdir(os.path.join(de_path, f))]
        method_list.sort()
        # if os.path.exists(os.path.join(folder_path,"config.json")):
        #     update_json(folder_path, FC=2, pval=0.05, iset="All", methods=method_list)
        # else:
        #
        to_config = {"folder": folder_path,  "methods": method_list,
        "pval": 0.05, "FC": 2,  "set": "All"}
        with open(os.path.join(folder_path,"config.json"),"w") as cj:
            json.dump(to_config,cj)
        # calculate_consensus(folder_path, method_list, 0.05, 2)
        # method_list = ["edgeR", "DESeq", "DESeq2","NOISeq"]
        plot_list = [["UpSet","UpSet"],["Barplot","Barplot"],["Venn","Venn Diagram"]]
        set_list = [["All","All DE genes"],["Over","Overexpressed only"],["Under","Underexpressed only"]]
        calculate_consensus(folder_path,method_list,0.05,2)
        draw_plots(os.path.join(folder_path,"config.json"))
        # con_head,con_body = consensusToJson(folder)
        start_plot = os.path.join(folder_path,"plots","UpSet.jpg").replace(MEDIA_ROOT,MEDIA_URL)

        #add links

        zip_file = os.path.join(folder_path, "de.zip")

        wait_list = []
        call_list = ["zip", "-r", zip_file, os.path.join(folder_path,"de","*")]
        wait_list.append(subprocess.Popen(call_list,
                                              stdout=subprocess.PIPE,
                                              stderr=subprocess.PIPE))

        full_de_link = zip_file.replace(MEDIA_ROOT,MEDIA_URL)
        unselected_link = os.path.join(folder_path,"de","DESeq","allGenes.csv").replace(MEDIA_ROOT,MEDIA_URL)
        selected_link = os.path.join(folder_path,"consensus","DESeq.txt").replace(MEDIA_ROOT,MEDIA_URL)

        return render(self.request, 'DE_result.html',
                      {"job_id": folder,
                       "method_list" : method_list,
                       "plot_list" : plot_list,
                       "start_plot": start_plot,
                       "set_list": set_list,
                       "full_de_link": full_de_link,
                       "unselected_link": unselected_link,
                       "selected_link": selected_link,
                       # "con_head": con_head,
                       # "con_body": con_body
                       })

def ajax_recalculate(request):
    data = {}
    id = request.GET.get('id', None)
    FC = request.GET.get('FC', None)
    pval = request.GET.get('pval', None)
    methods = request.GET.get('methods', None)
    methods = methods.split(",")
    iset = request.GET.get('set', None)
    folder = os.path.join(MEDIA_ROOT, id )
    if update_json(folder, FC=FC,pval=pval,iset=iset,methods=methods):
        calculate_consensus(folder, methods, pval, FC)
    barplot_url = draw_plots(os.path.join(folder, "config.json"))

    data["div_content"] = ''

    return JsonResponse(data)

def ajax_consensus(request):
    data = {}
    job = request.GET.get('id', None)
    con_head, con_body = consensusToJson(job)
    data["header"] = con_head
    data["body"] = con_body
    return JsonResponse(data)

def ajax_individual(request):
    data = {}
    job = request.GET.get('id', None)
    folder_path = os.path.join(MEDIA_ROOT,job)
    method = request.GET.get('method', None)
    con_head, con_body = methodToJson(job,method)
    data["header"] = con_head
    data["body"] = con_body
    data["method"] = method

    data["unselected_link"]  = os.path.join(folder_path, "de", method, "allGenes.csv").replace(MEDIA_ROOT, MEDIA_URL)
    data["selected_link"] = os.path.join(folder_path, "consensus", method + ".txt").replace(MEDIA_ROOT, MEDIA_URL)

    return JsonResponse(data)


#!/usr/bin/env python3
"""
This workflow pulls data from the sampledata/acl_data.yaml file.
This workflow performs the following steps:
1. Detach ACL from VLAN
   Ex:
    vlan 200
        no apply access-list ip acl_ipv4 in

2. Delete IPv4 ACL
    a. Delete entries in ACL
    b. Delete the ACL
   Ex:
    no access-list ip acl_ipv4

3. Delete IPv6 ACL
    a. Delete entries in ACL
    b. Delete the ACL
   Ex:
    no access-list ipv6 acl_ipv6

4. Delete MAC ACL
    a. Delete entries in ACL
    b. Delete the ACL
   Ex:
    no access-list mac acl_mac

5. Reset interfaces
   Ex:
    interface 1/1/10
        shutdown
        routing
    interface 1/1/11
        shutdown
        routing
        no apply access-list ip acl_ipv4 in
    interface 1/1/12
        shutdown
        routing
        no apply access-list ipv6 acl_ipv6 in
    interface 1/1/21
        shutdown
        no lag 13
    interface 1/1/22
        shutdown
        no lag 13
    no interface lag 13

Preconditions:
Must have run configure_acl workflow or have an equivalent configuration on the device
"""

from requests.packages.urllib3.exceptions import InsecureRequestWarning
import requests
import os
import sys

dirpath = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
sys.path.append(dirpath)
sys.path.append(os.path.join(dirpath, "src"))
sys.path.append(os.path.join(dirpath, "cx_utils"))

from cx_utils import yaml_ops
from src import session, acl, vlan, interface, lag, system

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def main():
    data = yaml_ops.read_yaml("acl_data.yaml")

    if not data['switchip']:
        data['switchip'] = input("Switch IP Address: ")

    if data['bypassproxy']:
        os.environ['no_proxy'] = data['switchip']
        os.environ['NO_PROXY'] = data['switchip']

    if not data['version']:
        data['version'] = "v10.04"

    base_url = "https://{0}/rest/{1}/".format(data['switchip'], data['version'])
    try:
        session_dict = dict(s=session.login(base_url, data['username'], data['password']), url=base_url)
        session_dict['platform_name'] = system.get_system_info(**session_dict).get('platform_name')

        # Clear Egress ACLs from L3 interface
        acl.clear_interface_acl(data['L3egressinterface'], acl_type="aclv4_out", **session_dict)

        # Clear Ingress ACLs from L2 interface
        acl.clear_interface_acl(data['ipv4L2ingressinterface'], acl_type="aclv4_in", **session_dict)
        acl.clear_interface_acl(data['ipv6L2ingressinterface'], acl_type="aclv6_in", **session_dict)

        # Clear Ingress ACLs from LAG interface
        acl.clear_interface_acl(data['LAGname'], acl_type="aclv4_in", **session_dict)

        # Detach ACL from VLAN
        vlan.detach_vlan_acl(data['aclVLANid'], "ipv4", **session_dict)

        # Remove and initialize L2 and L3 interfaces
        interface.initialize_interface(data['L3egressinterface'], **session_dict)
        interface.initialize_interface(data['ipv4L2ingressinterface'], **session_dict)
        interface.initialize_interface(data['ipv6L2ingressinterface'], **session_dict)
        interface.initialize_interface(data['interfaceVLAN'], **session_dict)

        # Remove LAG and initialize associated L2 interfaces
        lag.delete_lag_interface(data['LAGname'], data['LAGinterfaces'], **session_dict)
        for LAGinterface in data['LAGinterfaces']:
            interface.initialize_interface(LAGinterface, **session_dict)

        # Delete VLAN
        vlan.delete_vlan(data['aclVLANid'], **session_dict)

        # For each ACL that was configured
        for pair_dict in [{"name": data['ipv4aclname'], "type": "ipv4"},
                          {"name": data['ipv6aclname'], "type": "ipv6"},
                          {"name": data['macaclname'], "type": "mac"}]:

            # Delete ACL entries
            for i in range(10, 60, 10):
                acl.delete_acl_entry(pair_dict["name"], pair_dict["type"], i, **session_dict)

            # Delete the ACL
            acl.delete_acl(pair_dict["name"], pair_dict["type"], **session_dict)

    except Exception as error:
        print('Ran into exception: {}. Logging out..'.format(error))

    session.logout(**session_dict)

if __name__ == '__main__':
    main()


# reference: https://github.com/thuml/OpenDG-DAML

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.hub import load_state_dict_from_url
from torchvision.models.resnet import model_urls
from torch.nn.parameter import Parameter

Parameter.fast = None

class Linear_fw(nn.Linear):
    def __init__(self, in_features, out_features):
        super(Linear_fw, self).__init__(in_features, out_features)

    def forward(self, x):
        if self.weight.fast is not None and self.bias.fast is not None:
            out = F.linear(x, self.weight.fast,
                           self.bias.fast)
        else:
            out = super(Linear_fw, self).forward(x)
        return out

class Conv2d_fw(nn.Conv2d):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=True):
        super(Conv2d_fw, self).__init__(in_channels, out_channels, kernel_size, stride=stride, padding=padding,
                                        bias=bias)

    def forward(self, x):
        if self.bias is None:
            if self.weight.fast is not None:
                out = F.conv2d(x, self.weight.fast, None, stride=self.stride, padding=self.padding)
            else:
                out = super(Conv2d_fw, self).forward(x)
        else:
            if self.weight.fast is not None and self.bias.fast is not None:
                out = F.conv2d(x, self.weight.fast, self.bias.fast, stride=self.stride, padding=self.padding)
            else:
                out = super(Conv2d_fw, self).forward(x)

        return out

class BatchNorm2d_fw(nn.BatchNorm2d):
    def __init__(self, num_features):
        super(BatchNorm2d_fw, self).__init__(num_features)

    def forward(self, input):
        self._check_input_dim(input)

        if self.momentum is None:
            exponential_average_factor = 0.0
        else:
            exponential_average_factor = self.momentum

        if self.training and self.track_running_stats:
            if self.num_batches_tracked is not None:
                self.num_batches_tracked = self.num_batches_tracked + 1
                if self.momentum is None:
                    exponential_average_factor = 1.0 / float(self.num_batches_tracked)
                else:
                    exponential_average_factor = self.momentum

        """ Decide whether the mini-batch stats should be used for normalization rather than the buffers.
        Mini-batch stats are used in training mode, and in eval mode when buffers are None.
        """
        if self.training:
            bn_training = True
        else:
            bn_training = (self.running_mean is None) and (self.running_var is None)

        """Buffers are only updated if they are to be tracked and we are in training mode. Thus they only need to be
        passed when the update should occur (i.e. in training mode when they are tracked), or when buffer stats are
        used for normalization (i.e. in eval mode when buffers are not None).
        """

        if self.weight.fast is not None and self.bias.fast is not None:
            return F.batch_norm(
            input,
            self.running_mean if not self.training or self.track_running_stats else None,
            self.running_var if not self.training or self.track_running_stats else None,
            self.weight.fast, self.bias.fast, bn_training, exponential_average_factor, self.eps)
        else:
            return F.batch_norm(
                input,
                self.running_mean if not self.training or self.track_running_stats else None,
                self.running_var if not self.training or self.track_running_stats else None,
                self.weight, self.bias, bn_training, exponential_average_factor, self.eps)

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(BasicBlock, self).__init__()
        self.conv1 = Conv2d_fw(in_channels=inplanes, out_channels=planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn1 = BatchNorm2d_fw(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = Conv2d_fw(in_channels=planes, out_channels=planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = BatchNorm2d_fw(planes)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(
        self,
        inplanes,
        planes,
        stride=1,
        downsample=None,
    ):
        super(Bottleneck, self).__init__()
        self.conv1 = Conv2d_fw(in_channels=inplanes, out_channels=planes, kernel_size=1, stride=1, bias=False)
        self.bn1 = BatchNorm2d_fw(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = Conv2d_fw(in_channels=planes, out_channels=planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn2 = BatchNorm2d_fw(planes)
        self.conv3 = Conv2d_fw(in_channels=planes, out_channels=planes*self.expansion, kernel_size=1, stride=1, bias=False)
        self.bn3 = BatchNorm2d_fw(planes*self.expansion)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class ResNetFast(nn.Module):
    def __init__(self, block, layers):
        self.inplanes = 64
        super(ResNetFast, self).__init__()
        self.conv1 = Conv2d_fw(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = BatchNorm2d_fw(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)

        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.fc = nn.Linear(512 * block.expansion, 1000)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                Conv2d_fw(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                BatchNorm2d_fw(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        x = self.avgpool(x)
        x = torch.flatten(x, 1)

        return x

class ConvNet(nn.Module):
    def __init__(self):
        super(ConvNet, self).__init__()
        self.conv1 = Conv2d_fw(3, 64, kernel_size=3, stride=1, padding=1)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=2)
        self.conv2 = Conv2d_fw(64, 64, kernel_size=3, stride=1, padding=1)
        self.conv3 = Conv2d_fw(64, 64, kernel_size=3, stride=1, padding=1)
        self.conv4 = Conv2d_fw(64, 64, kernel_size=3, stride=1, padding=1)

        self._out_features = 256

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def _check_input(self, x):
        H, W = x.shape[2:]
        assert (
            H == 32 and W == 32
        ), "Input to network must be 32x32, " "but got {}x{}".format(H, W)

    def forward(self, x):
        self._check_input(x)
        x = self.conv1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.conv2(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.conv3(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.conv4(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = torch.flatten(x, 1)

        return x

class MutiClassifier(nn.Module):
    def __init__(self, net, num_classes, feature_dim=512):
        super(MutiClassifier, self).__init__()
        self.net = net
        self.num_classes = num_classes
        self.classifier = Linear_fw(feature_dim, self.num_classes)
        self.b_classifier = Linear_fw(feature_dim, self.num_classes*2)
        nn.init.xavier_uniform_(self.classifier.weight, .1)
        nn.init.constant_(self.classifier.bias, 0.)
        nn.init.xavier_uniform_(self.b_classifier.weight, .1)
        nn.init.constant_(self.b_classifier.bias, 0.)

    def forward(self, x):
        x = self.net(x)
        x = self.classifier(x)
        return x

    def b_forward(self, x):
        x = self.net(x)
        x = self.b_classifier(x)
        return x

    def c_forward(self, x):
        x = self.net(x)
        x1 = self.classifier(x)
        x2 = self.b_classifier(x)
        return x1, x2

class MutiClassifier_(nn.Module):
    def __init__(self, net, num_classes, feature_dim=512):
        super(MutiClassifier_, self).__init__()
        self.net = net
        self.num_classes = num_classes
        self.b_classifier = Linear_fw(feature_dim, self.num_classes*2)
        nn.init.xavier_uniform_(self.b_classifier.weight, .1)
        nn.init.constant_(self.b_classifier.bias, 0.)

    def forward(self, x):
        x = self.net(x)
        x = self.b_classifier(x)
        x = x.view(x.size(0), 2, -1)
        x = x[:, 1, :]

        return x

    def b_forward(self, x):
        x = self.net(x)
        x = self.b_classifier(x)
        return x

    def c_forward(self, x):
        x = self.net(x)
        x2 = self.b_classifier(x)
        x1 = x2.view(x.size(0), 2, -1)
        x1 = x1[:, 1, :]
        return x1, x2

def resnet18_fast(progress=True):
    """ResNet-18 model from
    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_

    Parameters:
        - **pretrained** (bool): If True, returns a model pre-trained on ImageNet
        - **progress** (bool): If True, displays a progress bar of the download to stderr
    """
    model = ResNetFast(BasicBlock, [2, 2, 2, 2])
    state_dict = load_state_dict_from_url(model_urls['resnet18'],
                                          progress=progress)
    model.load_state_dict(state_dict, strict=False)
    del model.fc

    return model

def resnet50_fast(progress=True):
    model = ResNetFast(Bottleneck, [3, 4, 6, 3])
    state_dict = load_state_dict_from_url(model_urls['resnet50'],
                                          progress=progress)
    model.load_state_dict(state_dict, strict=False)
    del model.fc

    return model


from copy import deepcopy
import numpy as np
import torch
from torch.utils.data import Dataset

from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from tokenizers.pre_tokenizers import Whitespace
from transformers import T5Tokenizer

from .build import DATASETWRAPPER_REGISTRY

class BaseTokenizer:
    def __init__(self, vocab, bos_token='<s>', eos_token='</s>', sep_token='<###>', pad_token='<pad>'):
        self.vocab = [pad_token, bos_token, eos_token, sep_token] + vocab
        self.w2i = {w: i for i, w in enumerate(self.vocab)}
        self.bos_token = bos_token
        self.eos_token = eos_token
        self.sep_token = sep_token
        self.pad_token = pad_token
        tokenizer = Tokenizer(WordLevel(vocab=self.w2i))
        tokenizer.pre_tokenizer = Whitespace()
        tokenizer.add_special_tokens([pad_token, bos_token, eos_token, sep_token])
        self.tokenizer = tokenizer

    def encode(self, text, pad_direction='right'):
        if isinstance(text, str):
            text = [text]
        self.tokenizer.enable_padding(direction=pad_direction, pad_id=self.vocab.index(self.pad_token), pad_token=self.pad_token)
        outputs = self.tokenizer.encode_batch(text)
        ids = torch.LongTensor([output.ids for output in outputs])
        masks = torch.LongTensor([output.attention_mask for output in outputs])
        max_len = len(outputs[0])
        return max_len, ids, masks

    def decode(self, ids):
        pass

@DATASETWRAPPER_REGISTRY.register()
class GPTWrapper(Dataset):
    def __init__(self, cfg, dataset):
        self.dataset = dataset
        self.tokenizer = BaseTokenizer(dataset.vocab_input + dataset.vocab_output)
        self.use_cot = cfg.use_cot

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, index):
        return self.dataset[index]

    def collate_fn(self, batch):
        bos_token, eos_token, sep_token = self.tokenizer.bos_token, self.tokenizer.eos_token, self.tokenizer.sep_token

        def concat_sentence(sample):
            sentence = [bos_token] + sample['input']
            if self.use_cot:
                for thought in sample['cot']:
                    sentence += [sep_token] + thought
            sentence += [sep_token] + sample['output'] + [eos_token]
            return ' '.join(sentence)

        _, concat_ids, concat_masks = self.tokenizer.encode([concat_sentence(sample) for sample in batch], pad_direction='right')
        _, input_ids, input_masks = self.tokenizer.encode([' '.join([bos_token] + sample['input'] + [sep_token])
                                                for sample in batch], pad_direction='left') # padding left to avoid the padding token in the middle of the sequence for generation.
        _, output_ids, output_masks = self.tokenizer.encode([' '.join(sample['output'] + [eos_token])
                                                for sample in batch], pad_direction='right')

        new_batch = {}
        for key in batch[0].keys():
            new_batch[key] = [sample[key] for sample in batch]

        new_batch['input_ids'] = input_ids
        new_batch['input_masks'] = input_masks
        new_batch['output_ids'] = output_ids
        new_batch['output_masks'] = output_masks
        new_batch['concat_ids'] = concat_ids
        new_batch['concat_masks'] = concat_masks
        return new_batch

@DATASETWRAPPER_REGISTRY.register()
class T5Wrapper(Dataset):
    def __init__(self, cfg, dataset):
        self.dataset = dataset
        self.tokenizer = T5Tokenizer.from_pretrained(cfg.model.variant)
        self.input_types = getattr(cfg, 'input_types', 'input').split('-')
        self.output_types = getattr(cfg, 'output_types', 'output').split('-')

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, index):
        return self.dataset[index]

    def collate_fn(self, batch):

        def tokenize_batch(types, delimiter=' ; '):
            sequences = []
            for sample in batch:
                seq = ''
                for tp in types:
                    if tp == 'input':
                        seq += 'input: ' + (sample['input'] if isinstance(sample['input'], str) else ' '.join(sample['input']))
                    elif tp == 'tree':
                        seq += 'tree: ' + sample['tree']
                    elif tp == 'steps':
                        seq += 'steps: ' + ' , '.join([f'{i} = {o}' for i, o in sample['steps']])
                    elif tp == 'results':
                        seq += 'results: ' + ' , '.join([f'{i} = {o}' for i, o in sample['results']])
                    elif tp == 'rir':
                        seq += 'intermediate: ' + sample['rir']
                    elif tp == 'output':
                        seq += 'output: ' + (sample['output'] if isinstance(sample['output'], str) else ' '.join(sample['output']))
                    else:
                        assert False, f'Unknown type: {tp}'
                    seq += delimiter
                sequences.append(seq)
            encoding = self.tokenizer(
                sequences,
                padding="longest",
                truncation=False,
                max_length=1024,
                return_tensors="pt",
            )
            assert self.tokenizer.unk_token_id not in encoding.input_ids, 'Some tokens are not recognized by the tokenizer!'
            return encoding.input_ids, encoding.attention_mask

        input_ids, input_masks = tokenize_batch(self.input_types)
        output_ids, output_masks = tokenize_batch(self.output_types)
        output_ids[output_ids == self.tokenizer.pad_token_id] = -100

        new_batch = {}
        for key in batch[0].keys():
            new_batch[key] = [sample[key] for sample in batch]

        new_batch['input_ids'] = input_ids
        new_batch['input_masks'] = input_masks
        new_batch['output_ids'] = output_ids
        new_batch['output_masks'] = output_masks
        return new_batch


from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from nltk.tokenize import RegexpTokenizer
from sklearn import preprocessing
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras import utils
import pandas as pd
import numpy as np
import joblib

# tokenizer to remove unwanted elements from out data like symbols and numbers
from sklearn.utils import compute_class_weight

df = pd.read_csv('Movie_Metadata_Sentiments.csv')
# Subset only emotions required to get overall emotion detected from the text content
sub_df = df[['anger', 'joy', 'fear', 'sadness']]
# Label the movie with the highest count of emotions
df['Max'] = sub_df.idxmax(axis=1)
token = RegexpTokenizer(r'[a-zA-Z0-9]+')
cv = CountVectorizer(lowercase=True, stop_words='english', ngram_range=(1, 1), tokenizer=token.tokenize)
cv = cv.fit(df['Text_Content'])
text_counts = cv.transform(df['Text_Content'])
# Save the vectorizer
joblib.dump(cv, "vectorizer.pkl")

X_train, X_test, y_train, y_test = train_test_split(
    text_counts, df['Max'], test_size=0.2, random_state=1)

# Neural Network
encoder = preprocessing.LabelEncoder()
encoder.fit(y_train)
print(encoder.classes_)
y_train = encoder.transform(y_train)
y_test = encoder.transform(y_test)

# Resolves the imbalance in dataset
class_weights = compute_class_weight('balanced', np.unique(y_train), y_train)

class_weights_dict = dict(zip(encoder.transform(list(encoder.classes_)), class_weights))
print(class_weights_dict)

num_classes = np.max(y_train) + 1
y_train = utils.to_categorical(y_train, num_classes)
y_test = utils.to_categorical(y_test, num_classes)
print(y_train)
print(y_train.shape)

batch_size = 64
epochs = 3

# Build the model
model = Sequential()
model.add(Dense(256, input_shape=(X_train.shape[1],), activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(4, activation='softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

history = model.fit(X_train, y_train,
                    batch_size=batch_size,
                    epochs=epochs,
                    verbose=1,
                    validation_split=0.1,
                    class_weight=class_weights_dict)

score = model.evaluate(X_test, y_test,
                       batch_size=batch_size, verbose=1)
print('Test accuracy:', score[1])

# Prints the Classification Report
y_pred = model.predict(X_test)
print(classification_report(y_test.argmax(axis=1), y_pred.argmax(axis=1), target_names=encoder.classes_))

# Export the Model
model.save('Movie_Metadata_Sentiments_Weighted_Keras.h5', history)


#!/usr/bin/env python3

from ase.cluster.icosahedron import Icosahedron
from ase.calculators.lj import LennardJones as ase_LJ
import numpy as np
import pytest

from pysisyphus.calculators.LennardJones import LennardJones
from pysisyphus.constants import BOHR2ANG
from pysisyphus.Geometry import Geometry
from pysisyphus.helpers import geom_loader
from pysisyphus.optimizers.RFOptimizer import RFOptimizer

def test_lennard_jones():
    atoms = Icosahedron("Ar", noshells=2, latticeconstant=3)
    atoms.calc = ase_LJ()
    ase_forces = atoms.get_forces()
    ase_energy = atoms.get_potential_energy()

    coords = atoms.positions.flatten()
    geom = Geometry(atoms.get_chemical_symbols(), coords / BOHR2ANG)
    geom.set_calculator(LennardJones())

    pysis_energy = geom.energy
    assert pysis_energy == pytest.approx(ase_energy)

    pysis_forces = geom.forces / BOHR2ANG
    np.testing.assert_allclose(pysis_forces, ase_forces.flatten(), atol=1e-15)

@pytest.mark.parametrize("max_micro_cycles, cur_cycle", ((0, 110), (25, 108)))
def test_ar_cluster(max_micro_cycles, cur_cycle):
    geom = geom_loader("lib:ar14cluster.xyz")
    geom.set_calculator(LennardJones())

    opt_kwargs = {
        "max_cycles": 150,
        "gediis": True,
        "thresh": "gau_vtight",
        "max_micro_cycles": max_micro_cycles,
    }
    opt = RFOptimizer(geom, **opt_kwargs)
    opt.run()

    assert geom.energy == pytest.approx(-43.63972413)
    assert opt.is_converged
    assert opt.cur_cycle == cur_cycle


from models import Teacher, Student
from flask_restful import Resource, reqparse
from config import app, db, api

class Teachers(Resource):
    def post(self):
        data = request.get_json()
        if not data:
            return {"message": "No input data provided"}, 400
        try:
            new_teacher = Teacher(name=name, age=age)
        except ValueError as e:
            return {"message": f"Error creating teacher: {e}"}, 400

        db.session.add(new_teacher)
        db.session.commit()

        return new_teacher.to_dict(), 201

api.add_resource(Teachers, '/teacher')

if __name__ == '__main__':
    app.run(port=5555, debug=True)


#!/usr/bin/env python3

import subprocess
from dataclasses import dataclass
from pathlib import Path
from tempfile import NamedTemporaryFile
import argparse
import os
import uuid
import hashlib
import sys
from typing import Dict, List

from calibre.utils.logging import Log
from calibre.customize.conversion import OptionRecommendation
from calibre.ebooks.conversion.plumber import Plumber
from calibre.ebooks.conversion.plugins.mobi_output import MOBIOutput
from calibre.ebooks.conversion.plugins.epub_output import EPUBOutput

import frontmatter

ROOT = Path(__file__).parent.resolve()

@dataclass
class Options:
    source_files: List[Path]
    output_dir: Path
    cover_dir: Path

def parse_args() -> Options:
    parser = argparse.ArgumentParser()
    help = "File to import rather than default.nix. Examples, ./release.nix"
    parser.add_argument("source_files", nargs="+", default="./.")
    parser.add_argument("--covers", default="./.")
    parser.add_argument("-o", "--output", default="./.")
    args = parser.parse_args()
    return Options(
        source_files=[Path(s) for s in args.source_files],
        cover_dir=Path(args.covers),
        output_dir=Path(args.output)
    )

def convert_document(source: Path, target: Path, cover_dir: Path):
    post = frontmatter.load(source)
    log = Log()

    title = post.get("title", "Untitled")
    author = post.get("creator", "Shannan Lekwati")

    args = [
        ("authors", author),
        ("language", post.get("lang", "en")),
        ("title", title),
    ]

    date = post.get("date")
    if date:
        args += [
            ("pubdate", str(post["date"])),
            ("timestamp", str(post["date"]))
        ]

    summary = post.get("summary")
    if summary:
        args += [ ("comments", summary) ]

    cover_image = post.get("cover", {}).get("image")
    if cover_image:
        cover_path = cover_dir.joinpath(cover_image).absolute()
        if cover_path.exists():
            args += [ ("cover", str(cover_path)) ]
        else:
            print(f"WARNING: {cover_image} in {source} does not exists", file=sys.stderr)

    with NamedTemporaryFile(suffix=source.suffix, mode="w") as f:
        f.write(f"# {title} by **{author}** \n")
        if "dedication" in post:
            f.write(f"{dedication}\n")
        f.write(post.content)
        f.flush()

        plumber = Plumber(f.name, target, log)
        recommendations = [(k, v, OptionRecommendation.HIGH) for (k,v) in args]
        plumber.merge_ui_recommendations(recommendations)
        plumber.run()

def main() -> None:
    opts = parse_args()

    for source_file in opts.source_files:
        target = opts.output_dir.joinpath(source_file.stem + ".epub")
        convert_document(source_file, target, opts.cover_dir)
        target = opts.output_dir.joinpath(source_file.stem + ".mobi")
        convert_document(source_file, target, opts.cover_dir)

if __name__ == "__main__":
    main()


import os
import sys
import json
import pickle
import random
from PIL import Image
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt

import torch
from torch.utils.data import Dataset
import torch.nn.functional as F

def read_split_data(root: str, DatasetName: str, val_rate: float = 0.2, plot_image: bool = False):
    random.seed(0)  # 保证随机结果可复现
    assert os.path.exists(root), "dataset root: {} does not exist.".format(root)

    # 遍历文件夹，一个文件夹对应一个类别
    data_class = [cla for cla in os.listdir(root) if os.path.isdir(os.path.join(root, cla))]
    # 排序，保证顺序一致
    data_class.sort()
    # 生成类别名称以及对应的数字索引
    class_indices = dict((k, v) for v, k in enumerate(data_class))
    json_str = json.dumps(class_indices)
    with open('{}_class_indices.json'.format(DatasetName), 'w') as json_file:
        json_file.write(json_str)

    train_images_path = []  # 存储训练集的所有图片路径
    train_images_label = []  # 存储训练集图片对应索引信息
    val_images_path = []  # 存储验证集的所有图片路径
    val_images_label = []  # 存储验证集图片对应索引信息
    every_class_num = []  # 存储每个类别的样本总数
    supported = [".jpg", ".JPG", ".png", ".PNG",".bmp"]  # 支持的文件后缀类型
    # 遍历每个文件夹下的文件
    for cla in data_class:
        cla_path = os.path.join(root, cla)
        # 遍历获取supported支持的所有文件路径
        images = [os.path.join(root, cla, i) for i in os.listdir(cla_path)
                  if os.path.splitext(i)[-1] in supported]
        # 获取该类别对应的索引
        image_class = class_indices[cla]
        # 记录该类别的样本数量
        every_class_num.append(len(images))
        # 按比例随机采样验证样本
        val_path = random.sample(images, k=int(len(images) * val_rate))

        for img_path in images:
            if img_path in val_path:  # 如果该路径在采样的验证集样本中则存入验证集
                val_images_path.append(img_path)
                val_images_label.append(image_class)
            else:  # 否则存入训练集
                train_images_path.append(img_path)
                train_images_label.append(image_class)

    print("{} images were found in the dataset.".format(sum(every_class_num)))
    print("{} images for training.".format(len(train_images_path)))
    print("{} images for validation.".format(len(val_images_path)))

    if plot_image:
        # 绘制每种类别个数柱状图
        plt.bar(range(len(flower_class)), every_class_num, align='center')
        # 将横坐标0,1,2,3,4替换为相应的类别名称
        plt.xticks(range(len(flower_class)), flower_class)
        # 在柱状图上添加数值标签
        for i, v in enumerate(every_class_num):
            plt.text(x=i, y=v + 5, s=str(v), ha='center')
        # 设置x坐标
        plt.xlabel('image class')
        # 设置y坐标
        plt.ylabel('number of images')
        # 设置柱状图的标题
        plt.title('data class distribution')
        plt.show()

    return train_images_path, train_images_label, val_images_path, val_images_label

def load_all_imgs(root: str):
    assert os.path.exists(root), "dataset root: {} does not exist.".format(root)
    images_path = {}
    supported = [".jpg", ".JPG", ".png", ".PNG", ".bmp"]
    data_class = [cla for cla in os.listdir(root) if os.path.isdir(os.path.join(root, cla))]
    data_class.sort()
    for cla in data_class:
        cla_path = os.path.join(root,cla)
        images = [os.path.join(root, cla, i) for i in os.listdir(cla_path)
                  if os.path.splitext(i)[-1] in supported]
        assert os.path.exists(cla_path), "file: '{}' dose not exist.".format(cla_path)
        images_path[cla] = images
        print("total {} images finding for predict class {}.".format(len(images_path[cla]),cla))
    return images_path

def plot_data_loader_image(data_loader):
    batch_size = data_loader.batch_size
    plot_num = min(batch_size, 4)

    json_path = './class_indices.json'
    assert os.path.exists(json_path), json_path + " does not exist."
    json_file = open(json_path, 'r')
    class_indices = json.load(json_file)

    for data in data_loader:
        images, labels = data
        for i in range(plot_num):
            # [C, H, W] -> [H, W, C]
            img = images[i].numpy().transpose(1, 2, 0)
            # 反Normalize操作
            img = (img * [0.229, 0.224, 0.225] + [0.485, 0.456, 0.406]) * 255
            label = labels[i].item()
            plt.subplot(1, plot_num, i+1)
            plt.xlabel(class_indices[str(label)])
            plt.xticks([])  # 去掉x轴的刻度
            plt.yticks([])  # 去掉y轴的刻度
            plt.imshow(img.astype('uint8'))
        plt.show()

def write_pickle(list_info: list, file_name: str):
    with open(file_name, 'wb') as f:
        pickle.dump(list_info, f)

def read_pickle(file_name: str) -> list:
    with open(file_name, 'rb') as f:
        info_list = pickle.load(f)
        return info_list

def train_one_epoch(model, optimizer, data_loader, device, epoch):
    model.train()
    loss_function = torch.nn.CrossEntropyLoss()
    accu_loss = torch.zeros(1).to(device)  # 累计损失
    accu_num = torch.zeros(1).to(device)   # 累计预测正确的样本数
    optimizer.zero_grad()

    sample_num = 0
    data_loader = tqdm(data_loader, file=sys.stdout, colour="green")
    for step, data in enumerate(data_loader):
        images, labels = data
        sample_num += images.shape[0]

        pred = model(images.to(device))
        pred_classes = torch.max(pred, dim=1)[1]
        accu_num += torch.eq(pred_classes, labels.to(device)).sum()

        loss = loss_function(pred, labels.to(device))

        loss.backward()
        accu_loss += loss.detach()

        data_loader.desc = "[train epoch {}] loss: {:.3f}, acc: {:.3f}".format(epoch,
                                                                               accu_loss.item() / (step + 1),
                                                                               accu_num.item() / sample_num)

        if not torch.isfinite(loss):
            print('WARNING: non-finite loss, ending training ', loss)
            sys.exit(1)

        optimizer.step()
        optimizer.zero_grad()

    return accu_loss.item() / (step + 1), accu_num.item() / sample_num

@torch.no_grad()
def evaluate(model, data_loader, device, epoch):
    loss_function = torch.nn.CrossEntropyLoss()

    model.eval()

    accu_loss = torch.zeros(1).to(device)  # 累计损失
    accu_num = torch.zeros(1).to(device)  # 累计预测正确的样本数
    sample_num = 0

    data_loader = tqdm(data_loader, file=sys.stdout)
    for step, data in enumerate(data_loader):
        images, labels = data
        sample_num += images.shape[0]

        pred = model(images.to(device))
        pred_classes = torch.max(pred, dim=1)[1]
        accu_num += torch.eq(pred_classes, labels.to(device)).sum()

        loss = loss_function(pred, labels.to(device))
        accu_loss += loss

        data_loader.desc = "[valid epoch {}] loss: {:.3f}, acc: {:.3f}".format(epoch,
                                                                               accu_loss.item() / (step + 1),
                                                                               accu_num.item() / sample_num)

    return accu_loss.item() / (step + 1), accu_num.item() / sample_num

@torch.no_grad()
def Test_A_Dataset(model, data_loader, classes_name, args):
    device = torch.device(args.device if torch.cuda.is_available() else "cpu")
    model.eval()
    cls_num = len(classes_name)
    conf_mat = np.zeros([cls_num, cls_num])  #混淆矩阵

    data_loader = tqdm(data_loader, file = sys.stdout, colour = 'red')
    for step, data in enumerate(data_loader):
        images, labels = data

        pred = model(images.to(device))
        pred_classes = torch.max(pred, dim = 1)[1]

        for i in range(len(labels)):
            cate_i = labels[i]
            pre_i = pred_classes[i]
            conf_mat[cate_i, pre_i] += 1.0

    for i in range(cls_num):
        print('class:{:<10}, total num:{:<6}, correct num:{:<5}  Recall: {:.2%} Precision: {:.2%}'.format(
            classes_name[i], np.sum(conf_mat[i, :]), conf_mat[i, i], conf_mat[i, i] / (1 + np.sum(conf_mat[i, :])),
                                                                     conf_mat[i, i] / (1 + np.sum(conf_mat[:, i]))))

    print('dataset Accuracy:{:.2%}'.format( np.trace(conf_mat) / np.sum(conf_mat)))
    return conf_mat, '{:.2}'.format(np.trace(conf_mat) / np.sum(conf_mat))

def show_confMat(confusion_mat, classes, args):

    # 归一化
    confusion_mat_N = confusion_mat.copy()
    for i in range(len(classes)):
        confusion_mat_N[i, :] = confusion_mat[i, :] / confusion_mat[i, :].sum()

    # 获取颜色
    cmap = plt.cm.get_cmap('Blues')  # 更多颜色: http://matplotlib.org/examples/color/colormaps_reference.html
    plt.imshow(confusion_mat_N, cmap=cmap)
    plt.colorbar()

    # 设置文字
    xlocations = np.array(range(len(classes)))
    plt.xticks(xlocations, list(classes), rotation=-45)
    plt.yticks(xlocations, list(classes))
    plt.xlabel('Predict label')
    plt.ylabel('True label')
    plt.title('Confusion_Matrix_last')

    # 打印数字
    for i in range(confusion_mat_N.shape[0]):
        for j in range(confusion_mat_N.shape[1]):
            plt.text(x=j, y=i, s=int(confusion_mat[i, j]), va='center', ha='center', color='red', fontsize=10)
    # 保存
    if os.path.exists(args.conf_Mat_dir) is False:  os.makedirs(args.conf_Mat_dir)
    plt.tight_layout()
    plt.savefig(os.path.join(args.conf_Mat_dir, 'Confusion_Matrix_last' + args.model_type + '.png'))
    plt.close()

class MyDataSet(Dataset):
    """自定义数据集"""

    def __init__(self, images_path: list, images_class: list, transform=None):
        self.images_path = images_path
        self.images_class = images_class
        self.transform = transform

    def __len__(self):
        return len(self.images_path)

    def __getitem__(self, item):
        img = Image.open(self.images_path[item])
        if img.mode != "RGB":
            img=img.convert("RGB")
        # RGB为彩色图片，L为灰度图片
        # if img.mode != 'RGB':
        #     raise ValueError("image: {} isn't RGB mode.".format(self.images_path[item]))
        label = self.images_class[item]

        if self.transform is not None:
            img = self.transform(img)
        return img, label


from __future__ import print_function
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os.path
import PIL as Image
import PIL.ImageDraw
from PIL import Image, ImageOps
import numpy

def border_one(original_image, percent_of_side=.3):
    """ Rounds the corner of a PIL.Image
        original_image must be a PIL.Image
        Returns a new PIL.Image with rounded corners, where
        0 < percent_of_side < 1 is the corner radius as a portion of the shorter
        dimension of original_image
    """
    #set the radius of the rounded corners
    width, height = original_image.size

    ###
    #create a mask
    ###

    #start with transparent mask
    border_mask = PIL.Image.new('RGBA', (width, height), (127,0,127,0))
    drawing_layer = PIL.ImageDraw.Draw(border_mask)

    # Overwrite the RGBA values with A=255.
    # The 127 for RGB values was used merely for visualizing the mask

       # Draw two rectangles to fill interior with opaqueness
    if width >= 300 or height >= 300:
        drawing_layer.polygon([(0,0),(width, 0),
                                (width, height),(0,height)],
                                fill=(127,0,127,170))

        drawing_layer.polygon([(0,0),(30, 0),
                      (30, height),(0,height)],
                      fill=(0,0,255,0),outline=(0,255,0,1))
        drawing_layer.polygon([(width, 0),(width-30,0),
                      (width-30, height),(width,height)],
                      fill=(0,0,255,0),outline=(0,255,0,1))
        drawing_layer.polygon([(0, 0),(width,0),
                      (width,30),(0, 30)],
                      fill=(0,0,255,0),outline=(0,255,0,1))
        drawing_layer.polygon([(0, height),(width,height),
                      (width,height-30),(0, height-30)],
                      fill=(0,0,255,0),outline=(0,255,0,1))

        # Make the new image, starting with all transparent
        result = PIL.Image.new('RGBA', original_image.size, (136,39,16,255))
        result.paste(original_image, (0,0), mask=border_mask)

        img = Image.open('ourlogo_test.png')
        logo_small = img.resize((85,93))
        result.paste(logo_small, (width-130,40), mask = logo_small)

        return result

    elif width <= 300 and height <= 300:

        original_image.size = 500, 500

        drawing_layer.polygon([(0,0),(width, 0),
                                (width, height),(0,height)],
                                fill=(127,0,127,170))

        drawing_layer.polygon([(0,0),(20, 0),
                      (20, height),(0,height)],
                      fill=(0,0,255,0),outline=(0,255,0,1))
        drawing_layer.polygon([(width, 0),(width-20,0),
                      (width-20, height),(width,height)],
                      fill=(0,0,255,0),outline=(0,255,0,1))
        drawing_layer.polygon([(0, 0),(width,0),
                      (width,20),(0, 20)],
                      fill=(0,0,255,0),outline=(0,255,0,1))
        drawing_layer.polygon([(0, height),(width,height),
                      (width,height-30),(0, height-30)],
                      fill=(0,0,255,0),outline=(0,255,0,1))

        # Make the new image, starting with all transparent
        result = PIL.Image.new('RGBA', original_image.size, (136,39,16,255))
        result.paste(original_image, (0,0), mask=border_mask)

        img = Image.open('ourlogo_test.png')
        logo_small = img.resize((10,23))
        result.paste(logo_small, (width-130,40), mask = logo_small)

        return result

def multi_border(originalimg, new):
    ''' Module for adding borders to images in folders '''

    img = Image.open(originalimg) #opens original image
    #adds border to original
    img_with_border1 = ImageOps.expand(img, border=30, fill='blue')
    img_with_border2 = ImageOps.expand(img_with_border1, border=30,fill='white')
    img_with_border3 = ImageOps.expand(img_with_border2, border=30, fill='red')

    return img_with_border3

def user_color(original, new):
    '''allows user to change border colors from set options'''
    img = Image.open(original) #opens original image
    #adds border to original
    color = raw_input('What color would you like the border to be?(Prompt \
will appear until all images are have a chosen color.): \n')
    time.sleep(1)
    thic = raw_input('What width would you like the border to be?(Prompt \
will appear until all images are have a chosen width.): \n')
    try:
        img_with_border = ImageOps.expand(img, border=int(thic), fill=str(color))
    except ValueError:
        img_with_border = ImageOps.expand(img, border=30, fill=str("blue"))
    try:
        img_with_border = ImageOps.expand(img, border=int(thic), fill=str(color))
    except TypeError:
        img_with_border = ImageOps.expand(img, border=30, fill=str("blue"))

    return img_with_border

def get_images(directory=None):
    """ Returns PIL.Image objects for all the images in directory.

    If directory is not specified, uses current directory.
    Returns a 2-tuple containing
    a list with a  PIL.Image object for each image file in root_directory, and
    a list with a string filename for each image file in root_directory
    """

    if directory == None:
        directory = os.getcwd() # Use working directory if unspecified

    image_list = [] # Initialize aggregaotrs
    file_list = []

    directory_list = os.listdir(directory) # Get list of files
    for entry in directory_list:
        absolute_filename = os.path.join(directory, entry)
        try:
            image = PIL.Image.open(absolute_filename)
            file_list += [entry]
            image_list += [image]
        except IOError:
            pass # do nothing with errors tying to open non-images
    return image_list, file_list

def usercolorIMG(directory=None):
    """ Saves a modfied version of each image in directory.

    Uses current directory if no directory is specified.
    Places images in subdirectory 'modified', creating it if it does not exist.
    New image files are of type PNG and have transparent rounded corners.
    """

    if directory == None:
        directory = os.getcwd() # Use working directory if unspecified

    # Create a new directory 'modified'
    new_directory = os.path.join(directory, 'modified')
    try:
        os.mkdir(new_directory)
    except OSError:
        pass # if the directory already exists, proceed

    # Load all the images
    image_list, file_list = get_images(directory)

    # Go through the images and save modified versions
    for n in range(len(image_list)):
        # Parse the filename
        print(n)
        filename, filetype = os.path.splitext(file_list[n])

        # Round the corners with default percent of radius
        curr_image = image_list[n]
        new_image = user_color(file_list[n],curr_image)

        # Save the altered image, suing PNG to retain transparency
        new_image_filename = os.path.join(new_directory, filename + '.png')
        new_image.save(new_image_filename)

print('Welcome to the Freshman Cars INC. image modifier!')
time.sleep(1)
pic_option = raw_input('What modification would you like? grayscale(1), \
border(2), multi border(3), distorted(4) or coolmask(5) *type the numbers that \
correlate with the specific modification*: ')

if pic_option == "1":
    usercolorIMG()


from typing import Any, Optional

from fastapi import HTTPException

from core.errors import BaseBusinessError

class HTTPCustomError(HTTPException):
    __slots__ = (
        "status_code",
        "detail",
        "headers",
        "_business_error",
    )

    def __init__(
        self,
        status_code: int,
        business_error: BaseBusinessError,
        detail: Optional[Any] = None,
        headers: Optional[dict[str, Any]] = None,
    ) -> None:
        super().__init__(status_code, detail, headers)
        self._business_error = business_error

    @property
    def business_error(self) -> BaseBusinessError:
        return self._business_error


import itertools
import collections
n = int(input())
a=list(map(int, input().split()))
operator = ['+','-','*','%']
operatorNum = list(map(int, input().split()))
operatorReal = []
for i in range(len(operatorNum)):
    operatorReal+=[operator[i]]*operatorNum[i]
opCombi = collections.deque(list(set(itertools.permutations(operatorReal,sum(operatorNum)))))
history=[]
while opCombi:
    travel = opCombi.popleft()
    aa=a[0]
   # print(a,aa, travel)
    for i in range(len(travel)):
            if travel[i]=='+':
                aa+=a[i+1]
            elif travel[i]=='-':
                aa-=a[i+1]
            elif travel[i]=='*':
                aa*=a[i+1]
            else:
                if aa<0:
                    aa=abs(aa)//a[i+1]
                    aa*=(-1)
                else:
                    aa//=a[i+1]
    if aa not in history:
        history.append(aa)
history.sort()
print(history[-1])
print(history[0])


import numpy as np
from PIL import Image

def load_train(wnids, dataset_path, shape):
	"""
        Load the train files  dataset

        Parameters
        ----------
        dataset_path : str
        	path to the dataset files

        shape : tuple
        	target shape of the loaded instances

        Returns
        -------
        x_train : np.array
            training instances
        y_train : np.array
            training labels
    """

	x_train = np.ndarray(shape = (100000, 32, 32, 3), dtype = np.uint8)
	y_train = np.ndarray(shape = (100000), dtype = np.uint8)
	for idx, wnid in enumerate(wnids):
		for j in range(500):
			im = Image.open('%s/train/%s/images/%s_%d.JPEG' % (dataset_path, wnid, wnid, j)).convert('RGB')

			if shape != (64, 64):
				im = im.resize((32, 32), Image.LANCZOS)

			x_train[idx*500+j] = np.asarray(im)
			y_train[idx*500+j] = idx

	return x_train, y_train

def load_test(wnids, dataset_path, shape):
	"""
        Load the test files  dataset

        Parameters
        ----------
        dataset_path : str
        	path to the dataset files

        shape : tuple
        	target shape of the loaded instances

        Returns
        -------
        x_test : np.array
            testing instances
        x_test : np.array
            testing labels
    """

	x_test = np.ndarray(shape = (10000, 32, 32, 3), dtype = np.uint8)
	y_test = np.ndarray(shape = (10000), dtype = np.uint8)

	for i, line in enumerate(map(lambda s: s.strip(), open('%s/val/val_annotations.txt' % dataset_path))):
		name, wnid = line.split('\t')[:2]

		im = Image.open('%s/val/images/%s' % (dataset_path, name)).convert('RGB')

		if shape != (64, 64):
			im = im.resize((32, 32), Image.LANCZOS)

		x_test[i] = np.asarray(im)
		y_test[i] = wnids.index(wnid)

	return x_test, y_test

def load_tiny_imagenet(dataset_path, shape):
	"""
        Load the tiny-imagenet dataset

        Parameters
        ----------
        dataset_path : str
        	path to the dataset files

        shape : tuple
        	target shape of the loaded instances

        Returns
        -------
        x_train : np.array
            training instances
        y_train : np.array
            training labels
        x_test : np.array
            testing instances
        x_test : np.array
            testing labels
    """

	wnids = map(lambda x: x.strip(), open('%s/wnids.txt' % dataset_path).readlines())

	x_train, y_train = load_train(wnids, dataset_path, shape)
	x_test, y_test = load_test(wnids, dataset_path, shape)

	return x_train, y_train, x_test, y_test