import numpy as np
import matplotlib.pyplot as plt

from scipy.integrate import solve_ivp
from scipy.optimize import minimize

from scipy.interpolate import interp1d

#  Vector format
#   x[0] = x position
#   x[1] = x velocity
#   x[2] = y position
#   x[3] = y velocity

#  For 30 degrees, when does it reach its max height?
#  For 30 degrees, how far does the ball travel in the x-direction?

def no_drag(t, X):
    Xprime = np.zeros(4)

    Xprime[0] = X[1]
    Xprime[1] = 0
    Xprime[2] = X[3]
    Xprime[3] = -9.8

    return(Xprime)

def events(t, X):
    return X[2] - 0

events.terminal = True

#  Calculates the entir trajectory to some specified end time
def calculate_trajectory(t_end):
    m = 0.145               #  mass in kg

    v0 = 45
    theta = 30; theta = np.radians(theta)
    X0 = np.array( [ 0, v0 * np.cos(theta), 1, v0 * np.sin(theta) ] )

    tspan = (0, t_end)
    t_eval = np.linspace(tspan[0], tspan[1], 100)

    sol = solve_ivp(no_drag, tspan, X0, method = 'RK45', t_eval = t_eval,
        events = events)

    t_evt = sol.t_events[0][0]

    sol2 = solve_ivp(no_drag, tspan, X0, method = 'RK45', t_eval = t_eval)
    t  = sol2.t
    x  = sol2.y[0,:]
    vx = sol2.y[1,:]
    y  = sol2.y[2,:]
    vy = sol2.y[3,:]

    #  Use a cubic spline to interpolate too get values at t_evt
    sp_x  = interp1d(t, x, kind = 'cubic')
    sp_vx = interp1d(t, vx, kind = 'cubic')
    sp_y  = interp1d(t, y, kind = 'cubic')
    sp_vy = interp1d(t, vy, kind = 'cubic')

    #  Place data in new array
    new_data = np.array( [
        sp_x(t_evt),
        sp_vx(t_evt),
        sp_y(t_evt),
        sp_vy(t_evt) ] )

    #  The data needs to be explicitly shaped into a column vector
    new_data.shape = (4, 1)

    #  Append the data to our original solution
    sol.t = np.append(sol.t, t_evt)
    sol.y = np.append(sol.y, new_data, 1)

    return sol

#  From Sympu:  R = 180.664729951254
#  From numeric integration with tf = 5 and N = 100
#  Event Time =  [array([4.63585913])]
#  Numerical range =  179.10979941905433
#  Error =  1.554930532199677

sol = calculate_trajectory( 5 )
x = sol.y[0,:]
y = sol.y[2,:]

plt.plot(x, y, '.')
plt.grid(True)
plt.xlabel('Distance (m)')
plt.ylabel('Height (m)')
plt.show()


from typing import Callable, List

import pytest

from twoSumAllAnswers import Solution

solution = Solution()

@pytest.mark.parametrize(
    "fn",
    (
        (solution.twoSumAlbert),
        (solution.twoSumDorian),
    ),
)
@pytest.mark.parametrize(
    "nums, target, want",
    (
        (
            [1, 1, 1, 1],
            2,
            [
                (0, 1),
                (0, 2),
                (0, 3),
                (1, 2),
                (1, 3),
                (2, 3),
            ],
        ),
        (
            [1, 2, 2, 1, 2, 1],
            2,
            [(0, 3), (0, 5), (3, 5)],
        ),
    ),
)
def test_twoSumAllAnswers(fn: Callable, nums: List[int], target: int, want: List[str]):
    got = fn(nums, target)
    got = sorted(tuple(sorted((a, b))) for a, b in got)
    assert got == want

@pytest.mark.benchmark()
@pytest.mark.parametrize(
    "fn",
    (
        (solution.twoSumAlbert),
        (solution.twoSumDorian),
    ),
)
@pytest.mark.parametrize(
    "nums, target",
    (
        ([1] * 100, 2),
        ([1] * 100, 0),
    ),
)
def test_benchmark_twoSumAllAnswers(
    benchmark, fn: Callable, nums: List[int], target: int
):
    benchmark(fn, nums, target)


"""This module contains an event to perform actions related to ROP render
scripts.

"""

# =============================================================================
# IMPORTS
# =============================================================================

# Standard Library
import logging
import time
from typing import Optional

# Houdini Toolbox
from houdini_toolbox.events.group import HoudiniEventGroup
from houdini_toolbox.events.item import HoudiniEventItem
from houdini_toolbox.events.types import RopEvents

# Houdini
import hou

_logger = logging.getLogger(__name__)

# =============================================================================
# CLASSES
# =============================================================================

class RopRenderEvent(HoudiniEventGroup):
    """Event to run on ROP render script events."""

    def __init__(self) -> None:
        super().__init__()

        self._frame_start = None
        self._render_start = None

        self.event_map.update(
            {
                RopEvents.PreRender: HoudiniEventItem((self.pre_render,)),
                RopEvents.PreFrame: HoudiniEventItem((self.pre_frame,)),
                RopEvents.PostFrame: HoudiniEventItem((self.post_frame,)),
                RopEvents.PostRender: HoudiniEventItem((self.post_render,)),
                RopEvents.PostWrite: HoudiniEventItem((self.post_write,)),
            }
        )

    # -------------------------------------------------------------------------
    # METHODS
    # -------------------------------------------------------------------------

    def pre_frame(self, scriptargs: dict) -> None:
        """Action run before the frame starts rendering.

        :param scriptargs: Event data.
        :return:

        """
        # Store the time as the "start time" so it can be referenced after the
        # frame is completed to get the duration.
        self._frame_start = scriptargs["time"]

        _logger.info("Starting Frame: %s", scriptargs["frame"])

    def pre_render(self, scriptargs: dict) -> None:
        """Action run before the render starts.

        :param scriptargs: Event data.
        :return:

        """
        self._render_start = scriptargs["time"]

        frame_range = scriptargs["frame_range"]

        if frame_range is not None:
            start, end, inc = frame_range

            _logger.info("Starting render: %s-%s:%s", start, end, inc)

        else:
            _logger.info("Starting render")

    def post_frame(self, scriptargs: dict) -> None:
        """Action run after the frame has rendered.

        :param scriptargs: Event data.
        :return:

        """
        _print_frame_write(scriptargs)

        # Ensure we had a start time.
        if self._frame_start is not None:
            end_time = scriptargs["time"]
            duration = end_time - self._frame_start

            # Print a complete message that includes our calculated duration.
            _logger.info("Completed Frame: %s (%0.5fs)", scriptargs["frame"], duration)

        # If we somehow didn't, just print the complete message.
        else:
            _logger.info("Completed Frame: %s", scriptargs["frame"])

    def post_render(self, scriptargs: dict) -> None:
        """Action run after the render is complete.

        :param scriptargs: Event data.
        :return:

        """
        if self._render_start is not None:
            end_time = scriptargs["time"]

            _logger.info("Completed Render: %0.5fs", end_time - self._render_start)

        else:
            _logger.info("Completed Render")

    @staticmethod
    def post_write(scriptargs: dict) -> None:
        """Action run after the frame is written to disk.

        :param scriptargs: Event data.
        :return:

        """
        if "path" in scriptargs:
            _logger.info(
                "Wrote frame %s to %s", scriptargs["frame"], scriptargs["path"]
            )

        else:
            _logger.info("Wrote frame %s", scriptargs["frame"])

# =============================================================================
# NON-PUBLIC FUNCTIONS
# =============================================================================

def _get_target_file(node: hou.RopNode) -> Optional[str]:
    """Attempt to determine the target output file:

    :param node: The running node.
    :return: The output file path, if any.

    """
    node_type = node.type()
    node_type_name = node_type.name()

    if node_type_name in ("geometry", "rop_geometry"):
        return node.evalParm("sopoutput")

    if node_type_name in ("alembic", "rop_alembic"):
        return node.evalParm("filename")

    if node_type_name in ("ifd",):
        if node.evalParm("soho_outputmode"):
            return node.evalParm("soho_diskfile")

        return node.evalParm("vm_picture")

    return None

def _print_frame_write(scriptargs: dict) -> None:
    """Print that a file was written.

    :param scriptargs: Event data.
    :return:

    """
    if "path" in scriptargs:
        node = scriptargs["node"]

        postwrite_parm = node.parm("tpostwrite")

        # If the node has a Post-Write script and it is enabled then we will
        # not handle printing out a write message.
        if postwrite_parm is not None and postwrite_parm.eval():
            pass

        else:
            _logger.info(
                "Wrote frame %s to %s", scriptargs["frame"], scriptargs["path"]
            )

# =============================================================================
# FUNCTIONS
# =============================================================================

def build_scriptargs(node: Optional[hou.Node] = None) -> dict:
    """Build relevant scriptargs for this action.

    :param node: Optionally rendering node.
    :return: Data related to the running script parm.

    """
    frame_range = None

    if node is not None:
        trange_parm = node.parm("trange")

        if trange_parm is not None and trange_parm.evalAsString() != "off":
            frame_range = node.evalParmTuple("f")

    scriptargs = {
        "node": node,
        "frame": hou.frame(),
        "frame_range": frame_range,
        "time": time.time(),
    }

    if node is not None:
        scriptargs["path"] = _get_target_file(node)

    return scriptargs


import plotly.plotly as py
from plotly.graph_objs import Scatter
import plotly

import datetime

def set_credentials():

#   Set credentials for online plotting
#     plotly.tools.set_credentials_file(username='aarjay00', api_key='89q89hwig4')

    plotly.tools.set_credentials_file(username='aarjay80', api_key='7dzds0fegi')

def basic_scatter_plot_online(xy,graph_name):
    set_credentials()
    x=[i[0] for i in xy ]
    y=[i[1] for i in xy]
    trace=Scatter(x=x,y=y,mode='markers')
    plotly.plotly.plot([trace],filename=graph_name)

def basic_scatter_plot_offline(xy,graph_name):
    set_credentials()
    x=[i[0] for i in xy ]
    y=[i[1] for i in xy]

    marker=dict(color='red',size='5',symbol='square-dot')
    trace=Scatter(x=x,y=y,mode='markers',name='col_name',marker=marker,connectgaps=True)
    plotly.offline.iplot([trace],filename=graph_name)

dev_1 = ['U','U','E','D','D','D','U','X']
dev_2 = ['U','U','U','D','D','U']
activity_color_map={'U':'blue','E':'green','D':'red'}

def create_trace_from_dev_activity(dev_activity,dev_name):

        # trace_x_understand = [idx for idx,activity in enumerate(dev_activity) if activity=='U']
        # trace_x_edit = [idx for idx,activity in enumerate(dev_activity) if activity=='E']
        # trace_x_debug= [idx for idx,activity in enumerate(dev_activity) if activity=='D']
        # trace_x_none  = [idx for idx,activity in enumerate(dev_activity) if activity=='X']

        trace_x_understand = [datetime.datetime.fromtimestamp(int(activity[1])) for activity in dev_activity if activity[0] == 'U']
        trace_x_edit = [datetime.datetime.fromtimestamp(int(activity[1])) for activity in dev_activity if activity[0] == 'E']
        trace_x_debug   = [datetime.datetime.fromtimestamp(int(activity[1])) for activity in dev_activity if activity[0] == 'D']
        trace_x_none = [datetime.datetime.fromtimestamp(int(activity[1])) for activity in dev_activity if activity[0] == 'X']

        marker_understand =dict(color='blue',size='15',symbol='square-dot')
        marker_edit =dict(color='green',size='15',symbol='square-dot')
        marker_debug =dict(color='red',size='15',symbol='square-dot')
        marker_none = dict(color='black',size='15',symbol='square-dot')

        trace_understand = Scatter(x=trace_x_understand,y=[dev_name]*len(trace_x_understand)
                                   ,marker=marker_understand
                                   ,hoverinfo='none'
                                   ,mode='markers'
                                   ,showlegend=False
                                  ,name='understand_actions')

        trave_edit = Scatter(x=trace_x_edit,y=[dev_name]*len(trace_x_edit)
                             ,marker=marker_edit
                             ,hoverinfo='none'
                             ,mode='markers'
                             ,showlegend=False
                            ,name='edit_actions')

        trace_debug = Scatter(x=trace_x_debug,y=[dev_name]*len(trace_x_debug)
                              ,marker=marker_debug,
                              hoverinfo='none',
                              mode='markers'
                              ,showlegend=False
                             ,name='debug_actions')

        trace_none = Scatter(x=trace_x_none,y=[dev_name]*len(trace_x_none)
                              ,marker=marker_none,
                              hoverinfo='none',
                              mode='markers'
                              ,showlegend=False
                             ,name='none_actions')

        return [trace_understand,trave_edit,trace_debug,trace_none]

def create_dev_activity_graph(dev_activity,dev_name):
    # plotly.offline.init_notebook_mode()
    dev_name=dev_name.split('/')[-1].strip()
    set_credentials()
    trace_collection=[]
    trace_collection.extend(create_trace_from_dev_activity(dev_activity,dev_name))
#     plotly.plotly.image.save_as(trace_collection,dev_name+'.png')
    plotly.plotly.plot(trace_collection,filename=dev_name)
    # plotly.offline.iplot(trace_collection)
# trace_collection.extend(create_trace_from_dev_activity(dev_2,'dev_2'))

def create_all_dev_activity_graph(dev_activity_dict):

    set_credentials()
    trace_collection=[]

    for user_name,user_activity in dev_activity_dict.iteritems():

        trace_collection.extend(create_trace_from_dev_activity(user_activity,user_name))

    plotly.plotly.plot(trace_collection,"All_Dev")

# basic_scatter_plot_offline([[1,'b'],[2,'a']],'trial')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import json
import random
import numpy as np
import jieba
import requests

from query_util import QueryUtils
from cn_util import print_cn
from cn_util import print_out
from itertools import chain

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

reload(sys)
sys.setdefaultencoding("utf-8")

def add_dict():
    path='../../data/sc/train/belief_rnn/decoder_vocab.txt'
    with open(path,'r') as f:
        for line in f:
            line=line.strip()
            jieba.add_word(line)
    # print_cn(cut('我要吃张氏宽窄巷和很高兴遇见你'))

def get_train_data(data_path,normal_path,shuffle_path):
    add_dict()
    normal_f=open(normal_path,'w')
    shuffle_f=open(shuffle_path,'w')
    # drop_f=open(drop_path)

    with open(data_path,'r') as f:
        for line in f:
            print_cn(line)
            line=json.loads(line)
            intentions=','.join(line['intention'])
            questions=line['question']

            for question in questions:
                normal_f.write(intentions + '#' + question + '\n')
                tokens = cut(question)
                if len(tokens)<=1:
                    continue
                elif len(tokens)==2:
                    random.shuffle(tokens)
                    shuffle_f.write(intentions + '#' + ''.join(tokens) + '\n')
                else:
                    for _ in range(2):
                        random.shuffle(tokens)
                        shuffle_f.write(intentions+'#'+''.join(tokens)+'\n')
    normal_f.close()
    shuffle_f.close()

def cut(input_):
    input_ = QueryUtils.static_remove_cn_punct(input_)
    tokens = list(jieba.cut(input_, cut_all=False))
    return tokens

def fasttext_wv(word):
    url = 'http://localhost:11425/fasttext/w2v?q='
    ff_url = url + word.decode('utf-8')
    r = requests.get(url=ff_url)
    vector = r.json()['vector']
    return vector

def maybe_process_data(data_path, files):
    # train_file=open(train_data_path,'w')
    pwd_path=os.path.abspath('.')
    paths=[os.path.join(pwd_path,f) for f in files]
    all_exist=True
    for path in paths:
        if not os.path.exists(path):
            all_exist=False
            break

    if all_exist:
        print('Data Exist.')
    else:
        print('Process Data.')
        encoder_vocab_file = open(files[0], 'w')
        decoder_vocab_file = open(files[1], 'w')
        fasttext_wv_file=open(files[2],'w')
        fasttext_embeddings={}

        querys = []
        encoder_vocab = []
        decoder_vocab = []

        with open(data_path, 'r') as f:
            for line in f:
                line = line.strip().replace(' ', '')
                query = line.split('#')[1]
                querys.append(query)
                label = line.split('#')[0].split(',')
                for word in label:
                    if word not in decoder_vocab:
                        decoder_vocab.append(word)
        for word in decoder_vocab:
            jieba.add_word(word)

        for query in querys:
            tokens = list(cut(query))
            for word in tokens:
                if word not in encoder_vocab:
                    encoder_vocab.append(word)

        encoder_vocab.extend(['#EOS#', '#UNK#', '#PAD#', '#GO#'])
        decoder_vocab.extend(['#EOS#', '#UNK#', '#PAD#', '#GO#', '/'])

        for word in encoder_vocab:
            encoder_vocab_file.write(word)
            encoder_vocab_file.write('\n')
            if word not in fasttext_embeddings:
                fasttext_embeddings[word]=fasttext_wv(word)

        for word in decoder_vocab:
            decoder_vocab_file.write(word)
            decoder_vocab_file.write('\n')
            if word not in fasttext_embeddings:
                fasttext_embeddings[word]=fasttext_wv(word)

        json.dump(fasttext_embeddings,fasttext_wv_file,ensure_ascii=False)
        encoder_vocab_file.close()
        decoder_vocab_file.close()
        fasttext_wv_file.close()
        print('Process Done.')

def recover_source(index, vocab):
    sentence = []
    ignore = ['#PAD#', '#UNK#', '#GO#']
    for ii in index:
        word = vocab[ii]
        # if word in ignore:
        #     continue
        # if ii == '#EOS#':
        #     break
        sentence.append(word)
    return ''.join(sentence)

def recover_label(index, vocab):
    sentence = []
    ignore = ['#PAD#', '#UNK#', '#GO#']
    for ii in index:
        word = vocab[ii]
        # if word in ignore:
        #     continue
        # if word == '#EOS#':
        #     break
        sentence.append(word)
    return ''.join(sentence)

def load_fasttext(encoder_vocab_path):
    print('Load encoder vocab')
    encoder_vocab = []
    embeddings = []

    with open(encoder_vocab_path) as f:
        for word in f:
            word = word.strip().replace(' ', '')
            if word not in encoder_vocab:
                encoder_vocab.append(word)
                embedding = fasttext_wv(word)
                embeddings.append(embedding)
    return encoder_vocab, embeddings

def load_decoder_vocab(decoder_vocab_path):
    print('Load decoder vocab')
    decoder_vocab = []

    with open(decoder_vocab_path) as f:
        for word in f:
            word = word.strip().replace(' ', '')
            if word not in decoder_vocab:
                decoder_vocab.append(word)
    return decoder_vocab

def padding(inputs, pad):
    batch_size = len(inputs)
    sequence_lengths = [len(seq) for seq in inputs]
    max_sequence_length = max(sequence_lengths)

    inputs_batch_major = np.ones(
        (batch_size, max_sequence_length), np.int32) * pad
    for i, seq in enumerate(inputs):
        for j, element in enumerate(seq):
            inputs_batch_major[i][j] = element

    return inputs_batch_major

def generate_batch_data(data_path, encoder_vocab, decoder_vocab, batch_size=32, train=True):
    sources = list()
    targets = list()

    with open(data_path, 'r') as f:
        for line in f:
            line = line.strip().replace(' ', '').encode('utf-8')
            target = line.split('#')[0].split(',')
            space = ['/' for _ in target]
            target = list(chain(*zip(target, space)))[:-1]
            source = line.split('#')[1]
            sources.append(source)
            targets.append(target)
    size = len(sources)

    for word in encoder_vocab:
        jieba.add_word(word)
    for word in decoder_vocab:
        jieba.add_word(word)

    while True:
        ss = np.random.randint(0, size, batch_size)

        encoder_inputs = list()
        decoder_inputs = list()
        labels = list()

        for ln in ss:
            source_tokens = list(cut(sources[ln].strip().decode('utf-8')))
            # print_cn(source_tokens)
            target_tokens = targets[ln]
            # print_cn(target_tokens)
            decoder_input = ['#GO#'] + target_tokens
            label = target_tokens + ['#EOS#']
            # print_cn(label)
            encoder_inputs.append(
                [encoder_vocab.index(word) if word in encoder_vocab else encoder_vocab.index('#UNK#') for word in
                 source_tokens])
            decoder_inputs.append(
                [decoder_vocab.index(word) if word in decoder_vocab else decoder_vocab.index('#UNK#') for word in
                 decoder_input])
            labels.append(
                [decoder_vocab.index(word) if word in decoder_vocab else decoder_vocab.index('#UNK#') for word in
                 label])

        encoder_inputs_length = [len(line) for line in encoder_inputs]
        decoder_inputs_length = [len(line) for line in decoder_inputs]

        # print(encoder_inputs)
        # print(labels)
        # print('----------------------------------------------')

        encoder_inputs = padding(encoder_inputs, encoder_vocab.index('#PAD#'))
        decoder_inputs = padding(decoder_inputs, decoder_vocab.index('#PAD#'))
        labels = padding(labels, decoder_vocab.index('#PAD#'))

        # encoder_embed_inputs = embedding(encoder_inputs, train)
        # decoder_embed_inputs = embedding(decoder_inputs, train)
        # labelss = [[label2index.get(word.decode('utf-8'), 3) for word in ln] for ln in labels]

        yield np.array(encoder_inputs), np.array(decoder_inputs), np.array(labels), np.asarray(
            encoder_inputs_length), np.asarray(decoder_inputs_length)

if __name__ == '__main__':
    # data_path='../../data/sc/train/sale_train0824.txt'
    # encoder_vocab_path='../../data/sc/train/belief_rnn/encoder_vocab.txt'
    # decoder_vocab_path='../../data/sc/train/belief_rnn/encoder_vocab.txt'
    # fasttext_path='../../data/sc/train/belief_rnn/fasttext_wv.txt'
    # # word2vec_path='../../data/sc/belief_rnn/encoder_vocab.txt'
    # files = [encoder_vocab_path, decoder_vocab_path, fasttext_path]
    # maybe_process_data(data_path,files)
    # add_dict()
    get_train_data('../../data/sc/0831/111sale','../../data/sc/train/sale_train0831.txt','../../data/sc/train/sale_train0831_shuffle.txt')


# coding: utf-8

# In[1]:

#For screening tools for light sheet microscopy image tiles from Pavel lab

# In[2]:

from skimage import io
import os
import numpy
import pandas
import sys
import shutil

# In[3]:

# Here are the positive and negative MIP folders
positive_folder = sys.argv[1]
negative_folder = sys.argv[2]
#Here is the test folder from another brain
test_folder = sys.argv[3]

# In[4]:

# get the statistics for the 18 positive MIP samples
num_pos = len(os.listdir(positive_folder))
pos_data = numpy.array([])
pos_max_array = numpy.zeros(num_pos)
pos_mean_array = numpy.zeros(num_pos)
pos_std_array = numpy.zeros(num_pos)

index_p = 0
for filename in os.listdir(positive_folder):
    #print(filename)
    im = (io.imread(positive_folder+filename)).astype(float)

    if pos_data.size == 0:
        pos_data=im
        pos_max_array[0] = im.max()
        pos_mean_array[0] = im.mean()
        pos_std_array[0] = im.std()
    else:
        pos_data=numpy.concatenate((pos_data, im), axis=0)
        pos_max_array[index_p] = im.max()
        pos_mean_array[index_p] = im.mean()
        pos_std_array[index_p] = im.std()

    index_p=index_p+1

# In[5]:

# get the statistics for the 18 negative MIP samples
num_neg = len(os.listdir(negative_folder))
neg_data = numpy.array([])
neg_max_array = numpy.zeros(num_neg)
neg_mean_array = numpy.zeros(num_neg)
neg_std_array = numpy.zeros(num_neg)

index_p = 0
for filename in os.listdir(negative_folder):
    #print(filename)
    im = (io.imread(negative_folder+filename)).astype(float)

    if neg_data.size == 0:
        neg_data=im
        neg_max_array[0] = im.max()
        neg_mean_array[0] = im.mean()
        neg_std_array[0] = im.std()
    else:
        neg_data=numpy.concatenate((neg_data, im), axis=0)
        neg_max_array[index_p] = im.max()
        neg_mean_array[index_p] = im.mean()
        neg_std_array[index_p] = im.std()

    index_p=index_p+1

# In[6]:

# calculate the threshold assuming two Gaussian distribution for background and foreground
# same for Max,Mean and std
thres_max = neg_max_array.max() + (pos_max_array.max()-neg_max_array.max())*neg_max_array.std()/(neg_max_array.std()+pos_max_array.std())

thres_mean = neg_mean_array.mean() + (pos_mean_array.mean()-neg_mean_array.mean())*neg_mean_array.std()/(neg_mean_array.std()+pos_mean_array.std())

thres_std = neg_std_array.max() + (pos_std_array.max()-neg_std_array.max())*neg_std_array.std()/(neg_std_array.std()+pos_std_array.std())

# In[7]:

# display the values
thres_max

# In[8]:

thres_mean

# In[9]:

thres_std

# In[10]:

#Using Otsu to find a segmentation threshold for foreground(bright) pixels
from skimage import filters
thres_seg = filters.threshold_otsu(pos_data)
thres_seg

# In[11]:

# bright pixel number threshold as 0.2% of the volume voxels
thres_birght_pixel_num = (im.size*0.002) # actually 0<this<2% all works
thres_birght_pixel_num

# In[12]:

#Design:
#Screen tool1: 'positive' if numpy.max(im)>thres_max else 'negative'  that is if maximum grayscale value larger than threshold, positive
#Screen tool2: 'positive' if numpy.mean(im)>thres_mean else 'negative' that is if mean of the grayscale values larger than threshold, positive
#Screen tool3: 'positive' if numpy.std(im)>thres_std else 'negative' that is if standard deviation of the grayscale values larger than threshold, positive
#Screen tool4: 'positive' if (im[im>thres_seg]).size>(im.size*0.01) else 'negative' that is if there are more than 0.2% of the volume voxels, consider this volumne as positive

# In[13]:

# Create the pandas DataFrame
df_s1234 = pandas.DataFrame(columns=['filename','category', 'max','mean','std', 'number>thresh','S1_max','S2_mean','S3_std','S4_seg'])
# Here first two columns are filename and if category is known
# then four columns of measurements: max, mean, std and number of pixels with grayscale larger than threshold
# the last t=four columns are the four screening tool results

# Start testing with positive folder
for filename in os.listdir(positive_folder):
    # for each MIP tiff in the folder, read in and get the values and judgements
    im = (io.imread(positive_folder+filename)).astype(float)
    data=[filename,'positive',numpy.max(im),round(numpy.mean(im),1),round(numpy.std(im),1),(im[im>thres_seg]).size,
          'positive' if (numpy.max(im))>thres_max else 'negative', 'positive' if numpy.mean(im)>thres_mean else 'negative',
          'positive' if numpy.std(im)>thres_std else 'negative', 'positive' if (im[im>thres_seg]).size>thres_birght_pixel_num else 'negative']
    #df_s1234.loc[len(df_s1234)]=data

# Start testing with negative folder
for filename in os.listdir(negative_folder):
    im = (io.imread(negative_folder+filename)).astype(float)
    data=[filename,'negative',numpy.max(im),round(numpy.mean(im),1),round(numpy.std(im),1),(im[im>thres_seg]).size,
          'positive' if (numpy.max(im))>thres_max else 'negative', 'positive' if numpy.mean(im)>thres_mean else 'negative',
          'positive' if numpy.std(im)>thres_std else 'negative', 'positive' if (im[im>thres_seg]).size>thres_birght_pixel_num else 'negative']
    #df_s1234.loc[len(df_s1234)]=data

# In[14]:
pos_tiles = list()

# Start testing with new testing folder from a second brain
for filename in os.listdir(test_folder):
    print('Processing file ' + filename, flush=True)
    im = (io.imread(test_folder+filename)).astype(float)
    if im.ndim == 3:
        im = numpy.max(im, axis=0)
    num_pos_criteria = 0
    if (numpy.max(im))>thres_max:
        num_pos_criteria = num_pos_criteria + 1
    if numpy.mean(im)>thres_mean:
        num_pos_criteria = num_pos_criteria + 1
    if numpy.std(im)>thres_std:
        num_pos_criteria = num_pos_criteria + 1
    if (im[im>thres_seg]).size>thres_birght_pixel_num:
        num_pos_criteria = num_pos_criteria + 1

    if num_pos_criteria >= 3:
        outcome = 'positive'
        pos_tiles.append(filename)
    elif num_pos_criteria >= 1:
        outcome = 'other'
    else:
        outcome = 'negative'

    data=[filename,outcome,numpy.max(im),round(numpy.mean(im),1),round(numpy.std(im),1),(im[im>thres_seg]).size,
          'positive' if (numpy.max(im))>thres_max else 'negative', 'positive' if numpy.mean(im)>thres_mean else 'negative',
          'positive' if numpy.std(im)>thres_std else 'negative', 'positive' if (im[im>thres_seg]).size>thres_birght_pixel_num else 'negative']
    df_s1234.loc[len(df_s1234)]=data

# In[15]:

df_sorted = df_s1234.sort_values(by = ['category', 'filename'], ascending = [True, True])
df_sorted.to_csv(sys.argv[4])

# In[16]:
has_max_prefix = True
if sys.argv[5] == sys.argv[3]:
    has_max_prefix = False
for f in pos_tiles:
    f2 = f
    if has_max_prefix:
        f2 = f[4:]
    shutil.copy(os.path.join(sys.argv[5], f2), os.path.join(sys.argv[6], f2))

# Final tools for positives:
# Screen tool1: numpy.max(im)>3992
# Screen tool2: numpy.mean(im)>240
# Screen tool3: numpy.std(im)>181
# Screen tool4: (im[im>2836]).size>(im.size*0.01)

# In[ ]:


import json
import logging
import os

from kafka import KafkaProducer

from configuration import constants

class LogRetrievalMessage:
    def __init__(self, experiment_id: str, host_name: str, target_path: str, node_id: str):
        self.experimentId = experiment_id
        self.hostName = host_name
        self.userName = os.getlogin()
        self.targetPath = target_path
        self.nodeId = node_id

def resolve_local_log_path(working_directory: str, experiment_id: str):
    return os.path.join(working_directory, experiment_id, "logs", "*")

class LogTransfer:
    def __init__(self, host_name: str, node_id: str, working_directory: str, kafka_bootstrap: str):
        self.host_name = host_name
        self.node_id = node_id
        self.working_directory = working_directory
        self.retrieval_msg_producer = KafkaProducer(
            bootstrap_servers=kafka_bootstrap,
            value_serializer=lambda x: json.dumps(x, default=lambda o: o.__dict__).encode("utf-8")
        )

    def initiate_log_retrieval(self, experiment_id: str):
        local_log_path = resolve_local_log_path(self.working_directory, experiment_id)

        logging.info("Initiating log retrieval")

        self.retrieval_msg_producer.send(
            constants.LOG_RETRIEVAL_TOPIC,
            LogRetrievalMessage(
                experiment_id,
                self.host_name,
                local_log_path,
                self.node_id
            )
        )

    def create_logging_directory(self, experiment_id: str):
        os.makedirs(os.path.join(self.working_directory, experiment_id, "logs"), exist_ok=True)

transfer_handler = None  # To be populated by main program


import argparse
import asyncio
import sys
import traceback as tb
import types
from typing import Any, Optional, Union

import quamash
from PyQt5.QtCore import Qt, QThread, pyqtRemoveInputHook
from PyQt5.QtGui import QColor, QGradient, QPaintEvent, QPixmap
from PyQt5.QtWidgets import QApplication, QSplashScreen

from bubblesub.api import Api
from bubblesub.api.log import LogLevel
from bubblesub.cfg import ConfigError
from bubblesub.ui.assets import ASSETS_DIR
from bubblesub.ui.main_window import MainWindow

class MySplashScreen(QSplashScreen):
    def __init__(self, *args: Any, **kwargs: Any) -> None:
        super().__init__(*args, **kwargs)
        self._painted = False

        self.setAttribute(Qt.WidgetAttribute.WA_DeleteOnClose)
        self.setWindowFlags(
            Qt.WindowType(
                Qt.WindowType.SplashScreen
                | Qt.WindowType.FramelessWindowHint
                | Qt.WindowType.WindowStaysOnTopHint
            )
        )
        self.setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground)

    def paintEvent(self, event: QPaintEvent) -> None:
        super().paintEvent(event)
        self._painted = True

    def showMessage(
        self,
        text: str,
        alignment: int = Qt.AlignmentFlag.AlignLeft,
        color: Union[QColor, Qt.GlobalColor, QGradient] = Qt.GlobalColor.black,
    ) -> None:
        self._painted = False
        super().showMessage(
            text, Qt.AlignmentFlag.AlignBottom, Qt.GlobalColor.white
        )
        self._ensure_painted()

    def _ensure_painted(self) -> None:
        while not self._painted:
            QThread.usleep(1000)
            QApplication.processEvents()

class Logger:
    def __init__(self, api: Api) -> None:
        self._api = api
        self._main_window: Optional[MainWindow] = None
        self._queued_logs: list[tuple["LogLevel", str]] = []

        sys.excepthook = self._on_error
        api.log.logged.connect(self._on_log)

    def set_main_window(self, main_window: MainWindow) -> None:
        self._main_window = main_window
        for log_level, text in self._queued_logs:
            self._on_log(log_level, text)
        self._queued_logs.clear()

    def _on_log(self, level: LogLevel, text: str) -> None:
        if self._main_window:
            self._main_window.console.log_window.log(level, text)
        else:
            self._queued_logs.append((level, text))

    def _on_error(
        self,
        type_: type[BaseException],
        value: BaseException,
        traceback: types.TracebackType,
    ) -> None:
        self._api.log.error("An unhandled error occurred: ")
        self._api.log.error(
            "".join(tb.format_exception(type_, value, traceback))
        )

class Application:
    def __init__(self, args: argparse.Namespace):
        self._args = args
        self._splash: Optional[QSplashScreen] = None

        pyqtRemoveInputHook()

        self._app = QApplication(sys.argv)
        self._app.setApplicationName("bubblesub")
        self._loop = quamash.QEventLoop(self._app)
        asyncio.set_event_loop(self._loop)

    def splash_screen(self) -> None:
        pixmap = QPixmap(str(ASSETS_DIR / "bubblesub.png"))
        pixmap = pixmap.scaledToWidth(640)
        self._splash = MySplashScreen(pixmap)
        self._splash.show()
        self._splash.showMessage("Loading API...")

    def run(self, api: Api) -> None:
        with self._loop:
            logger = Logger(api)

            try:
                if self._splash:
                    self._splash.showMessage("Loading config...")
                if not self._args.no_config:
                    api.cfg.load(api.cfg.DEFAULT_PATH)
            except ConfigError as ex:
                api.log.error(str(ex))

            if self._splash:
                self._splash.showMessage("Loading commands...")
            api.cmd.reload_commands()

            if self._splash:
                self._splash.showMessage("Loading UI...")
            main_window = MainWindow(api)
            api.gui.set_main_window(main_window)
            logger.set_main_window(main_window)

            # load empty file
            api.subs.unload()

            main_window.show()

            def save_config() -> None:
                if not self._args.no_config:
                    assert api.cfg.root_dir is not None
                    api.cfg.save(api.cfg.root_dir)

            api.cfg.opt.changed.connect(save_config)

            if self._args.file:
                api.cmd.run_cmdline([["open", "--path", self._args.file]])

            if self._splash:
                self._splash.finish(main_window)

            self._loop.run_forever()

        save_config()


import numpy as np
from keras.layers import Layer, InputSpec
from keras import backend as K

class Normalize(Layer):
    def __init__(self, scale, **kwargs):
        self.axis = 3 if K.image_dim_ordering() == 'tf' else 1
        self.scale = scale
        super(Normalize, self).__init__(**kwargs)

    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]
        self.W_shape = (input_shape[self.axis],)
        gamma = np.ones(self.W_shape) * self.scale
        self.gamma = K.variable(gamma, name='{}_gamma'.format(self.name))
        self.trainable_weights = [self.gamma]

    def compute_output_shape(self, input_shape):
        return input_shape

    def call(self, x, mask=None):
        x_normed = K.l2_normalize(x, self.axis)
        x_normed *= self.gamma
        return x_normed

    def get_config(self):
        config = { 'scale': self.scale, }
        base_config = super(Normalize, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class PriorBox(Layer):
    def __init__(self, img_size, min_size, max_size=None, aspect_ratios=None,
                 flip=True, variances=[1.0], clip=False, **kwargs):
        if min_size <= 0:
            raise Exception('min_size must be positive.')
        if max_size:
            if max_size < min_size:
                raise Exception('max_size must be greater than min_size.')
        if K.image_dim_ordering() == 'tf':
            self.haxis = 1
            self.waxis = 2
        else:
            self.haxis = 2
            self.waxis = 3
        self.img_size = img_size
        self.min_size = min_size
        self.max_size = max_size
        self.aspect_ratios = [1.0, 1.0]
        if aspect_ratios:
            for ar in aspect_ratios:
                if ar in self.aspect_ratios:
                    continue
                self.aspect_ratios.append(ar)
                if flip:
                    self.aspect_ratios.append(1.0 / ar)
        self.variances = np.array(variances)
        self.clip = clip
        self.flip = flip
        super(PriorBox, self).__init__(**kwargs)

    def compute_output_shape(self, input_shape):
        num_priors = len(self.aspect_ratios)
        layer_width = input_shape[self.waxis]
        layer_height = input_shape[self.haxis]
        num_boxes = num_priors * layer_width * layer_height
        return (input_shape[0], num_boxes, 8)

    def call(self, x, mask=None):
        if hasattr(x, '_keras_shape'):
            input_shape = x._keras_shape
        elif hasattr(K, 'int_shape'):
            input_shape = K.int_shape(x)
        layer_width = input_shape[self.waxis]
        layer_height = input_shape[self.haxis]
        img_width = self.img_size[0]
        img_height = self.img_size[1]
        num_priors = len(self.aspect_ratios)

        step_x = float(img_width) / float(layer_width)
        step_y = float(img_height) / float(layer_height)

        linx = np.linspace(0.5 * step_x, img_width - 0.5 * step_x, layer_width)
        liny = np.linspace(0.5 * step_y, img_height - 0.5 * step_y, layer_height)

        centers_x, centers_y = np.meshgrid(linx, liny)
        centers_x = centers_x.reshape(-1, 1)
        centers_y = centers_y.reshape(-1, 1)

        prior_boxes = np.concatenate((centers_x, centers_y), axis=1)
        prior_boxes = np.tile(prior_boxes, (1, 2 * num_priors))

        box_widths = []
        box_heights = []
        for ar in self.aspect_ratios:
            if ar == 1 and len(box_widths) == 0:
                box_widths.append(self.min_size)
                box_heights.append(self.min_size)
            elif ar == 1 and len(box_widths) > 0:
                box_widths.append(np.sqrt(self.min_size * self.max_size))
                box_heights.append(np.sqrt(self.min_size * self.max_size))
            elif ar != 1:
                box_widths.append(self.min_size * np.sqrt(ar))
                box_heights.append(self.min_size / np.sqrt(ar))
        box_widths = 0.5 * np.array(box_widths)
        box_heights = 0.5 * np.array(box_heights)

        prior_boxes[:, ::4] -= box_widths
        prior_boxes[:, 1::4] -= box_heights
        prior_boxes[:, 2::4] += box_widths
        prior_boxes[:, 3::4] += box_heights
        prior_boxes[:, ::2] /= img_width
        prior_boxes[:, 1::2] /= img_height
        prior_boxes = prior_boxes.reshape(-1, 4)
        if self.clip:
            prior_boxes = np.minimum(np.maximum(prior_boxes, 0.0), 1.0)

        num_boxes = len(prior_boxes)
        if len(self.variances) == 1:
            variances = np.ones((num_boxes, 4)) * self.variances[0]
        elif len(self.variances) == 4:
            variances = np.tile(self.variances, (num_boxes, 1))
        else:
            raise Exception('Must provide one or four variances.')

        prior_boxes = np.concatenate((prior_boxes, variances), axis=1)
        prior_boxes_tensor = K.expand_dims(K.variable(prior_boxes), 0)
        pattern = [K.shape(x)[0], 1, 1]
        prior_boxes_tensor = K.tile(prior_boxes_tensor, pattern)
        return prior_boxes_tensor

    def get_config(self):
        config = {
            'img_size': tuple(self.img_size),
            'min_size': self.min_size,
            'max_size': self.max_size,
            'aspect_ratios': tuple(self.aspect_ratios),
            'variances': tuple(self.variances),
            'flip': self.flip,
            'clip': self.clip }
        base_config = super(PriorBox, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


import requests
from urllib.parse import urlencode
import pandas as pd
import random
import time
import json
import os
import csv
from lxml import etree
import re

class Lagou_Position(object):

    def __init__(self):
        self.User_Agent = [
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36",
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
        ]
        self.headers = {
            "Host":"www.lagou.com",
            "Referer": "https://www.lagou.com/jobs/list_%E7%BD%91%E7%BB%9C%E5%AE%89%E5%85%A8/p-city_2?px=default",
            "User_Agent": random.choice(self.User_Agent),
            "X-Requested-With": "XMLHttpRequest"
        }
        self.headers1 = {
            "Host": "www.lagou.com",
            "Referer": "https://www.lagou.com/jobs/list_%E7%BD%91%E7%BB%9C%E5%AE%89%E5%85%A8/p-city_2?px=default",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": random.choice(self.User_Agent)
        }
        self.First_url = "https://www.lagou.com/jobs/list_%E7%BD%91%E7%BB%9C%E5%AE%89%E5%85%A8/p-city_2?px=default"
        #搜索网络安全
        self.Target_url = "https://www.lagou.com/jobs/positionAjax.json?px=default&city=%E5%8C%97%E4%BA%AC&needAddtionalResult=false"
        #网络安全页下ajax页面

    #请求原始链接获得cookie
    def Get_First_Url(self):
        rs = requests.session()
        rs.get(self.First_url, headers = self.headers, timeout=3)
        cookies = rs.cookies
        print(cookies)
        return cookies

    #构造请求Ajax的数据包
    def Post_Target_Url(self):
        cookie = self.Get_First_Url()
        return_cookie = requests.utils.dict_from_cookiejar(cookie)
        cookie.update(return_cookie)
        pn = 1
        for pn in range(30):
            params = {
                "first":"false",
                "pn":pn,
                "kd":"网络安全"
            }
            pn += 1
            response = requests.post(self.Target_url,data=params, cookies = cookie, headers = self.headers, timeout=3)
            print(response.text)
            self.Analysis(response)
            time.sleep(60)

    #解析Ajax返回请求内职位信息
    def Analysis(self,response):
        items = []
        data = json.loads(response.text)["content"]["positionResult"]["result"]
        if len(data):
            for i in range(len(data)):
                positionID = data[i]["positionId"]
                positionName = data[i]["positionName"]
                education = data[i]['education']
                companyShortName = data[i]["companyShortName"]
                workYear = data[i]["workYear"]
                salary = data[i]["salary"] + "*" + data[i]["salaryMonth"]
        self.Save_ajax_data(items)
        time.sleep(1.3)

    # 保存在ajax里收集到的职位和条目信息
    def Save_ajax_data(self, items):
        file_size = os.path.getsize(r'C:\Users\Administrator\PycharmProjects\pythonlearn1\Strain\Lagou_Spider\analyst.csv')
        if file_size == 0:
            name = ["ID","职位名称","学历要求","公司名称","工作年限","薪水待遇"]
            file_test = pd.DataFrame(columns=name, data=items)
            file_test.to_csv(r"analyst.csv", encoding="gbk", index=False)
        else:
            with open(r"analyst.csv", "a+", newline="") as file_test:
                writer = csv.writer(file_test)
                writer.writerows(items)

    #请求职位详情页
    def Get_Url(self):
        urls = []
        with open("analyst.csv","r",newline="") as file:
            reader = csv.reader(file)
            for row in reader:
                if row[0] != "ID":
                    url = "https://www.lagou.com/jobs/{}.html".format(row[0])
                    urls.append(url)
        return  urls

    #对职位详情页的内容进行解析
    def Get_Detail(self):
        urls = self.Get_Url()
        length = len(urls)
        for url in urls:
            description = ""
            print(url)
            rs = requests.get(url, headers = self.headers1)
            rs.encoding = "utf-8"
            time.sleep(5)
            content = etree.HTML(rs.text)
            detail = content.xpath('//*[@id="job_detail"]/dd[2]/div/p/text()')
            print(detail)

            for i in range(1, len(detail)):

                if '要求' in detail[i - 1]:
                    for j in range(i, len(detail)):
                        detail[j] = detail[j].replace('\xa0', '')
                        detail[j] = re.sub('[、;；.0-9。]', '', detail[j])
                        description = description + detail[j] + '/'
                    print(description)
            self.Save_Position_Detail(description)
            length -= 1
            time.sleep(3)
    #保存职位详情页的信息
    def Save_Position_Detail(self,description):
        with open("description.txt", "a+", encoding="utf-8") as file:
            file.write(description)

spider = Lagou_Position()
spider.Post_Target_Url()
spider.Get_Detail()


import os
import argparse
from enlp.corpus.youtube import SamsungGalaxyS5
from enlp.corpus.semeval import LaptopsTest, LaptopsTrain, RestaurantsTest, RestaurantsTrain
from enlp.embeddings import GoogleNews, SennaEmbeddings, WikiDeps
from enlp.data import build_json_dataset

if __name__ == "__main__":

    Corpora = {"LaptopsTrain": LaptopsTrain,
               "LaptopsTest": LaptopsTest,
               "RestaurantsTrain": RestaurantsTrain,
               "RestaurantsTest": RestaurantsTest,
               "SamsungGalaxyS5": SamsungGalaxyS5}

    desc = "Help for build_json_fold_datasets, a script that takes processed corpora and " \
           "embeddings and generates JSON files for training attenttion-RNNs for aspect-based " \
           "opinion mining using k-fold cross validation"

    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument("--json_path",
                        required=True,
                        help="Absolute path to store JSON files" )

    parser.add_argument("--sentiment", "-s",
                        action='store_true',
                        help="To generate IOB tags that include sentiment (TARG+/TARG-/TARG0)")

    parser.add_argument("--joint", "-j",
                        action='store_true',
                        help="To generate IOB tags and classes")

    parser.add_argument("--strict", "-st",
                        action='store_true',
                        help="When using joint, use only single aspect or single sentiment sentences")

    parser.add_argument("--padding", "-p",
                        action='store_true',
                        default=True,
                        help="Add padding. Default=True")

    parser.add_argument("--minfreq", "-mf",
                        type=int,
                        default=1,
                        help="Minimum frequency (default=1)")

    parser.add_argument('--train', "-tr",
                        nargs='?',
                        choices=Corpora,
                        help="Corpus to use as training. Allowed values are " + ', '.join(Corpora),
                        metavar='')

    parser.add_argument('--test', "-ts",
                        nargs='?',
                        choices=Corpora,
                        help="Corpus to use as test. Allowed values are " + ', '.join(Corpora),
                        metavar='')

    embs = ["GoogleNews", "SennaEmbeddings", "WikiDeps"]

    parser.add_argument('--embeddings', "-e",
                        nargs='*',
                        choices=embs,
                        help="Names of embeddings to use. Allowed values are " + ', '.join(embs),
                        metavar='')

    parser.add_argument('--folds', "-f",
                        type=int,
                        default=1,
                        help="Number of folds to use. Default, no folds (1)")

    parser.add_argument('--ratio', "-r",
                        type=float,
                        default=0.8,
                        help="Train/Test ratio when test set not given. Default=0.8")

    feat_funcs = [lambda t: "JJ" in t.pos,
                  lambda t: "NN" in t.pos,
                  lambda t: "RB" in t.pos,
                  lambda t: "VB" in t.pos,
                  lambda t: t.iob == "B-NP",
                  lambda t: t.iob == "B-PP",
                  lambda t: t.iob == "B-VP",
                  lambda t: t.iob == "B-ADJP",
                  lambda t: t.iob == "B-ADVP",
                  lambda t: t.iob == "I-NP",
                  lambda t: t.iob == "I-PP",
                  lambda t: t.iob == "I-VP",
                  lambda t: t.iob == "I-ADJP",
                  lambda t: t.iob == "I-ADVP"]

    args = parser.parse_args()

    if args.joint and args.sentiment:
        raise KeyError("Please choose --sentiment or --joint")

    if not os.path.exists(args.json_path):
        print("Creating " + args.json_path)
        os.makedirs(args.json_path)

    if args.embeddings:
        embeddings_list = []
        if "GoogleNews" in args.embeddings:
            embeddings_list.append(GoogleNews)
        if "SennaEmbeddings" in args.embeddings:
            embeddings_list.append(SennaEmbeddings)
        if "WikiDeps" in args.embeddings:
            embeddings_list.append(WikiDeps)
    else:
        embeddings_list = [None]

    for Embeddings in embeddings_list:
        if Embeddings:
            print("loading " + str(Embeddings.__name__))
            embeddings = Embeddings()
        else:
            embeddings = Embeddings

        TrainCorpus = Corpora[args.train]
        TestCorpus = Corpora.get(args.test, None)
        for pipeline in TrainCorpus.list_frozen():
            if any(["Chunker" in item for item in pipeline]):
                train_corpus = TrainCorpus.unfreeze(pipeline)
                print("Using " + train_corpus.name + " " + str(train_corpus.pipeline))
                if TestCorpus:
                    for pipeline in TestCorpus.list_frozen():
                        if any(["Chunker" in item for item in pipeline]) \
                        and pipeline == train_corpus.pipeline:
                            test_corpus = TestCorpus.unfreeze(pipeline)
                            print("Using " + test_corpus.name + " " + str(test_corpus.pipeline))
                            build_json_dataset(args.json_path,
                                               train_corpus, test_corpus=test_corpus, min_freq=args.minfreq,
                                               add_padding=args.padding, feat_funcs=feat_funcs,
                                               embeddings=embeddings, sentiment=args.sentiment,
                                               joint=args.joint, strict=args.strict)
                else:
                    build_json_dataset(args.json_path, train_corpus, min_freq=args.minfreq,
                                       add_padding=args.padding, feat_funcs=feat_funcs, embeddings=embeddings,
                                       sentiment=args.sentiment, test_ratio=args.ratio, folds=args.folds,
                                       joint=args.joint, strict=args.strict)


# -*- coding: utf-8 -*-
"""

"""

#import os
#os.chdir("C:/Users/Philo/Documents/3A -- ENSAE/apprentissage avancé/détection d'anomalies/Detection-anomalies-et-nouveautes")
#import keras
from keras.models import Model, load_model
from keras.layers import Input, Dense
from keras.callbacks import ModelCheckpoint, TensorBoard
from keras import regularizers
from sklearn.preprocessing import  StandardScaler, MinMaxScaler
from utils.evaluate import plot_var_in_out, evaluation_detection, deep_predict, evaluate
from utils.preprocessing import upload_data, split_data, prepro_data, simul_gaussian_data, simul_uniform_data
import numpy as np

def CustomAutoencoder(input_dim, encoding_dim=10):
    input_layer = Input(shape=(input_dim, ))
    encoder = Dense(encoding_dim, activation="tanh",activity_regularizer=regularizers.l1(10e-5))(input_layer)
    encoder = Dense(int(encoding_dim / 2), activation="tanh")(encoder)
    encoder = Dense(int(2), activation="tanh")(encoder)
    decoder = Dense(int(encoding_dim/ 2), activation='tanh')(encoder)
    decoder = Dense(int(encoding_dim), activation='tanh')(decoder)
    decoder = Dense(input_dim, activation='tanh')(decoder)
    autoencoder = Model(inputs=input_layer, outputs=decoder)
    print(autoencoder.summary())
    return autoencoder

def AlgoTrainPredict(name, algorithm, algo_particuliers, X_train, X_test, y_test, config, scale="MinMax", var1=10, var2=20):
    if (name == "KNN") | (name == "ABOD") | (name == "HBOS"):
        algorithm.fit(X_train)
        y_pred = algorithm.predict(X_test)
        y_pred[y_pred == 0] = -1 #normal
        print('---------'+name+'-----------')
        eval_=evaluate(y_test,y_pred)
        print(eval_)
        evaluation_detection(X_test, y_test,y_pred, var1, var2)
    if name == "Local Outlier Factor":
        algorithm.fit(X_train)
        y_pred = algorithm.fit_predict(X_test)
        y_pred = y_pred * -1 #outlier = 1
        print('---------'+name+'-----------')
        eval_=evaluate(y_test,y_pred)
        print(eval_)
        evaluation_detection(X_test, y_test,y_pred, var1, var2)
    if name == "Deep MLP":
        # scale data here
        if scale == "StandardScaler":
            scaler = StandardScaler()
        else :
            scaler = MinMaxScaler()
        X_train_scaled = scaler.fit_transform(X_train)
        X_test_scaled  = scaler.transform(X_test)

        algorithm.fit(X_train_scaled, X_train_scaled, epochs=config["nb_epoch"],
                      batch_size=config["batch_size"],shuffle=True,validation_split=0.33, verbose=0)
        y_pred = deep_predict(algorithm,X_test_scaled,config["outlier_prop"],y_test)#outlier = 1
        y_pred = y_pred * -1
        print('---------'+name+'-----------')
        eval_=evaluate(y_test,y_pred)
        print(eval_)
        evaluation_detection(X_test_scaled, y_test,y_pred, var1, var2)

    if name == "Robust covariance" :
        print("EllipticEnvelope")
        algorithm.fit(X_train)
        y_pred = -algorithm.predict(X_test) #outlier = 1
        if np.sum(y_pred==-1) < np.sum(y_pred==1):
            print("adjust outlier definition")
            y_pred = -y_pred
        print('---------'+name+'-----------')
        eval_=evaluate(y_test,y_pred)
        print(eval_)
        evaluation_detection(X_test, y_test,y_pred, var1, var2)

    if name not in algo_particuliers:
        algorithm.fit(X_train)
        y_pred = -algorithm.predict(X_test) #outlier = 1
        print('---------'+name+'-----------')
        eval_=evaluate(y_test,y_pred)
        print(eval_)
        evaluation_detection(X_test, y_test,y_pred, var1, var2)

    return y_pred, eval_


# Given an integer n, return a string array answer (1-indexed) where:
#
# answer[i] == "FizzBuzz" if i is divisible by 3 and 5.
# answer[i] == "Fizz" if i is divisible by 3.
# answer[i] == "Buzz" if i is divisible by 5.
# answer[i] == i (as a string) if none of the above conditions are true.
from typing import List

class Solution:
    def fizzBuzz(self, n: int) -> List[str]:
        result = []

        for i in range(1, n + 1):
            if i % 5 == 0 and i % 3 == 0:
                result.append("FizzBuzz")
            elif i % 5 == 0:
                result.append("Buzz")
            elif i % 3 == 0:
                result.append("Fizz")
            else:
                result.append(str(i))

        return result

obj = Solution()

n = 3
print(obj.fizzBuzz(n))  # Output: ["1","2","Fizz"]

n = 5
print(obj.fizzBuzz(n))  # Output: ["1","2","Fizz","4","Buzz"]

n = 15
print(obj.fizzBuzz(n))
# Output: ["1","2","Fizz","4","Buzz","Fizz","7","8","Fizz",
# "Buzz","11","Fizz","13","14","FizzBuzz"]


# BUILD USING PYTHON 3

from ancestor import MrA
import logging

class Robot(MrA):
    """
    Our final project class. This inherits from Mr. A's starter which inherts from the
    GoPiGo3 class.
    """

    def __init__(self):
        """
        Constructor; our place to setup instance variables
        """
        super().__init__()
        self.reset_all()

        # run the menu loop
        self.menu()

    def menu(self):
        """Displays menu dictionary, takes key-input and calls method"""
        ## This is a DICTIONARY, it's a list with custom index values
        # You may change the menu if you'd like to add an experimental method

        menu = {
                "s": ("Check status", self.status),
                "q": ("Quit", quit_now)
                }
        while True:
            # loop and print the menu...
            for key in sorted(menu.keys()):
                print(key + ":" + menu[key][0])
            # store the user's answer
            ans = input("Your selection: ")
            # activate the item selected
            menu.get(ans, [None, error])[1]()

    def status(self):
        self.dist()
        self.angle()

#####################################################################
## FUNCTIONS are methods that float in the module (not in a class)

def error():
    """records general, less specific error"""
    logging.error("ERROR")
    print('ERROR')

def quit_now():
    """shuts down app"""
    raise SystemExit

# SETUP LOGGING TO HELP TRACK ERRORS
LOG_LEVEL = logging.INFO
LOG_FILE = "/home/pi/itp/log_robot.log"  # don't forget to make this file!
LOG_FORMAT = "%(asctime)s %(levelname)s %(message)s"
logging.basicConfig(filename=LOG_FILE, format=LOG_FORMAT, level=LOG_LEVEL)

# this is where the robot is instantiated
if __name__ == "__main__":
    try:
        r = Robot()
    except (KeyboardInterrupt, SystemExit):
        r.reset_all()
    except Exception as ee:
        logging.error(ee.__str__())


import numpy as np
#import liblary numpy
import cv2
#import liblary openCV
from matplotlib import pyplot as plt
#import liblary pyplot, pemanggilan fungsi dengan "plt"
#untuk langsung plotting 2/ lebih histogram dalam satu window/ figure
scr = cv2.imread('HIMPROTE.png')
#deklarasi file sumber gambar
gray= cv2.cvtColor(scr, cv2.COLOR_BGR2GRAY)
#konvert file sumber (RGB) ke grayscale
equ = cv2.equalizeHist(gray)
cv2.imshow('Gambar sumber',scr)
#menampilkan gambar asli
cv2.imshow('hasil Histogram Equalization',equ)
#menampilkan gambar hasil Low Pass Filter
plt.figure('hasil Histogram Equalization')
#memunculkan 1 figure dengan nama 'hasil Histogram Equalization'
plt.subplot(2,1,1),plt.hist(gray.ravel(),256,[0,256]),plt.title('Histogram Awal')
#plotting histogram awal di urutan ke-1 dalam figure
plt.subplot(2,1,2),plt.hist(equ.ravel(),256,[0,256]), plt.title('Histogram Equalization')
#plotting histogram equalization di urutan ke-2 dalam figure
plt.show()
#memunculkan plot
cv2.waitKey()
#menunggu input dari keyboard
cv2.destroyAllWindows()
#lalu tutup semua windows jika ada keyboard yang ditekan


"""
Contains definitions and methods to store and access Collections
"""

import bpy
from typing import Iterable

def link_objects_to_collection(
    objects: Iterable[bpy.types.Object], new_col: bpy.types.Collection, unlink_from_existing: bool = True
) -> None:
    for obj in objects:
        if not obj:
            continue
        if unlink_from_existing:
            for col in obj.users_collection:
                col.objects.unlink(obj)
        if obj.name not in new_col.objects:
            new_col.objects.link(obj)

def get_or_create_collection(scene: bpy.types.Scene, name: str, parent_col=None) -> bpy.types.Collection:
    col = bpy.data.collections.get(name)
    if not col:
        col = bpy.data.collections.new(name)
    if parent_col is None:
        parent_col = scene.collection
    if col.name not in parent_col.children:
        parent_col.children.link(col)
    return col


import torch.nn as nn
import torch
import time
import os
import copy
import argparse
from torchvision import datasets, models
from torch.utils.data import Dataset
from PIL import Image

from transform import data_transforms

parser = argparse.ArgumentParser(description='Arguments for inference')
parser.add_argument('--infer_data_dir', default='', help='data folder path for inference')
parser.add_argument('--infer_image_path', default='', help='image path for inference')
parser.add_argument('--model_path', default='', help='model path to infer')

args = parser.parse_args()

class ImgDataset(Dataset):

    def __init__(self, data_dir, transform):
        self.file_list = [os.path.join(data_dir, fname) for fname in os.listdir(data_dir)]
        self.transform = transform

    def __len__(self):
        return len(self.file_list)

    def __getitem__(self, index):
        img_path = self.file_list[index]
        img = Image.open(img_path)
        img_transformed = self.transform(img)

        return img_transformed

def infer_model(dataloader, model):
    since = time.time()

    for inputs in dataloader:
        inputs = inputs.to(device)
        with torch.set_grad_enabled(False):
            outputs = model(inputs)
            _, preds = torch.max(outputs, 1)

        print(preds)

    time_elapsed = time.time() - since
    print('Inference complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))

def infer_image(image, model, device):
    input = data_transforms['val'](image)
    input = input.to(device)
    with torch.set_grad_enabled(False):
        outputs = model(input.unsqueeze(0))
        _, preds = torch.max(outputs, 1)
    return preds

if __name__=='__main__':

    # arguments
    infer_data_dir = args.infer_data_dir
    infer_image_path = args.infer_image_path
    model_path = args.model_path

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    model_ft = models.resnet18(pretrained=True)
    num_ftrs = model_ft.fc.in_features
    # Here the size of each output sample is set to 2.
    # Alternatively, it can be generalized to nn.Linear(num_ftrs, len(class_names)).
    model_ft.fc = nn.Linear(num_ftrs, 2)

    assert os.path.isfile(model_path)

    model_ft.load_state_dict(torch.load(model_path))
    model_ft = model_ft.to(device)

    if os.path.isdir(infer_data_dir):
        print("infer from {}".format(infer_data_dir))
        image_dataset = ImgDataset(infer_data_dir, data_transforms['val'])
        dataloader = torch.utils.data.DataLoader(image_dataset, batch_size=1, shuffle=False, num_workers=0)

        infer_model(dataloader, model_ft)

    elif os.path.isfile(infer_image_path):
        preds = infer_image(Image.open(infer_image_path), model_ft, device)


# -*- coding: utf-8 -*-
import pandas as pd
from sklearn.externals import joblib
import sys

"""
加载模型，并对经过数据预处理的数据进行测试
"""

def test(data_path, model_path, result_path):
    """
    对数据进行预测
    :return:
    """
    data = pd.read_csv(data_path)
    print(data['cate_score'].value_counts())
    brand_sum = 0
    cate_sum = 0
    name_sum=0
    count_nozero_cate = 0
    count_nozero_brand = 0
    count_nozero_name = 0
    for index in range(data.shape[0]):
        if (data.iloc[index, 4] != 2):
            brand_sum += data.iloc[index, 4]
            count_nozero_brand += 1
        if (data.iloc[index, 5] != 2):
            cate_sum += data.iloc[index, 5]
            count_nozero_cate += 1
        if (data.iloc[index, 6] != 2):
            name_sum += data.iloc[index, 6]
            count_nozero_name += 1
    brand_aver = brand_sum / (count_nozero_brand)
    cate_aver = cate_sum / (count_nozero_cate)
    name_aver=name_sum/(count_nozero_name)
    data.loc[data["brand_score"] == 2, 'brand_score'] = brand_aver
    data.loc[data["cate_score"] == 2, 'cate_score'] = cate_aver
    data.loc[data["name_score"] == 2, 'name_score'] = name_aver
    data_id = data.ix[:, [1, 2]]
    data.drop(['raw_id', 'simi_id', 'label'], axis=1, inplace=True)
    model = joblib.load(model_path)
    X = data.ix[:, data.columns != 'label'].values
    y_pred = model.predict(X)
    data = data.reset_index(drop=True)
    result = pd.DataFrame({'label': y_pred})
    data = pd.concat([result, data_id, data], axis=1)
    data.to_csv(result_path, index=None)

if __name__ == "__main__":
    if (len(sys.argv) < 4):
        print("one of the file path not found ")
    else:
        data_path = sys.argv[1]
        model_path = sys.argv[2]
        result_path = sys.argv[3]
        test(data_path, model_path, result_path)


import matplotlib.pyplot as plt

with open("test_output.txt","r") as f:
    data = [line.strip().split() for line in f.readlines()]

fig = plt.figure()
ax = fig.add_subplot(111)
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
col_labels=("Test", "Expected Output", "Actual Output")
table = ax.table(cellText=data,
          colLabels=col_labels,
          loc='center')

plt.savefig("initial_results.png")
plt.show()


from django.shortcuts import render
from django.http import HttpResponse
from django.core.paginator import Paginator, PageNotAnInteger
from .models import Question, Tag, Answer
from django.shortcuts import get_object_or_404, get_list_or_404

def get_top_tags(rows=3):
    res = []
    idx = 0
    arr = Tag.objects.popular()
    for i in range(rows):
        res.append([])
        for _ in range(4):
            if len(arr) <= idx:
                break
            res[i].append(arr[idx])
            idx += 1
    return res

def paginate(objects_list, request, per_page=10):
    p = Paginator(objects_list, per_page)
    page_idx = request.GET.get('page', 1)
    try:
        page_idx = int(page_idx)
    except ValueError:
        page_idx = 1
    if page_idx < 1:
        page_idx = 1
    elif page_idx > p.num_pages:
        page_idx = p.num_pages
    raw_page = p.page(page_idx)
    correct_page_range = raw_page.paginator.get_elided_page_range(page_idx, on_each_side=2, on_ends=1)
    return {
        'page_obj': raw_page,
        'obj_list': raw_page.object_list,
        'page_range': correct_page_range,
        }

def index(request):
    page = paginate(Question.objects.recent(), request, 15)
    return render(request, 'index.html', {'questions': page['obj_list'],
                                          'tags': get_top_tags(), 'is_logged_in': True,
                                          'page': page})

def hot(request):
    page = paginate(Question.objects.best(), request, 15)
    return render(request, 'index.html', {'questions': page['obj_list'],
                                          'tags': get_top_tags(), 'is_logged_in': True,
                                          'page': page})

def detail_question(request, id):
    question = get_object_or_404(Question, pk=id)
    page = paginate(Answer.objects.filter(parent=question), request, 7)
    return render(request, 'question.html', {'question': question, 'answers': page['obj_list'],
                                             'tags': get_top_tags(), 'is_logged_in': True,
                                             'page': page})

def questions_by_tag(request, slag):
    questions = get_list_or_404(Question.objects.by_tag(slag))
    page = paginate(questions, request, 7)
    return render(request, 'tag.html', {'questions': page['obj_list'], 'tag_name': slag,
                                        'tags': get_top_tags(), 'is_logged_in': True,
                                        'page': page})

def login(request):
    return render(request, 'login.html', {'tags': get_top_tags()})

def signup(request):
    return render(request, 'signup.html', {'tags': get_top_tags()})

def ask(request):
    return render(request, 'ask.html', {'tags': get_top_tags(), 'is_logged_in': True})

def user_settings(request):
    return render(request, 'settings.html', {'tags': get_top_tags(), 'is_logged_in': True})


#!/usr/bin/env python
# coding: utf8
# Author: Tong ZHAO

import numpy as np

from sklearn.neighbors import KDTree
from sklearn.metrics import pairwise_distances

def estimate_transform(point_1, point_2, normal_1, normal_2, curvature_1, curvature_2):
    """Find the transformation from point 1 to point 2

    Params:
        point_1     (np.array): the coordinate of the first point
        point_2     (np.array): the coordinate of the second point
        normal_1    (np.array): the normal of the first point
        normal_2    (np.array): the normal of the second point
        curvature_1 (float)   : the curvature of the first point
        curvature_2 (float)   : the curvature of the second point

    Return:
        T (np.array): translation matrix of size 3 x 3 (SIM(2))
    """

    # anti-clockwise: 1, clockwise: -1
    x = np.array([0, 1])
    flag_dir = 1 if np.dot(x, normal_1) >= np.dot(x, normal_2) else -1

    theta = np.arccos(np.clip(np.dot(normal_1, normal_2), None, 1.)) * flag_dir
    T = np.zeros((3, 3))
    # Rotation
    T[0, 0] = np.cos(theta)
    T[0, 1] = -np.sin(theta)
    T[1, 0] = np.sin(theta)
    T[1, 1] = np.cos(theta)

    # Scale
    T[2, 2] = np.abs(curvature_2 / curvature_1)

    # Translation
    offset = point_2 - T[:2, :2].dot(point_1) * T[2, 2]
    T[:2, -1] = offset

    return T

def transform_neighbors(point_1, point_2, pts, tree, T, radius):
    """Transform the neighbors of point 1 to point 2

    Params:
        point_1 (np.array): the coordinate of the first point
        point_2 (np.array): the coordinate of the second point
        pts     (np.array): the point cloud
        tree    (KDTree)  : the kdtree to find neighbors
        T       (np.array): the translation from point 1 to point 2
        radius  (float)   : the radius to collect neighbors

    Returns:
        neighbors_1   (np.array): the neighbor points of point 1
        neighbors_2   (np.array): the neighbor points of point 2
        new_neighbors (np.array): the transformed neigbor points of point 1
        distances     (float)   : the alignment error after transformation
    """

    neighbors_1 = pts[tree.query_radius(point_1.reshape((1, -1)), radius)[0]]
    new_neighbors = neighbors_1.dot(T[:2, :2].T) * T[2, 2] + T[:2, -1]
    max_dist = np.max(np.linalg.norm(new_neighbors - point_2, axis = 1))
    neighbors_2 = pts[tree.query_radius(point_2.reshape((1, -1)), max_dist + 0.01)[0]]

    distances = np.min(pairwise_distances(new_neighbors, neighbors_2), axis = 1)

    return neighbors_1, neighbors_2, new_neighbors, distances.sum() / len(distances)

def point_pairing(points, normals, curvatures, tree, radius, thresh, ratio = 0.2):
    """Calculate transformations between pairs of points

    Params:
        points     (np.array): the point cloud
        normals    (np.array): the normals
        curvatures (np.array): the curvatures
        tree       (KDTree)  : the tree to find neighbors of each point
        radius     (float)   : the radius to choose neighbors
        thresh     (float)   : the threshold to eliminate transformations with large alignment error
        ratio      (float)   : the proportion to choose a sub point set P' (fixed to 0.2)

    Returns:
        lT         (list[np.array]): the list of all transformations
        dict_index (dict)          : transformation index as the key, corresponding indices of points as the value
    """

    selected_num = int(ratio * len(points))
    selected_idx = np.random.choice(len(points), selected_num, replace = False)
    lT = []
    dict_index = {}
    index = 0

    for i in selected_idx:
        for j in range(len(points)):
            if i == j:
                pass
            else:
                T = estimate_transform(points[i], points[j], normals[i], normals[j], curvatures[i], curvatures[j])
                loss = transform_neighbors(points[i], points[j], points, tree, T, radius)[-1]

                if loss < thresh and T.sum() > 1e-6:
                    lT.append(T)
                    dict_index[index] = tuple((i, j))
                    index += 1

    lT = np.stack(lT)

    return lT, dict_index


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
__author__ = 'GonnaZero'

import numpy as np
from sklearn import preprocessing

data = np.array([[3, -1.5, 2, -5.4], [0, 4, -0.3, 2.1], [1, 3.3, -1.9, -4.3]])
print("原始数据\n", data)

#均值移除
data_standardized = preprocessing.scale(data)
print("\nMean = " , data_standardized.mean(axis=0)) #axis=0 列
print("Std deviation = " , data_standardized.std(axis=0))

#范围缩放
data_scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))
data_scaler = data_scaler.fit_transform(data)
print("\nMin max scaled data = ", data_scaler)

#归一化
data_normalized = preprocessing.normalize(data, norm='l1') #是 l（字母） 不是 1（数字）
print("\n normalized data = ", data_normalized)

#二值化
data_binarized = preprocessing.Binarizer(threshold=1.4).transform(data)
print("\nBinarized data = ", data_binarized)

print("######################################")
#独热编码
encoder = preprocessing.OneHotEncoder()
encoder.fit([[0, 2, 1, 12], [1, 3, 5, 3], [2, 3, 2, 12], [1, 2, 4, 3]])
encoder_vector = encoder.transform([[2, 3, 5, 3]]).toarray()
print("\nEncoder vector = ", encoder_vector)


import asyncio
import concurrent.futures
import logging

from ._asyncio_worker import AsyncioWorker
from .server import _CustomSock
from .server import _FAST_TIMEOUT
from .server import _KEEPALIVE_TIMEOUT
from .server import _NEGOTIATION_SOURCE
from .server import _NEGOTIATION_TIMEOUT
from .server import _OPS_TIMEOUT
from .server import _QUEUE_SIZE
from .server import DATASERV_PORT
from .streaming._pickle import _squash_pickle_diff_queue
from .streaming._pickle import serialize_pickle_diff
from .streaming._pickle import streaming_pickle_diff

_logger = logging.getLogger(__name__)

class DataSource(AsyncioWorker):
    """For sourcing data to the :py:class:`~nspyre.data.server.DataServer`.
    See :py:meth:`~nspyre.data.sink.DataSink.pop` for typical usage example."""

    def __init__(
        self,
        name: str,
        addr: str = 'localhost',
        port: int = DATASERV_PORT,
        auto_reconnect: bool = False,
    ):
        """
        Args:
            name: Name of the data set.
            addr: Network address of the data server.
            port: Port of the data server.
            auto_reconnect: If True, automatically reconnect to the data
                server if it is disconnected. Otherwise raise an error if
                connection fails.
        """
        super().__init__()
        # name of the dataset
        self._name = name
        # IP address of the data server to connect to
        self._addr = addr
        # port of the data server to connect to
        self._port = port
        # whether the source should try to reconnect to the data server
        self._auto_reconnect = auto_reconnect

    def start(self):
        """Connect to the data server."""
        # do this just to generate docs
        super().start()

    def stop(self):
        """Disconnect from the data server."""
        # do this just to generate docs
        super().stop()

    async def _main(self):
        """asyncio main loop"""
        try:
            # asyncio queue for buffering pickles to send to the server
            self._queue = asyncio.Queue(maxsize=_QUEUE_SIZE)
            while True:
                try:
                    # connect to the data server
                    sock_reader, sock_writer = await asyncio.wait_for(
                        asyncio.open_connection(self._addr, self._port),
                        timeout=_NEGOTIATION_TIMEOUT,
                    )
                except OSError as err:
                    _logger.warning(
                        'Source failed connecting to data server '
                        f'[{(self._addr, self._port)}].'
                    )
                    await asyncio.sleep(_FAST_TIMEOUT)
                    if self._auto_reconnect:
                        continue
                    else:
                        raise ConnectionError(
                            'Failed connecting to data server '
                            f'[{(self._addr, self._port)}].'
                        ) from err

                sock = _CustomSock(sock_reader, sock_writer)
                _logger.info(f'Source connected to data server [{sock.addr}].')

                try:
                    # notify the server that this is a data source client
                    await asyncio.wait_for(
                        sock.send_msg(_NEGOTIATION_SOURCE),
                        timeout=_NEGOTIATION_TIMEOUT,
                    )
                    # send the dataset name
                    await asyncio.wait_for(
                        sock.send_msg(self._name.encode()),
                        timeout=_NEGOTIATION_TIMEOUT,
                    )
                except (ConnectionError, asyncio.TimeoutError) as err:
                    _logger.warning(
                        'Source failed negotiation process with data server '
                        f'[{sock.addr}] - attempting reconnect.'
                    )
                    try:
                        await sock.close()
                    except IOError:
                        pass
                    await asyncio.sleep(_FAST_TIMEOUT)
                    if self._auto_reconnect:
                        continue
                    else:
                        raise ConnectionError(
                            'Failed connecting to data server '
                            f'[{(self._addr, self._port)}].'
                        ) from err

                _logger.debug(
                    f'Source finished negotiation with data server [{sock.addr}].'
                )

                # connection succeeded, so trigger the main thread to continue execution
                self._sem.release()

                while True:
                    try:
                        # get pickle data from the queue
                        pickle_diff = await asyncio.wait_for(
                            self._queue.get(), timeout=_KEEPALIVE_TIMEOUT
                        )
                        _logger.debug(
                            'Source dequeued pickle diff - sending to data server '
                            f'[{sock.addr}].'
                        )
                    except asyncio.TimeoutError:
                        # if there's no data available, send a keepalive message
                        new_data = b''
                        _logger.debug('Source sending keepalive to data server.')
                    else:
                        new_data = serialize_pickle_diff(pickle_diff)
                        _logger.debug(
                            f'Source sending pickle of [{len(new_data)}] bytes to data '
                            f'server [{sock.addr}].'
                        )

                    # send the data to the server
                    try:
                        await asyncio.wait_for(
                            sock.send_msg(new_data), timeout=_OPS_TIMEOUT
                        )
                        _logger.debug(
                            f'Source sent pickle of [{len(new_data)}] bytes to data '
                            f'server [{sock.addr}].'
                        )
                        if new_data:
                            # mark that the queue data has been fully processed
                            self._queue.task_done()
                    except (ConnectionError, asyncio.TimeoutError):
                        _logger.warning(
                            f'Source failed sending to data server [{sock.addr}] - '
                            'attempting reconnect.'
                        )
                        try:
                            await sock.close()
                        except IOError:
                            pass
                        break
        except ConnectionError as err:
            self._exc = err
            # release the main thread if there's a connection error
            self._sem.release()
        except asyncio.CancelledError as exc:
            _logger.debug(
                f'Source stopped - closing connection with data server [{sock.addr}].'
            )
            try:
                await sock.close()
            except (IOError, NameError):
                # socket is broken or hasn't been created yet
                pass
            raise asyncio.CancelledError from exc

    async def _push(self, pickle_diff):
        """Coroutine that puts a pickle onto the queue.
        Args:
            pickle_diff: PickleDiff returned by
                :py:meth:`~nspyre.data._streaming_pickle.streaming_pickle_diff`.

        Returns:
            True if successful, False otherwise
        """
        try:
            self._queue.put_nowait(pickle_diff)
        except asyncio.QueueFull as err:
            # the server isn't accepting data fast enough
            # so we will empty the queue and merge all of its entries
            _logger.debug(
                f'Data server [{(self._addr, self._port)}] can\'t keep up with source.'
            )
            if not _squash_pickle_diff_queue(self._queue, pickle_diff):
                raise RuntimeError(
                    'Maximum diff size exceeded. This is a consequence of memory '
                    'build-up due to the data server not being able to keep up with '
                    'the data rate. Reduce the data rate to allow the data server to '
                    'catch up.'
                ) from err
        except asyncio.CancelledError:
            _logger.debug('Source push cancelled.')
            raise
        else:
            _logger.debug('Source queued pickle.')

    def push(self, data):
        """Push new data to the data server.

        Args:
            data: Any python object (must be pickleable) to send. Ideally, \
                this should be a dictionary to allow for simple attribute access \
                from the sink side like :code:`sink.my_var`.
        """
        # pickle the data and generate diffs
        pickle_diff = streaming_pickle_diff(data)
        # put it on the queue
        future = asyncio.run_coroutine_threadsafe(
            self._push(pickle_diff), self._event_loop
        )
        # wait for the coroutine to return
        try:
            future.result()
        except concurrent.futures.TimeoutError:
            _logger.error(
                '_push timed out (this shouldn\'t happen since timeout is handled by '
                '_push itself).'
            )
            future.cancel()
        except concurrent.futures.CancelledError:
            logging.debug('_push was cancelled.')
        self._check_exc()

    def __str__(self):
        return f'Data Source (running={self.is_running()}) [name={self._name}, '
        f'ip={self._addr}, port={self._port}, auto_reconnect={self._auto_reconnect}]'


import os
from PIL import Image
base_img_dir = r'../../medical_data/dental/turf/Radiographs'
base_mask_dir =r'../../medical_data/dental/turf/Expert/mask'
images = os.listdir(base_img_dir)

train_base_dir = r'../../medical_data/dental/turf/train'
os.makedirs(train_base_dir, exist_ok=True)
train_base_image_dir = r'../../medical_data/dental/turf/train/original'
os.makedirs(train_base_image_dir, exist_ok=True)
train_base_mask_dir = r'../../medical_data/dental/turf/train/mask'
os.makedirs(train_base_mask_dir, exist_ok=True)

test_base_dir = r'../../medical_data/dental/turf/valid'
os.makedirs(test_base_dir, exist_ok=True)
test_base_image_dir = r'../../medical_data/dental/turf/valid/original'
os.makedirs(test_base_image_dir, exist_ok=True)
test_base_mask_dir = r'../../medical_data/dental/turf/valid/mask'
os.makedirs(test_base_mask_dir, exist_ok=True)

for i, image in enumerate(images) :

    img_dir = os.path.join(base_img_dir,image)
    pil_img = Image.open(img_dir).convert('L')
    pil_img = pil_img.resize((512,512))
    mask_dir = os.path.join(base_mask_dir, image)
    pil_mask = Image.open(mask_dir).convert('L')
    pil_mask = pil_mask.resize((512,512))

    if i < 200 :
        pil_img.save(os.path.join(test_base_image_dir, image))
        pil_mask.save(os.path.join(test_base_mask_dir, image))

    else :
        pil_img.save(os.path.join(train_base_image_dir, image))
        pil_mask.save(os.path.join(train_base_mask_dir, image))