text:
import pandas as pd
import numpy as np
import multiprocessing
from multiprocessing import Process, Manager, Queue
import math

from PyProM.src.data.importing import Import

import sys
import os
from PyProM.src.utility.util_profile import Util_Profile
from PyProM.src.utility.util_multiprocessing import Util_Multiprocessing
import time
from functools import wraps

def timefn(fn):
	@wraps(fn)
	def measure_time(*args, **kwargs):
		t1 = time.time()
		result = fn(*args, **kwargs)
		t2 = time.time()
		print("@timefn: {} took {} seconds".format(fn.__name__, t2-t1))
		return result
	return measure_time

timefn = Util_Profile.timefn
class Eventlog(pd.DataFrame):
	"""docstring for Eventlog"""
	def __init__(self, *args, **kwargs):
		super(Eventlog, self).__init__(*args, **kwargs)
		self._columns = []

	@property
	def _constructor(self):
		return Eventlog

	@classmethod
	def from_xes(cls, path):
		_import = Import(path, format='xes')
		dict_eventlog = _import.eventlog
		if isinstance(dict_eventlog, dict):
			print("import dict and produce eventlog")
			df =  Eventlog.from_dict(dict_eventlog)
			return df

	@classmethod
	def from_txt(cls, path, sep='\t', encoding=None, **kwargs):
		if 'dtype' in kwargs:
			dtype = kwargs['dtype']
		else:
			dtype = None
		if 'index_col' in kwargs:
			index_col = kwargs['index_col']
		else:
			index_col=False
		df = pd.read_csv(path, sep = sep, index_col = index_col, dtype=dtype, encoding=encoding)
		return Eventlog(df)

	"""
	def __call__(self, path, format='xes'):
		if format == 'xes':
			_import = Import(path, format='xes')
			dict_eventlog = _import.eventlog
			return self.dict_to_dataframe(dict_eventlog)

		if format == 'txt':
			return self.csv_to_dataframe(path)
	"""
	@timefn
	def assign_caseid(self, *args):
		count = 0
		for arg in args:
			if count == 0:
				self['CASE_ID'] = self[arg].apply(str)
			else:
				self['CASE_ID'] += '_' + self[arg].apply(str)
			#del self[arg]
			count +=1
		self._columns.append('CASE_ID')
		return self

	@timefn
	def assign_activity(self, *args):
		count = 0
		for arg in args:
			if count == 0:
				self['Activity'] = self[arg].apply(str)
			else:
				self['Activity'] += '_' + self[arg].apply(str)
			#del self[arg]
			count +=1
		self._columns.append('Activity')
		return self

	@timefn
	def assign_resource(self, *args):
		count = 0
		for arg in args:
			if count == 0:
				self['Resource'] = self[arg].astype(str)
			else:
				self['Resource'] += '_' + self[arg].astype(str)
			#del self[arg]
			count +=1
		self._columns.append('Resource')
		return self

	@timefn
	def assign_timestamp(self, name, new_name = 'TIMESTAMP', _format = '%Y/%m/%d %H:%M:%S', errors='ignore'):
		print(_format)
		self[name] = pd.to_datetime(self[name], format = _format, errors=errors)
		self.rename(columns={name: new_name}, inplace=True)
		#self.loc[pd.isna(self[name]),name] = '-'
		self._columns.append(new_name)
		return self

	def assign_attr(self, **kwargs):
		"""
		이 함수는, ~~~~다.
		#할일: 컬럼명만 바꾸는 것으로!
		:param kwargs: old_col=데이터에 포함된 컬럼명,  new_col=생성한 이벤트로그에 지정할 컬럼명
		:return: 이벤트로그
		"""
		if 'old_col' in kwargs:
			old_col = kwargs['old_col']
		if 'new_col' in kwargs:
			new_col = kwargs['new_col']
		else:
			new_col = kwargs['old_col']
		self[new_col] = self[old_col]
		self._columns.append(new_col)
		del self[old_col]
		self._columns.append(new_col)
		return self

	def assign_cluster(self, *args):
		count = 0
		for arg in args:
			if count == 0:
				self['Cluster'] = self[arg].astype(str)
			else:
				self['Cluster'] += '_' + self[arg].astype(str)
			#del self[arg]
			count +=1
		self._columns.append('Cluster')
		return self

	def sort(self, by=['CASE_ID']):
		self = self.sort_values(by)
		return self

	def clear_columns(self, *args, **kwargs):
		if 'extra' in kwargs:
			extra = kwargs['extra']
		else:
			extra = []
		self = self[self._columns]
		return self

	def join_columns(self, col_name, *args):
		if len(args) < 2:
			print("join_columns requires at least 2 columns")
		count = 0
		tmp = self.copy(deep=True)
		for arg in args:
			if count == 0:
				self[col_name] = tmp[arg].astype(str)
			else:
				self[col_name] += '/' + tmp[arg].astype(str)
			#del self[arg]
			count +=1
		return self

	"""
	utility functions
	"""
	def get_event_trace(self, workers, value = 'Activity'):
		output = self.parallelize(self._get_event_trace, workers, value)
		event_trace = Util_Multiprocessing.join_dict(output)
		return event_trace

	def _get_event_trace(self, eventlog, x, value='Activity'):
		event_trace = dict()
		count = 0
		for instance in eventlog.itertuples():
			index = instance.Index
			if value == 'Activity':
				ai = eventlog.get_activity_by_index(index)
			elif value == 'Resource':
				ai = eventlog.get_resource_by_index(index)
			elif value == 'TIMESTAMP':
				ai = eventlog.get_timestamp_by_index(index)
			else:
				ai = eventlog.get_col_value_by_index(value, index)
			if index == 0:
				event_trace[instance.CASE_ID] = [ai]
				continue

			caseid = eventlog.get_caseid_by_index(index-1)

			if instance.CASE_ID == caseid:
				event_trace[instance.CASE_ID].append(ai)

			else:
				event_trace[instance.CASE_ID] = [ai]

		print("Finish")

		x.append(event_trace)

	def _get_trace_count(self, event_trace):
		trace_count = dict()
		traces = event_trace.values()
		for trace in traces:
			trace = tuple(trace)
			if trace not in trace_count:
				trace_count[trace] = 0
			trace_count[trace] += 1
		return trace_count

	def get_caseids(self):
		unique_caseids = self['CASE_ID'].unique()
		return unique_caseids

	def get_activities(self):
		unique_activities = self['Activity'].unique()
		return unique_activities

	def get_resources(self):
		unique_resources = self['Resource'].unique()
		return unique_resources

	def get_timestamps(self):
		unique_timestamps = self['TIMESTAMP'].unique()
		return unique_timestamps

	#특정 col의 unique한 값을 리스트 형태로 리턴
	def get_col_values(self,col):
		return list(set(self[col]))

	def get_first_caseid(self):
		return self['CASE_ID'][0]

	def get_caseid_by_index(self,index):
		return self['CASE_ID'][index]

	def get_resource_by_index(self, index):
		return self['Resource'][index]

	def get_activity_by_index(self, index):
		return self['Activity'][index]

	def get_timestamp_by_index(self, index):
		return self['TIMESTAMP'][index]

	def get_col_value_by_index(self, col, index):
		return self[col][index]

	#특정 col의 특정 value를 포함하는 row를 리턴
	def get_col_value(self, col, value):
		value_df = self.loc[self[col]==value]
		value_df.name = value
		return value_df

	def change_col_value(self, col, old_val, new_val):
		self.loc[self[col]==old_val, col] = new_val
		return self

	def col_val_to_numeric(self, col):
		"""
		To make a chart using bokeh, x values and y values must be numeric.
		Accordingly, change column values to numeric so that it can be properly drawn by bokeh

		Key arguements
		col -- column to be converted to numeric
		"""
		self.sort_values(by=col, inplace=True)
		self.reset_index(drop=True, inplace=True)
		indexs = []
		i=1
		for index, instance in self.iterrows():
			if index==0:
				indexs.append(i)
				continue
			value = self[col][index-1]
			if instance[col] != value:
				i+=1
			indexs.append(i)
		self.loc[:, 'new_col'] = indexs
		return self

	def filter(self, criterion, value):
		return self.loc[self[criterion] == value, :]

	# 특정 col에 특정 value를 포함하는 row를 삭제
	def remove_col_value(self, col, value):
		return self.loc[self[col] != value]

	#eventlog의 event 총 개수를 리턴
	def count_event(self):
		return len(self.index)

	#eventlog 내 case의 개수를 리턴
	def count_case(self):
		return len(set(self['CASE_ID']))

	#특정 col의 unique한 값의 개수를 리턴
	def count_col_values(self, col):
		return len(set(self[col]))

	#모든 col의 unique한 값의 개수를 프린트함
	def show_col_counts(self):
		columns = self.columns
		for col in columns:
			print("unique counts of {}: {}".format(col,len(set(self[col]))))

	def count_col_case(self, col):
		col_case = self.groupby(col).CASE_ID.apply(list).apply(set)
		col_case_count = col_case.apply(len)
		col_case_count_mean = np.mean(col_case_count)
		col_case_count_std = np.std(col_case_count)
		print("CLUSTER count: {}".format(col_case_count))
		print("CLUSTER count mean: {}".format(col_case_count_mean))
		print("CLUSTER count std: {}".format(col_case_count_std))
		return col_case_count

	def count_duplicate_values(self, eventlog, **kwargs):
		"""특정 값이 중복되는 경우 중복횟수의 빈도를 return함
		e.g. 1번 중복: 100, 2번 중복: 300

		Keyword arguments:
		col -- 특정 col이 중복된 것을 확인하고 싶은 경우 (default: Activity)

		"""
		if 'col' in kwargs:
			col = kwargs['col']
			traces = eventlog.get_event_trace(workers=4, value=col)
		else:
			traces = eventlog.get_event_trace(workers=4, value='Activity')
		count=0
		inv_act_counts = []
		for t in traces:
			act_count = dict(Counter(traces[t]))

			inv_act_count = dict()
			for k,v in act_count.items():
				if v < 2:
					continue
				if v in inv_act_count:
					inv_act_count[v].append(k)
				else:
					inv_act_count[v] = [k]
			inv_act_counts.append(inv_act_count)

		count_result_step = dict()
		for inv_act_count in inv_act_counts:
			for k in inv_act_count:
				if k not in count_result_step:
					count_result_step[k] = 1
				else:
					count_result_step[k] += 1

		result = pd.DataFrame(list(count_result_step.items()), columns=['repetition', 'count'])
		return result

	def count_loops(self, eventlog, **kwargs):
		"""step이 연속된 경우를 count함. Step1-->Step1인 경우 1, Step1-->Step1-->Step1인 경우 2, 동시에 동일 device에서 수행되었는지도 계산함

		Keyword arguments:
		col -- 특정 col이 중복된 것을 확인하고 싶은 경우 (default: Activity)
		value -- 특정 값이 연속된 것을 확인하고 싶은 경우 e.g. 'Null'
		"""
		if 'col' in kwargs:
			col = kwargs['col']
			traces = eventlog.get_event_trace(workers=4, value=col)
		else:
			traces = eventlog.get_event_trace(workers=4, value='Activity')
		count=0
		if 'value' in kwargs:
			value = kwargs['value']
		else:
			value = 'default'
		for t, r in zip(traces, resource_traces):
			for index, act in enumerate(traces[t]):
				if index == len(traces[t]) -1:
					continue
				if value == 'default':
						count+=1
				else:
					if act == value and traces[t][index+1] == value:
						count+=1
		print("count_consecutives: {}".format(count))
		return count

	def describe(self):
		print("# events: {}".format(len(self)))
		print("# cases: {}".format(len(set(self['CASE_ID']))))
		print("# activities: {}".format(len(set(self['Activity']))))
		print("# resources: {}".format(len(set(self['Resource']))))
		try:
			print("average yield: {}".format(np.mean(self['VALUE'])))
		except AttributeError:
			print("yield not exists")

	def split_on_case(self, split):
		caseid = self.get_caseids()
		sub_cases = []
		for d in np.array_split(caseid, split):
			sub_cases.append(d)
		sub_logs = []
		for i in range(len(sub_cases)):
			sub_log = self.loc[self['CASE_ID'].isin(sub_cases[i]), :]
			sub_log.reset_index(drop=True, inplace=True)
			sub_logs.append(sub_log)
		return sub_logs

	def parallelize(self, func, workers=multiprocessing.cpu_count(), *args):
		sublogs = self.split_on_case(workers)
		output = Queue()
		manager = Manager()
		output = manager.list()
		# Setup a list of processes that we want to run
		processes = [Process(target=func, args=(sublogs[i], output)+args) for i in range(len(sublogs))]
		# Run processes
		for p in processes:
		    p.start()

		# Exit the completed processes
		for p in processes:
			p.join()

		return output

	#Relation Dictionary(key : AfterActovoty,, value : PreActivity list)
	##You need to specify the objective of this function
	##Additionally, please try to make the code below more efficient. (Both in terms of performance and visibility)
	def relation_dictionary(self, pre_col, aft_col):
		relation_set = {}
		aft_activity_list = self.get_col_values(pre_col)
		for i in aft_activity_list:
			relation_set[i] = []
		for i in range(len(self)):
			relation_set[self[aft_col][i]].append(self[pre_col][i])

		return relation_set

if __name__ == '__main__':
	"""
	eventlog = Eventlog.from_xes('./example/running_example.xes')
	print(type(eventlog))
	"""
	eventlog = Eventlog.from_txt('/Users/GYUNAM/Desktop/LAB/SAMSUNG_PROJECT/IMPLE/input/Sample_data.txt')

	eventlog = eventlog.assign_caseid('ROOT_LOT_ID', 'WAFER_ID')
	eventlog = eventlog.assign_timestamp('TKIN_TIME', 'TKOUT_TIME')
	print(eventlog)


text:
"""
******************************
process graphml files
******************************
"""

import networkx as nx
import os

temp_path="temp.graphml"

def preprocess_graphml(path):
    #modify graphml
    with open(path, encoding="utf-8") as f:
        data_lines = f.read()

    data_lines = data_lines.replace("GenericNode", "ShapeNode")
    data_lines = data_lines.replace("<y:GroupNode>", "")
    data_lines = data_lines.replace("</y:GroupNode>", "")
    data_lines = data_lines.replace("ProxyAutoBoundsNode", "ShapeNode")
    data_lines = data_lines.replace("""<y:Realizers active="0">""", "")
    data_lines = data_lines.replace("</y:Realizers>", "")

    with open(temp_path, mode="w", encoding="utf-8") as f:
        f.write(data_lines)

def load_graphml(path):
    preprocess_graphml(path)
    g=nx.read_graphml(temp_path)
    os.remove(temp_path)
    return g

ext: py

max_stars_repo_name: KanHatakeyama/flowmater

max_issues_repo_name: KanHatakeyama/flowmater

max_forks_repo_name: KanHatakeyama/flowmater

reason: import networkx


text:
# -*- coding: utf-8 -*-
# @Author       : William
# @Project      : TextGAN-william
# @FileName     : cot_instructor.py
# @Time         : Created at 2020/4/21
# @Blog         : http://zhiweil.ml/
# @Description  :
# Copyrights (C) 2018. All Rights Reserved.

import numpy as np
import torch
import torch.optim as optim
from tqdm import tqdm

import config as cfg
from instructor.real_data.instructor import BasicInstructor
from models.CoT_D_trans import Cot_D_trans
from models.CoT_G_trans import CoT_G_trans
from utils.data_loader import GenDataIter

class CoTTransInstructor(BasicInstructor):
    def __init__(self, opt):
        super(CoTTransInstructor, self).__init__(opt)

        # generator, discriminator
        self.gen = CoT_G_trans(cfg.gen_embed_dim,  cfg.vocab_size, cfg.max_seq_len,
                         cfg.padding_idx, gpu=cfg.CUDA)
        self.dis = Cot_D_trans(cfg.gen_embed_dim * 2,  cfg.vocab_size, cfg.max_seq_len,
                         cfg.padding_idx, gpu=cfg.CUDA)  # embed_dim and hidden_dim is larger
        self.init_model()

        # Optimizer
        self.gen_opt = optim.Adam(self.gen.parameters(), lr=cfg.gen_lr)
        self.gen_adv_opt = optim.Adam(self.gen.parameters(), lr=cfg.gen_lr)
        self.dis_opt = optim.Adam(self.dis.parameters(), lr=cfg.gen_lr)

    def _run(self):
        # ===PRE-TRAINING===
        # TRAIN GENERATOR
        if not cfg.gen_pretrain:
            self.log.info('Starting Generator MLE Training...')
            self.pretrain_generator(cfg.MLE_train_epoch)
            if cfg.if_save and not cfg.if_test:
                torch.save(self.gen.state_dict(), cfg.pretrained_gen_path)
                print('Save pre-trained generator: {}'.format(cfg.pretrained_gen_path))

        # ===ADVERSARIAL TRAINING===
        self.log.info('Starting Adversarial Training...')

        progress = tqdm(range(cfg.ADV_train_epoch))
        for epoch in progress:
            g_loss = self.adv_train_generator(cfg.ADV_g_step)  # Generator
            d_loss = self.train_mediator(epoch, cfg.ADV_d_step)  # Discriminator

            progress.set_description('g_loss: %.4f, d_loss: %.4f' % (g_loss, d_loss))

            if epoch % cfg.adv_log_step == 0 or epoch == cfg.ADV_train_epoch - 1:
                self.log.info('[ADV]: epoch = %d, %s' % (epoch, self.cal_metrics(fmt_str=True)))
                if cfg.if_save and not cfg.if_test:
                    self._save('ADV', epoch)
                    torch.save(self.dis.state_dict(), cfg.pretrained_dis_path)

    def _test(self):
        print('>>> Begin test...')

        self._run()
        pass

    def pretrain_generator(self, epochs):
        """
        Max Likelihood Pre-training for the generator
        """
        for epoch in range(epochs):
            self.sig.update()
            if self.sig.pre_sig:
                pre_loss = self.train_gen_epoch(self.gen, self.train_data.loader, self.mle_criterion, self.gen_opt)

                # ===Test===
                if epoch % cfg.pre_log_step == 0 or epoch == epochs - 1:
                    self.log.info(
                        '[MLE-GEN] epoch %d : pre_loss = %.4f, %s' % (epoch, pre_loss, self.cal_metrics(fmt_str=True)))
                    if cfg.if_save and not cfg.if_test:
                        self._save('MLE', epoch)
            else:
                self.log.info('>>> Stop by pre signal, skip to adversarial training...')
                break

    def adv_train_generator(self, g_step):
        """
        Train the generator with mediator rewards
        """
        g_loss = []
        for step in range(g_step):
            inp, target = GenDataIter.prepare(self.gen.sample(cfg.batch_size, cfg.batch_size), gpu=cfg.CUDA)

            # ===Train===
            rewards = self.dis(inp, self.dis.init_hidden(cfg.batch_size))
            loss = self.gen.get_loss(inp, rewards)
            self.optimize(self.gen_adv_opt, loss)
            g_loss.append(loss.item())

        return np.mean(g_loss)

    def train_mediator(self, cur_epoch, d_step):
        """
        Training the mediator on real_data_samples (positive) and generated samples from gen (negative).
        """
        d_loss = []
        for step in range(d_step):
            # prepare loader for training
            real = list(self.train_data.loader)[cur_epoch % len(self.train_data.loader)]  # traverse all real data
            real_inp, real_tar = real['input'], real['target']
            fake_inp, fake_tar = GenDataIter.prepare(self.gen.sample(cfg.batch_size, cfg.batch_size), gpu=cfg.CUDA)
            if cfg.CUDA:
                real_inp, real_tar = real_inp.cuda(), real_tar.cuda()

            real_pred = self.dis.get_pred(real_inp, real_tar)
            fake_pred = self.dis.get_pred(fake_inp, fake_tar)
            loss = -torch.mean(real_pred + fake_pred) / 2.0

            self.optimize(self.dis_opt, loss)
            d_loss.append(loss.item())

        return np.mean(d_loss)

max_stars_repo_name: ripplesaround/DialTransGAN

max_issues_repo_name: ripplesaround/DialTransGAN

max_forks_repo_name: ripplesaround/DialTransGAN

reason: import numpy


text:
import random
import os
import time
import neat
import visualize
import pickle
from bareSnake import snakeGame
import numpy as np
import math
from neat.graphs import feed_forward_layers
from neat.graphs import required_for_output
import tensorflow as tf

tf.config.run_functions_eagerly(True)

gen = 0

stillGradTrain = True
stillDo = True
avgHld = []

pastGamesStates = [[],[]]

def makeList(mat):
	hld = []
	for i in mat:
		for j in i:
			#print(j)
			hld.append(j)
	return hld

def find_index(array):
    if array == [0,0,0,0]:
        return np.random.choice(4, 4, replace=False)
    s = np.array(array)
    sort_index = np.argsort(s)
    return sort_index

def eval_genomes(genomes, config):
    #print('in eval')
    print(len(pastGamesStates[0]))

    global gen
    global stillDo
    global stillGradTrain

    gen += 1

    nets = []
    games = []
    ge = []

    for genome_id, genome in genomes:
        genome.fitness = 0
        net = neat.nn.FeedForwardNetwork.create(genome, config)
        nets.append(net)
        games.append(snakeGame())
        ge.append(genome)
        #print(genome)

    run = True
    tab = 0

    while run and len(games) > 0:
        # print("\n\n")
        # print("got out of enum looop")
        # print("\n")
        tab +=1

        if stillDo == False and stillGradTrain == True:
            print("flipped gradtrain")
            stillGradTrain = False

        for x, game in enumerate(games):
            # print("the games szie is ")
            # print(len(games))
            # print("\n\n\n in top of enumerate")
            assert ge[x] == ge[games.index(game)]

            # print(x)
            ge[x].game = game
            moved = False
            #output = nets[games.index(game)].activate((makeList(game.toString())))
            output = nets[games.index(game)].activate((makeList(game.sight)))
            ind_arr = find_index(output)
            max_index = ind_arr[-1]
            nex_index = ind_arr[-2]
            third_index = ind_arr[-3]
            fourth_index = ind_arr[-4]

            enumTime1 = time.time()

            lr = .05
            batchSize = 50
            genSize = 10

            if gen % genSize == 0 and len(pastGamesStates[0]) > batchSize and stillGradTrain:
                stillDo = False
                print("flipped stillDo")
                batchTime1 = time.time()
                print("about to start 10 sized training loop on all examples")
                for i in range(200):
                    print(i)
                    nets[games.index(game)].backProp_GenomeFast(pastGamesStates[0][-batchSize:-1], pastGamesStates[1][-batchSize:-1], lr,ge[x])
                avgHld.append(nets[games.index(game)].batchAcc(pastGamesStates[0][-batchSize:-1],pastGamesStates[1][-batchSize:-1]))
            #
            #     print("\n the batch acc is")
            #     print(nets[games.index(game)].batchAcc(pastGamesStates[0][0:10],pastGamesStates[1][0:10]))
            #     batchTime2 = time.time()
            #     print("the post training time for genome")
               # print(batchTime2-batchTime1)
            if gen % (genSize + 1) == 0:
                stillDo = True
                stillGradTrain = True
            # print("\n\n\n")
            # print("got out of decsent loop")
            # print("\n\n")

            initGame = game
            initScore = game.score
            initFit = ge[x].fitness

            commit = False
            #print(output)
            if max_index == 0:
                #print("went left")
                direction = "left"
                commit = initGame.check4Mem(direction)
                # if commit == True:
                #     print("in indexing")
                #     print(direction)

                #game.move_left()
            if max_index == 1:
                #print("went right")
                direction = "right"
                commit = initGame.check4Mem(direction)
                # if commit == True:
                #     print("in indexing")
                #     print(direction)
                #game.move_right()
            if max_index == 2:
                #print("went up")
                direction = "up"
                commit = initGame.check4Mem(direction)
                # if commit == True:
                #     print("in indexing")
                #     print(direction)
                #game.move_up()
            if max_index == 3:
                #print("went down")
                direction = "down"
                commit = initGame.check4Mem(direction)
                # if commit == True:
                #     print("in indexing")
                #     print(direction)
                #game.move_down()

            if commit == True:
                innies, outties = initGame.commit2Mem(max_index)
                pastGamesStates[0].append(makeList(innies))
                pastGamesStates[1].append(outties)
                # print("\n \n \n")
                # print(makeList(innies))
                # print(outties)
                # print(direction)
                # print(initGame.munchMove(direction))
                # print(initGame.movedFromWall(direction))
                # print(initGame.movedTowardsFood(direction))
                # print(initGame.check4Mem(direction))

            if max_index == 0:
                game.move_left()
            if max_index == 1:
                game.move_right()
            if max_index == 2:
                game.move_up()
            if max_index == 3:
                game.move_down()

            move_dict = {0:game.propDirect([-10, 0]),1:game.propDirect([10,0]),2:game.propDirect([0, -10]),3:game.propDirect([0,10])}
            newScore = game.score

            alive = game.checkGame()

            enumTime2 = time.time()

            # if gen % 5 == 0 and len(pastGamesStates) > batchSize:
            #     print("the full genome training time")
            #     print(enumTime2 - enumTime1)

            if game.popCount > 400:
                alive = False

            if alive == False:
                if len(games) == 1:

                    hlder = 33

                nets.pop(games.index(game))
                ge.pop(games.index(game))
                games.pop(games.index(game))

            else:

                if game.popCount < 40:
                    ge[x].fitness += 5
                    # if move_dict[max_index][1] == 1:
                    #     ge[x].fitness += 10

                if newScore - initScore > 0:
                    ge[x].fitness += 200

                if ge[x].fitness > initFit and game.popCount >40:
                    print("something is not good")

def run(config_file):
    """
    runs the NEAT algorithm to train a neural network to play flappy bird.
    :param config_file: location of config file
    :return: None
    """
    config = neat.config.Config(neat.DefaultGenome, neat.DefaultReproduction,
                         neat.DefaultSpeciesSet, neat.DefaultStagnation,
                         config_file)
    #print('post config')
    # Create the population, which is the top-level object for a NEAT run.
    p = neat.Population(config)

    # Add a stdout reporter to show progress in the terminal.
    p.add_reporter(neat.StdOutReporter(True))
    stats = neat.StatisticsReporter()
    p.add_reporter(stats)
    #p.add_reporter(neat.Checkpointer(5))

    prerunTime = time.time()

    # Run for up to 50 generations.
    winner = p.run(eval_genomes,30)

    postRunTime = time.time()

    print("the entire time is ")
    print(postRunTime-prerunTime)

    # show final stats
    print('\nBest genome:\n{!s}'.format(winner))
    node_names = {0:'left',1:'right',2:'up',3:'down',-1:'E body',-2:'E food',-3:'E dist'
                                                    ,-4:'S body',-5:'S food',-6:'S dist'
                                                    ,-7:'N body',-8:'N food',-9:'N dist'
                                                    ,-10:'W body',-11:'W food',-12:'W dist'
                                                    ,-13:'SE body',-14:'SE food',-15:'SE dist'
                                                    ,-16:'NE body',-17:'NE food',-18:'NE dist'
                                                    ,-19:'SW body',-20:'SW food',-21:'SW dist'
                                                    ,-22:'NW body',-23:'NW food',-24:'NW dist'}
    #visualize.draw_net(config, winner, True, 'snakeMultLayerStnd',node_names=node_names)
    print(repr(winner))
    print("the fitness is ")
    print(winner.fitness)

    print("the averages are")
    print(avgHld)
    print("mean of means")
    print(np.mean(avgHld))
    print("standard D")
    print(np.std(avgHld))

    visualize.plot_stats(stats,"Population's average and best fitness: Standard noConnected", ylog=False, view=True)
    visualize.plot_species(stats, view=True)

    connections = [cg.key for cg in winner.connections.values() if cg.enabled]
    print("printing connections")
    print(connections)

    layers = feed_forward_layers(config.genome_config.input_keys, config.genome_config.output_keys, connections)
    print("printing layers")
    print(layers)

    req = required_for_output(config.genome_config.input_keys, config.genome_config.output_keys, connections)
    print("printing required")
    print(req)

    #print(winner.game.history)

if __name__ == '__main__':
    #print('in start')
    # Determine path to configuration file. This path manipulation is
    # here so that the script will run successfully regardless of the
    # current working directory.
    local_dir = os.path.dirname(__file__)
    config_path = os.path.join(local_dir, 'configTimed.txt')
    run(config_path)

max_stars_repo_name: QuantumChamploo/NeilNeat

max_issues_repo_name: QuantumChamploo/NeilNeat

max_forks_repo_name: QuantumChamploo/NeilNeat


text:
# -*- coding: utf-8 -*-
"""machine learning.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1D94ewZwA_xvA1hsyLlTOeGDZ4TNM48JE

# **Customer Personality Prediction or Analysis![824-8247150_click-happy-customers-cartoon.png](data:image/png;base64,[encoded data, 204936 characters])**

# Import what we neeed for this work ::
"""

# Commented out IPython magic to ensure Python compatibility.
import os, sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from datetime import datetime
import seaborn as sns
import scipy
import random
import math
import dabl
from scipy.stats.mstats import winsorize
from tqdm import tqdm
from sklearn import preprocessing
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.linear_model import LogisticRegression
from sklearn.kernel_ridge import KernelRidge
from sklearn.model_selection import train_test_split
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import fbeta_score, make_scorer
from sklearn import svm

import plotly.graph_objects as go
import plotly.express as px
import plotly.figure_factory as ff
# %matplotlib inline

from IPython.display import clear_output
!pip install dabl
clear_output()

import dabl

import numpy as np
import pandas as pd

from sklearn.model_selection import train_test_split

import matplotlib.pyplot as plt

from IPython.display import clear_output
!pip install dabl
clear_output()

"""

```
# This is formatted as code
```

**Add Google Drive in colab :**

---

"""

from google.colab import drive
drive.mount('/content/drive')

"""Adding CSV data file from google drive :    """

data = pd.read_csv('/content/drive/MyDrive/machine learning Lab Project/customer.csv')
data_copy = data.copy()

"""Reading that CSV file :"""

data

"""Showing Head of data file :"""

data.head()

"""Showing Tail of CSV data :"""

data.tail()

"""Seeing Data information :"""

data.info()

data.columns

data.describe().T

"""Data Clearing Work :"""

data.isnull().sum()

dabl_data=dabl.clean(data, verbose=1 )

types = dabl.detect_types(dabl_data)
types

Target ="Response"
ID="ID"

X = data.drop([ID,Target],axis=1)
Y = data[Target]

data.head()

data.info()

"""## Model Train here : """

X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.4, random_state=45,stratify=Y)

print(X_train.shape,X_test.shape)

print(Y_train.shape,Y_test.shape)

train = pd.concat([X_train,Y_train], axis=1)

train.head()

"""## Data Testing : """

Test = dabl.SimpleClassifier(random_state=42).fit(train, target_col=Target)

Test.current_best_

dabl.explain(Test)

Test.est_

"""## Data Visualization"""

data = data.set_index('ID')

data['Age'] = int(pd.datetime.now().year) - data['Year_Birth']
data_copy['Age'] = int(pd.datetime.now().year) - data_copy['Year_Birth']

print("Columns with string datatype are:")
for col in data.columns:
    if data[col].dtypes == object:
        print(col)

data_Edu = pd.DataFrame(data['Education'].value_counts()).reset_index()
data_Edu.columns = ['Education', 'Count']

data['Education'] = np.where(data['Education'] == '2n Cycle', 'Master', data['Education'])

data_Edu = pd.DataFrame(data['Education'].value_counts()).reset_index()
data_Edu.columns = ['Education', 'Count']

fig = px.bar(data_Edu,
            x='Education',
            y='Count',
            color='Education')
fig.update_layout(width=800, height=400, title='Education ')
fig.show()

fig = plt.figure(figsize=(10,6))

plt.hist(data.Year_Birth, color='#adc987')
plt.ylabel('Number of Person')
plt.xlabel('Birth Years')
plt.title('Customers Birth Year Distrubiton')

data_Mar = pd.DataFrame(data['Marital_Status'].value_counts()).reset_index()
data_Mar.columns = ['Marital_Status', 'Count']
data_Mar

mar_stat = ['Single', 'Widow', 'Alone', 'Absurd', 'YOLO']
data['Marital_Status'] = np.where(data['Marital_Status'].isin(mar_stat), 'Single', data['Marital_Status'])
data['Marital_Status'] = np.where(data['Marital_Status'].isin(['Married', 'Together']), 'Relationship', 'Single')
data_Mar = pd.DataFrame(data['Marital_Status'].value_counts()).reset_index()
data_Mar.columns = ['Marital_Status', 'Count']
data_Mar

data_Edu = pd.DataFrame(data['Marital_Status'].value_counts()).reset_index()
data_Edu.columns = ['Marital_Status', 'Count']

fig = px.bar(data_Edu,
            x='Marital_Status',
            y='Count',
            color='Marital_Status')
fig.update_layout(width=800, height=400, title='Relation ')
fig.show()

data['Dt_Customer'] = pd.to_datetime(data['Dt_Customer'], utc=False)

print(f"The youngest customer is {data['Age'].min()} years old and oldest customer is {data['Age'].max()} years old")

data.isna().sum()[lambda x: x>0]

data['Income'] = data['Income'].fillna(data['Income'].mean())

age_data = data.groupby(by = ['Year_Birth']).agg({'Income':'mean'}).reset_index()
age_data['Year_Birth'] = 2021 - age_data['Year_Birth']

fig = px.bar(age_data, x = 'Year_Birth', y = 'Income')
fig.update_layout(height=400, width=700, title_text="Age Vs Average Income")
fig.show()

"""## Shopping Products Selling Prediction :"""

from plotly.subplots import make_subplots
import plotly.graph_objects as go

def create_interval_column(age_data, interval):
    inter = []
    interval = interval
    j = 0
    while (j<100):
        j = j + interval
        inter.append(j)
    interval_column = []
    for i in age_data['Year_Birth']:
        for j in range(len(inter)-1):
            if inter[j]<i <=inter[j+1]:
                interval_column.append(str(inter[j]) + '-' + str(inter[j+1]))
                break
    return interval_column

interval_you_want_to_plot = 10

columns_to_be_analyzed = ['MntWines', 'MntFruits' ,'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts', 'MntGoldProds']

age_data = data.groupby(by = ['Year_Birth']).agg({'MntWines':'sum','MntFruits':'sum' ,'MntMeatProducts':'sum',
                                              'MntFishProducts':'sum', 'MntSweetProducts':'sum', 'MntGoldProds':'sum' }).reset_index()
age_data['Year_Birth'] = 2021 - age_data['Year_Birth']
age_data.drop([0,1,2], axis = 0, inplace=True)

interval_column = create_interval_column(age_data, interval=interval_you_want_to_plot )# Creating interval of 5

age_data['Interval_column'] = interval_column

fig = make_subplots(rows = 3, cols = 3, subplot_titles=columns_to_be_analyzed)
cnt = 0
for i in range(2):
    for j in range(3):
        fig.add_trace(go.Bar(x = age_data['Interval_column'].to_numpy(),
                             y = age_data[columns_to_be_analyzed[cnt]].to_numpy()),
                    row = i+1, col=j+1 )
        cnt+=1
fig.update_layout(  title = 'Columns Vs Amount of quantity',font=dict(
        family="Courier New, monospace",
        size=12,
        color="#7f7f7f"),
                 showlegend=False,autosize=True,
                  width=1200,
                  height=800)
fig.show()

Edu_data = data.groupby(by = ['Education']).agg({'MntWines':'sum','MntFruits':'sum' ,'MntMeatProducts':'sum',
                                              'MntFishProducts':'sum', 'MntSweetProducts':'sum', 'MntGoldProds':'sum' }).reset_index()

fig = make_subplots(rows = 3, cols = 3, subplot_titles=columns_to_be_analyzed)
cnt = 0
for i in range(2):
    for j in range(3):
        fig.add_trace(go.Bar(x = Edu_data['Education'].to_numpy(),
                             y = Edu_data[columns_to_be_analyzed[cnt]].to_numpy()), row = i+1, col=j+1 )
        cnt+=1
fig.update_layout(  title = 'Columns Vs Amount of quantity',font=dict(
        family="Courier New, monospace", size=12, color="#7f7f7f"), showlegend=False,autosize=True,width=1200,height=800)
fig.show()

Marital_data = data.groupby(by = ['Marital_Status']).agg({'MntWines':'sum','MntFruits':'sum' ,'MntMeatProducts':'sum',
                                              'MntFishProducts':'sum', 'MntSweetProducts':'sum', 'MntGoldProds':'sum' }).reset_index()

fig = make_subplots(rows = 3, cols = 3, subplot_titles=columns_to_be_analyzed)
cnt = 0
for i in range(2):
    for j in range(3):
        fig.add_trace(go.Bar(x = Marital_data['Marital_Status'].to_numpy(),
                             y = Marital_data[columns_to_be_analyzed[cnt]].to_numpy()), row = i+1, col=j+1 )
        cnt+=1
fig.update_layout(  title = 'Columns Vs Amount of quantity',font=dict(
        family="Courier New, monospace", size=12, color="#7f7f7f"), showlegend=False,autosize=True,width=1200,height=800)
fig.show()

interval_you_want_to_plot = 10

columns_to_be_analyzed = ['NumDealsPurchases', 'NumWebPurchases', 'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth']

age_data = data.groupby(by = ['Year_Birth']).agg({'NumDealsPurchases':'sum','NumWebPurchases':'sum' ,'NumCatalogPurchases':'sum',
                                              'NumStorePurchases':'sum', 'NumWebVisitsMonth':'sum' }).reset_index()
age_data['Year_Birth'] = 2021 - age_data['Year_Birth']
age_data.drop([0,1,2], axis = 0, inplace=True)

interval_column = create_interval_column(age_data, interval=interval_you_want_to_plot )# Creating interval of 5

age_data['Interval_column'] = interval_column

fig = make_subplots(rows = 2, cols = 3, subplot_titles=columns_to_be_analyzed)
cnt = 0
for i in range(2):
    for j in range(3):
        if cnt == 5 :
          break
        fig.add_trace(go.Bar(x = age_data['Interval_column'].to_numpy(),
                             y = age_data[columns_to_be_analyzed[cnt]].to_numpy()), row = i+1, col=j+1 )
        cnt+=1
fig.update_layout(  title = 'Columns Vs Number of Purchase',font=dict(
        family="Courier New, monospace",size=12,color="#7f7f7f"),showlegend=False,autosize=True,width=1200,height=800)
fig.show()

Edu_data = data.groupby(by = ['Education']).agg({'NumDealsPurchases':'sum','NumWebPurchases':'sum' ,'NumCatalogPurchases':'sum',
                                              'NumStorePurchases':'sum', 'NumWebVisitsMonth':'sum' }).reset_index()

fig = make_subplots(rows = 2, cols = 3, subplot_titles=columns_to_be_analyzed)
cnt = 0
for i in range(2):
    for j in range(3):
        if cnt == 5 :
          break
        fig.add_trace(go.Bar(x = Edu_data['Education'].to_numpy(),
                             y = Edu_data[columns_to_be_analyzed[cnt]].to_numpy()), row = i+1, col=j+1 )
        cnt+=1
fig.update_layout(  title = 'Columns Vs Number of Purchase',font=dict(
        family="Courier New, monospace",size=12,color="#7f7f7f"),showlegend=False,autosize=True,width=1200,height=800)
fig.show()

PALETTE = sns.color_palette("Set2")

num = data.filter(regex='Num[^Deals].+Purchases').sum(axis=0)

sizes = dict(num)

plt.figure(figsize=(12, 8))
plt.title("Shopping types proportions")
plt.pie(sizes.values(), labels=['Website', 'Catalog', 'Store'], autopct="%.1f%%", pctdistance=0.85, shadow=True, colors=PALETTE)
plt.legend(title="Purchased at", labels=['Website', 'Catalog', 'Store'], bbox_to_anchor=(1, 1))

plt.show()

fig = plt.figure(figsize=(10,6))

plt.scatter(data.Age, data.NumWebPurchases, color='#88c999', alpha=0.4, label='Web Buys')
plt.scatter(data.Age, data.NumStorePurchases, color='#5f79c9', alpha=0.4, label='Store Buys')

plt.legend()

plt.ylabel('Web and Store Purchases')
plt.xlabel('Customers Age')
plt.title('Web vs Store Pruchases According to Age')

"""# Customer Offer Acceptance Prediction"""

# imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as se
import warnings
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report,plot_confusion_matrix
warnings.filterwarnings('ignore')
from sklearn.preprocessing import MinMaxScaler
from sklearn.svm import LinearSVC

# Selected Columns
features=['Income', 'Kidhome', 'Teenhome', 'Recency', 'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts', 'MntGoldProds', 'NumWebPurchases', 'NumCatalogPurchases', 'NumStorePurchases', 'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1', 'AcceptedCmp2', 'Education', 'Marital_Status']
target='Response'
# X & Y
X=data[features]
Y=data[target]

# Data Cleaning
def NullClearner(value):
	if(isinstance(value, pd.Series) and (value.dtype in ['float64','int64'])):
		value.fillna(value.mean(),inplace=True)
		return value
	elif(isinstance(value, pd.Series)):
		value.fillna(value.mode()[0],inplace=True)
		return value
	else:return value
x=X.columns.to_list()
for i in x:
	X[i]=NullClearner(X[i])
Y=NullClearner(Y)

# Handling AlphaNumeric Features
X=pd.get_dummies(X)

f,ax = plt.subplots(figsize=(18, 18))
matrix = np.triu(X.corr())
se.heatmap(X.corr(), annot=True, linewidths=.5, fmt= '.1f',ax=ax, mask=matrix)
plt.show()

columns=X.columns
X=MinMaxScaler().fit_transform(X)
X=pd.DataFrame(data = X,columns = columns)
X.head()

# Data split for training and testing
X_train,X_test,Y_train,Y_test=train_test_split(X,Y,test_size=0.2,random_state=123)

#Model Parameters
param={'C': 1, 'loss': 'squared_hinge', 'tol': 0.05092207964551096, 'penalty': 'l2'}
# Model Initialization
model=LinearSVC(**param)
model.fit(X_train,Y_train)

# Confusion Matrix
plot_confusion_matrix(model,X_test,Y_test,cmap=plt.cm.Blues)
# Classification Report
print(classification_report(Y_test,model.predict(X_test)))

reason: import numpy,import scipy,from scipy


text:
import numpy as np
import numba as nb
import importlib
# from numba.typed import List
import mcmc.util as util
import mcmc.fourier as fourier
import mcmc.L as L
import mcmc.layer as layer
import mcmc.randomGenerator as randomGenerator
import mcmc.pCN as pCN
import mcmc.measurement as meas
import mcmc.simulationResults as simRes
# import scipy as scp
import time
import h5py

# fourier_type = nb.deferred_type()
# fourier_type.define(fourier.FourierAnalysis.class_type.instance_type)
# Layer_type = nb.deferred_type()
# Layer_type.define(layer.Layer.class_type.instance_type)
# L_matrix_type = nb.deferred_type()
# L_matrix_type.define(L.Lmatrix.class_type.instance_type)
# Rand_gen_type = nb.deferred_type()
# Rand_gen_type.define(randomGenerator.RandomGenerator.class_type.instance_type)
# pCN_type = nb.deferred_type()
# pCN_type.define(pCN.pCN.class_type.instance_type)
# meas_type = nb.deferred_type()
# meas_type.define(meas.Measurement.class_type.instance_type)
# sim_result_type = nb.deferred_type()
# sim_result_type.define(simRes.SimulationResult.class_type.instance_type)
# spec = [
#     ('fourier',fourier_type),
#     ('random_gen',Rand_gen_type),
#     ('pcn',pCN_type),
#     ('sim_result',sim_result_type),
#     ('Layers',Layer_type[:]),
#     # ('LMat',L_matrix_type),
#     # ('measurement',meas_type),
#     ('n_layers',nb.int64),
#     ('n_samples',nb.int64),
#     ('meas_samples_num',nb.int64),
#     ('evaluation_interval',nb.int64),
#     ('random_seed',nb.int64),
#     ('accepted_count',nb.int64),
#     ('burn_percentage',nb.float64),
#     ('total_time',nb.float64),
#     ('yBar',nb.float64[:]),
#     ('H',nb.complex128[:,:]),
#     ('u_history',nb.complex128[:,:]),
#     ('v_history',nb.complex128[:,:]),
#     ('printProgress',nb.boolean),

# ]

# @nb.jitclass(spec)
class Simulation():
    def __init__(self,n_layers,n_samples,n,beta,num,kappa,sigma_0,sigma_v,sigma_scaling,evaluation_interval,printProgress,
                    seed,burn_percentage,enable_beta_feedback,pcn_variant,measurement_signal_type):
        self.n_samples = n_samples
        self.meas_samples_num = num
        self.evaluation_interval = evaluation_interval
        self.burn_percentage = burn_percentage
        #set random seed
        self.random_seed = seed
        self.printProgress = printProgress
        self.n_layers=n_layers
        self.kappa = kappa
        self.sigma_0 = sigma_0
        self.sigma_v = sigma_v
        self.sigma_scaling = sigma_scaling
        self.enable_beta_feedback = enable_beta_feedback
        np.random.seed(self.random_seed)

        #setup parameters for 1 Dimensional simulation
        self.d = 1
        self.nu = 2 - self.d/2
        self.alpha = self.nu + self.d/2
        self.t_start = 0.0
        self.t_end = 1.0
        self.beta_0 = (sigma_0**2)*(2**self.d * np.pi**(self.d/2))* 1.1283791670955126#<-- this numerical value is scp.special.gamma(alpha))/scp.special.gamma(nu)
        self.beta_v = self.beta_0*(sigma_v/sigma_0)**2
        self.sqrtBeta_v = np.sqrt(self.beta_v)
        self.sqrtBeta_0 = np.sqrt(self.beta_0)

        f =  fourier.FourierAnalysis(n,num,self.t_start,self.t_end)
        self.fourier = f

        rg = randomGenerator.RandomGenerator(f.basis_number)
        self.random_gen = rg

        LuReal = (1/self.sqrtBeta_0)*(self.fourier.Dmatrix*self.kappa**(-self.nu) - self.kappa**(2-self.nu)*self.fourier.Imatrix)
        Lu = LuReal + 1j*np.zeros(LuReal.shape)

        uStdev = -1/np.diag(Lu)
        # uStdev = uStdev[self.fourier.basis_number-1:]
        # uStdev[0] /= 2 #scaled

        meas_std = 0.1
        measurement = meas.Measurement(num,meas_std,self.t_start,self.t_end,measurement_signal_type)
        # pcn = pCN.pCN(n_layers,rg,measurement,f,beta)
        self.pcn_variant = pcn_variant
        self.pcn = pCN.pCN(n_layers,rg,measurement,f,beta,self.pcn_variant)
        # self.pcn_pair_layers = pcn_pair_layers

        #initialize Layers
        typed_list_status = importlib.util.find_spec('numba.typed.typedlist')
        if typed_list_status is None:
            Layers = []
        else:
            from numba.typed.typedlist import List
            Layers = List()
        # Layers = []
        # factor = 1e-8
        self.pcn.record_skip = np.max([1,self.n_samples//self.pcn.max_record_history])
        history_length = np.min([self.n_samples,self.pcn.max_record_history])
        self.pcn.sqrtBetas_history = np.empty((history_length, self.n_layers), dtype=np.float64)

        for i in range(self.n_layers):
            if i==0:
                init_sample = np.linalg.solve(Lu,self.random_gen.construct_w())[self.fourier.basis_number-1:]
                lay = layer.Layer(True,self.sqrtBeta_0,i,self.n_samples,self.pcn,init_sample)
                lay.LMat.current_L = Lu
                lay.LMat.latest_computed_L = Lu
                lay.stdev_sym = uStdev
                lay.stdev = uStdev[self.fourier.basis_number-1:]
                lay.stdev[0] /=2
                lay.current_sample_scaled_norm = util.norm2(lay.current_sample/lay.stdev)#ToDO: Modify this
                lay.new_sample_scaled_norm = lay.current_sample_scaled_norm
            else:

                if i == n_layers-1:

                    lay = layer.Layer(False,self.sqrtBeta_v,i,self.n_samples,self.pcn,Layers[i-1].current_sample)
                    wNew = self.pcn.random_gen.construct_w()
                    eNew = np.random.randn(self.pcn.measurement.num_sample)
                    wBar = np.concatenate((eNew,wNew))

                    LBar = np.vstack((self.pcn.H,lay.LMat.current_L))

                    #update v
                    lay.current_sample_sym, res, rnk, s = np.linalg.lstsq(LBar,self.pcn.yBar-wBar,rcond=-1)#,rcond=None)
                    lay.current_sample = lay.current_sample_sym[self.pcn.fourier.basis_number-1:]
                else:
                    # lay = layer.Layer(False,self.sqrtBeta_v*np.sqrt(sigma_scaling),i,self.n_samples,self.pcn,Layers[i-1].current_sample)
                    lay = layer.Layer(False,self.sqrtBeta_v*sigma_scaling,i,self.n_samples,self.pcn,Layers[i-1].current_sample)

            lay.update_current_sample()
            self.pcn.Layers_sqrtBetas[i] = lay.sqrt_beta
            # if self.pcn_variant:
            #     self.pcn.record_skip = np.max([1,(lay.n_samples*self.n_layers)//self.pcn.max_record_history])
            #     history_length = np.min([lay.n_samples*(self.n_layers),self.pcn.max_record_history])
            # else:

            lay.samples_history = np.empty((history_length, self.fourier.basis_number), dtype=np.complex128)
            Layers.append(lay)

        self.Layers = Layers
        sim_result = simRes.SimulationResult()
        self.sim_result = sim_result

    def run(self):

        self.accepted_count = 0
        self.accepted_count_SqrtBeta = 0
        average_time_intv =0.0
        with nb.objmode(start_time='float64',start_time_intv='float64'):
            if self.printProgress:
                util.printProgressBar(0, self.n_samples, prefix = 'Preparation . . . . ', suffix = 'Complete', length = 50)
            start_time = time.time()
            start_time_intv = start_time
        # print('preparation . . . ')
        # print('Checking random numbers normalRand & logUniform:')

        # totalTime = 0.0
        accepted_count_partial = 0
        accepted_count_partial_SqrtBeta = 0
        linalg_error_occured = False
        for i in range(self.n_samples):#nb.prange(nSim):
            try:
                if self.pcn_variant == "dunlop":
                    accepted_count_partial += self.pcn.one_step_non_centered_dunlop(self.Layers)
                    # accepted,accepted_SqrtBeta = self.pcn.one_step_non_centered_dunlop(self.Layers)
                    # accepted_count_partial += accepted
                    # accepted_count_partial_SqrtBeta += accepted_SqrtBeta
                elif self.pcn_variant == "sari":
                    # accepted,acceptedSqrtBeta =self.pcn.one_step_non_centered_sari(self.Layers)
                    accepted_count_partial += self.pcn.one_step_non_centered_sari(self.Layers)
                else:
                    accepted_count_partial += self.pcn.oneStep(self.Layers)
            except np.linalg.LinAlgError as err:
                linalg_error_occured = True
                print("Linear Algebra Error :",err)
                continue
            else:
                # for j in range(self.fourier.basis_number):
                    # accepted_count_partial += self.pcn.one_step_one_element(self.Layers,j)
                if (i+1)%(self.evaluation_interval) == 0:
                    self.accepted_count += accepted_count_partial
                    # self.accepted_count_SqrtBeta += accepted_count_partial_SqrtBeta

                    # if self.pcn_variant:
                    #     self.acceptancePercentage = self.accepted_count/((i+1)*(self.n_layers))
                    # else:
                    self.acceptancePercentage = self.accepted_count/(i+1)
                    # self.acceptancePercentage_SqrtBeta = self.accepted_count_SqrtBeta/(i+1)

                    if self.enable_beta_feedback:
                        self.pcn.adapt_step(self.acceptancePercentage)
                        # self.pcn.adapt_step_sqrtBeta(self.acceptancePercentage_SqrtBeta)
                    # else:
                    #     if self.acceptancePercentage> 0.5:
                    #         self.pcn.more_aggresive()
                    #     elif self.acceptancePercentage<0.3:
                    #         self.pcn.less_aggresive()
                    #TODO: toggle this if pcn.one_step_one_element is not used
                    # acceptancePercentage = self.accepted_count/((i+1)*self.fourier.basis_number)

                    accepted_count_partial = 0
                    mTime = (i+1)/(self.evaluation_interval)

                    # with nb.objmode(average_time_intv='float64',start_time_intv='float64'):

                    end_time_intv = time.time()
                    time_intv = end_time_intv-start_time_intv
                    average_time_intv +=  (time_intv-average_time_intv)/mTime
                    start_time_intv = end_time_intv
                    remainingTime = average_time_intv*((self.n_samples - i)/self.evaluation_interval)
                    remainingTimeStr = time.strftime("%j-1 day(s),%H:%M:%S", time.gmtime(remainingTime))
                    if self.printProgress:
                        util.printProgressBar(i+1, self.n_samples, prefix = 'Time Remaining {0}- Acceptance Rate {1:.2%} - Progress:'.format(remainingTimeStr,self.acceptancePercentage), suffix = 'Complete', length = 50)

        # with nb.objmode():
        if linalg_error_occured:
            if self.printProgress:
                #truncating the each layer history
                end_index = i%self.pcn.record_skip
                for l in range(self.n_layers):
                    self.Layers[l].samples_history = self.Layers[l].samples_history[:end_index,:]
                print('Linear algebra errors occured during some simulation step(s). The simulation result may not be valid')
        else:
            elapsedTimeStr = time.strftime("%j day(s),%H:%M:%S", time.gmtime(time.time()-start_time))
            self.total_time = time.time()-start_time
            # print('Complete')
            if self.printProgress:
                util.printProgressBar(self.n_samples, self.n_samples, 'Iteration Completed in {0}- Acceptance Rate {1:.2%} - Progress:'.format(elapsedTimeStr,self.acceptancePercentage), suffix = 'Complete', length = 50)

    def analyze(self):
        recorded_chain_length = self.Layers[0].samples_history.shape[0]
        startIndex = np.int(self.burn_percentage*recorded_chain_length//100)

        # vtEs = np.empty((self.measurement.t.shape[0],len(vHistoryBurned)))
        # ut = np.empty((self.measurement.t.shape[0],len(uHistoryBurned)))
        # lU = np.empty((self.measurement.t.shape[0],len(uHistoryBurned)))

        utM = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        utM2 = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        utCount = np.zeros(self.n_layers)
        utAggregateNow=[]
        for i in range(self.n_layers):
            utAggregate_Layer_i_Now = (utCount[i],utM[i,:],utM2[i,:])
            utAggregateNow.append(utAggregate_Layer_i_Now)

        elltM = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        elltM2 = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        elltCount = np.zeros(self.n_layers)
        elltAggregateNow=[]
        for i in range(self.n_layers):
            elltAggregate_Layer_i_Now = (elltCount[i],elltM[i,:],elltM2[i,:])
            elltAggregateNow.append(elltAggregate_Layer_i_Now)

        uHalfRealM = np.zeros((self.n_layers,self.fourier.basis_number),dtype=np.float64)
        uHalfRealM2 = np.zeros((self.n_layers,self.fourier.basis_number),dtype=np.float64)
        uHalfRealCount = np.zeros(self.n_layers)
        uHalfRealAggregateNow=[]
        for i in range(self.n_layers):
            uHalfRealAggregate_Layer_i_Now = (uHalfRealCount[i],uHalfRealM[i,:],uHalfRealM2[i,:])
            uHalfRealAggregateNow.append(uHalfRealAggregate_Layer_i_Now)

        uHalfImagM = np.zeros((self.n_layers,self.fourier.basis_number),dtype=np.float64)
        uHalfImagM2 = np.zeros((self.n_layers,self.fourier.basis_number),dtype=np.float64)
        uHalfImagCount = np.zeros(self.n_layers)
        uHalfImagAggregateNow=[]
        for i in range(self.n_layers):
            uHalfImagAggregate_Layer_i_Now = (uHalfImagCount[i],uHalfImagM[i,:],uHalfImagM2[i,:])
            uHalfImagAggregateNow.append(uHalfImagAggregate_Layer_i_Now)

        sigmas = util.sigmasLancos(self.fourier.basis_number)

        vtHalf = self.fourier.fourierTransformHalf(self.pcn.measurement.vt)
        vtF = self.fourier.inverseFourierLimited(vtHalf*sigmas)
        for i in range(startIndex,recorded_chain_length):
            for j in range(self.n_layers):
                utNow = self.fourier.inverseFourierLimited(self.Layers[j].samples_history[i,:]*sigmas)
                # lUNow = 1/np.flip(util.kappaFun(utNow))#<-  ini aneh ni kenapa harus di flip!!!
                elltNow = util.kappaFun(-utNow)#<-  ini aneh ni kenapa harus di flip!!!

                utAggregateNow[j] = util.updateWelford(utAggregateNow[j],utNow)
                elltAggregateNow[j] = util.updateWelford(elltAggregateNow[j],elltNow)
                uHalfRealAggregateNow[j] = util.updateWelford(uHalfRealAggregateNow[j],self.Layers[j].samples_history[i,:].real)
                uHalfImagAggregateNow[j] = util.updateWelford(uHalfImagAggregateNow[j],self.Layers[j].samples_history[i,:].imag)

        # for i in range(len(vHistoryBurned)):
        # utMean, variance, sampleVariance = util.finalizeWelford(utAggregateNow)
        utMean = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        utVar = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        elltMean = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        elltVar = np.zeros((self.n_layers,self.pcn.measurement.t.shape[0]),dtype=np.float64)
        uHalfMean = np.zeros((self.n_layers,self.fourier.basis_number),dtype=np.complex128)
        uHalfRealVar = np.zeros((self.n_layers,self.fourier.basis_number),dtype=np.float64)
        uHalfImagVar = np.zeros((self.n_layers,self.fourier.basis_number),dtype=np.float64)
        for j in range(self.n_layers):
            utMean[j,:] = utAggregateNow[j][1]
            utVar[j,:] = utAggregateNow[j][2]/utAggregateNow[j][0]

            elltMean[j,:] = elltAggregateNow[j][1]
            elltVar[j,:] = elltAggregateNow[j][2]/elltAggregateNow[j][0]

            uHalfMean[j,:] = uHalfRealAggregateNow[j][1]+1j*uHalfImagAggregateNow[j][1]
            uHalfRealVar[j,:] = uHalfRealAggregateNow[j][2]/uHalfRealAggregateNow[j][0]
            uHalfImagVar[j,:] = uHalfImagAggregateNow[j][2]/uHalfImagAggregateNow[j][0]

        # sim_result = simRes.SimulationResult()
        # sim_result.assign_values(vtHalf,vtF,uHalfMean,np.sqrt(uHalfRealVar),np.sqrt(uHalfImagVar),elltMean,np.sqrt(elltVar),utMean,np.sqrt(utVar))
        self.sim_result.assign_values(vtHalf,vtF,uHalfMean,np.sqrt(uHalfRealVar),np.sqrt(uHalfImagVar),elltMean,np.sqrt(elltVar),utMean,np.sqrt(utVar))

    def save(self,file_name,include_history=False):
        with h5py.File(file_name,'w') as f:
            for key,value in self.__dict__.items():
                NumbaType = 'numba.' in str(type(value))
                ListType = isinstance(value,list)
                if not NumbaType and not ListType:
                    f.create_dataset(key,data=value)
                else:
                    if key == 'sim_result':
                        f.create_dataset('vtHalf',data=value.vtHalf,compression='gzip')
                        f.create_dataset('vtF',data=value.vtF,compression='gzip')
                        f.create_dataset('uHalfMean',data=value.uHalfMean,compression='gzip')
                        f.create_dataset('uHalfStdReal',data=value.uHalfStdReal,compression='gzip')
                        f.create_dataset('uHalfStdImag',data=value.uHalfStdImag,compression='gzip')
                        f.create_dataset('elltMean',data=value.elltMean,compression='gzip')
                        f.create_dataset('elltStd',data=value.elltStd,compression='gzip')
                        f.create_dataset('utMean',data=value.utMean,compression='gzip')
                        f.create_dataset('utStd',data=value.utStd,compression='gzip')
                        continue
                    if key == 'pcn':
                        f.create_dataset('beta',data=value.beta)
                        f.create_dataset('record_skip',data=value.record_skip)
                        f.create_dataset('record_count',data=value.record_count)
                        f.create_dataset('max_record_history',data=value.max_record_history)
                        f.create_dataset('target_acceptance_rate',data=value.target_acceptance_rate)
                        f.create_dataset('beta_feedback_gain',data=value.beta_feedback_gain)
                        f.create_dataset('Layers_sqrtBetas',data=value.Layers_sqrtBetas)
                        f.create_dataset('stdev_sqrtBetas',data=value.stdev_sqrtBetas)
                        f.create_dataset('sqrtBetas_history',data=value.sqrtBetas_history)
                        continue
                    if include_history and key == 'Layers':
                        for i in range(self.n_layers):
                            f.create_dataset('Layer - {0} samples'.format(i),data=value[i].samples_history,compression='gzip')

reason: import numpy,import scipy,import numba,from numba


text:
from __future__ import absolute_import
from __future__ import print_function
import sys,os
import pandas as pd
#import get_compounds_from_wikidata as wd
sys.path.append('/global/project/projectdirs/openmsi/jupyterhub_libs/anaconda/lib/python2.7/site-packages')
from rdkit import Chem
import numpy as np
from rdkit.Chem import PandasTools

# MolToSmiles( (Mol)mol [, (bool)isomericSmiles=False
# http://www.rdkit.org/Python_Docs/rdkit.Chem.rdmolfiles-module.html#MolToSmiles
#         - isomericSmiles: (optional) include information about stereochemistry in
#           the SMILES.  Defaults to false.

# https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/inchikey/RZJQGNCSTQAWON-UHFFFAOYSA-N/synonyms/json

def desalt(mol):
    #input is an rdkit mol
    #returns an rdkit mol keeping the biggest component
    #returns original mol if only one component
    #returns a boolean indicated if cleaning was necessary
    d = Chem.rdmolops.GetMolFrags(mol) #these are atom indices
    if len(d) == 1: #If there are fragments or multiple molecules this will be greater than 1
        return mol,False
    my_smiles=Chem.MolToSmiles(mol,True)
    parent_atom_count=0;
    disconnected=my_smiles.split('.')
    #With GetMolFrags, we've already established that there is more than one disconnected structure
    status = False
    for s in disconnected:
        little_mol=Chem.MolFromInchi(Chem.MolToInchi(Chem.MolFromSmiles(s)))
        if little_mol is not None:
            count = little_mol.GetNumAtoms()
            if count > parent_atom_count:
                parent_atom_count = count
                parent_mol = little_mol
                status = True
    return parent_mol,status

def get_wikidata(terms_to_keep):
    prop_file = '/project/projectdirs/openmsi/projects/compound_data/wikidata/wikidata_compound_properties.xlsx'
    result = wd.get_wikidata(prop_file)
    df = pd.DataFrame(result)
    df.rename(columns={'canonicalSMILES': 'smiles'}, inplace=True)
    df.rename(columns={'InChI': 'inchi'}, inplace=True)
    df.rename(columns={'compoundLabel': 'common_name'}, inplace=True)
    df['source_database'] = 'wikidata'
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def get_img(terms_to_keep):
    df = pd.read_csv('/project/projectdirs/openmsi/projects/compound_data/img_abc/NPlist32763_30-mar-2016.xls',delimiter='\t')
    df.rename(columns={'SMILES': 'smiles'}, inplace=True)
    df.rename(columns={'InChl': 'inchi'}, inplace=True)
    df.rename(columns={'SM ID': 'img_abc_id'}, inplace=True)
    df.rename(columns={'Secondary Metabolite (SM) Name': 'common_name'}, inplace=True)
    df['source_database'] = 'img'
    df['ROMol'] = ''
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def get_enzo(terms_to_keep):
    df = pd.read_csv('/project/projectdirs/openmsi/projects/compound_data/enzo/BML-2865.txt',delimiter='\t')
    df.rename(columns={'SMILES': 'smiles'}, inplace=True)
    df.rename(columns={'Name': 'common_name'}, inplace=True)
    df['inchi'] = np.nan
    df['source_database'] = 'enzo'
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def get_msmls(terms_to_keep):
    df = pd.read_excel('/project/projectdirs/openmsi/projects/compound_data/msmls/MSMLS map - mz overlap edit sk.xlsx')
    df.rename(columns={'SMILES': 'smiles'}, inplace=True)
    df.rename(columns={'CNAME': 'common_name'}, inplace=True)
    df.rename(columns={'PC_CID': 'pubchem_compound_id'}, inplace=True)
    df['source_database'] = 'msmls'
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def dequote(s):
    """
    If a string has single or double quotes around it, remove them.
    Make sure the pair of quotes match.
    If a matching pair of quotes is not found, return the string unchanged.
    """
    if (s[0] == s[-1]) and s.startswith(("'", '"')):
        return s[1:-1]
    return s

def get_metacyc(terms_to_keep):
    df = pd.read_excel('/project/projectdirs/openmsi/projects/compound_data/metacyc/MetAtlas_Export_MetaCyc_Compounds.xlsx', encoding='utf-8')
    df.rename(columns={'InChI': 'inchi'}, inplace=True)
    df.rename(columns={'KEGG': 'kegg_id'}, inplace=True)
    df.rename(columns={'PubChem': 'pubchem_compound_id'}, inplace=True)
    df.rename(columns={'Common-Name': 'common_name'}, inplace=True)
    df.rename(columns={'Names': 'synonyms'}, inplace=True) # reduced DCPIP // "2,6-dichloro-4-[(4-hydroxyphenyl)amino]phenol" // "reduced dichloroindophenol" // "reduced 2,6-dichlorophenolindophenol" // "reduced DCIP"
    df.rename(columns={'Object ID': 'metacyc_id'}, inplace=True)
    #df.synonyms.astype(str,inplace=True)
    #df['synonyms'].astype(basestring)
    df.loc[:,'synonyms'] = [[ s.strip() for s in mystr.split('//')] for mystr in df['synonyms'].astype(str).tolist() ]
    df.loc[:,'synonyms'] = [[ dequote(s) for s in mystr] for mystr in df['synonyms'].tolist() ]

    df['source_database'] = 'metacyc'
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

# terms_to_keep = ['smiles','inchi','source_database','ROMol','common_name','synonyms','pubchem_compound_id','lipidmaps_id','metacyc_id','hmdb_id','img_abc_id','chebi_id','kegg_id']
def get_gnps(terms_to_keep):
    from pyteomics import mgf
    gnps = [s['params'] for s in mgf.read('/project/projectdirs/openmsi/projects/compound_data/gnps/ALL_GNPS (1).mgf')]
    df = pd.DataFrame(gnps)
    df['source_database'] = 'gnps'
#     df.rename(columns={'name': 'name'}, inplace=True)
#name has adduct "Hoiamide B M+H"
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def get_dr_dukes():
    df = pd.read_csv('/project/projectdirs/openmsi/projects/compound_data/dr_dukes_phytochemicals/CHEMICALS.csv',delimiter=',')
    df['source_database'] = 'dr_dukes'
    print(list(df.keys()))
    return df

def get_lipid_maps(terms_to_keep):
    df = PandasTools.LoadSDF('/project/projectdirs/openmsi/projects/compound_data/lipidmaps/LMSDFDownload28Jun15FinalAll.sdf')
    df['source_database'] = 'lipidmaps'
    df.rename(columns={'KEGG_ID': 'kegg_id'}, inplace=True)
    df.rename(columns={'PUBCHEM_CID': 'pubchem_compound_id'}, inplace=True)
    df.rename(columns={'COMMON_NAME': 'common_name'}, inplace=True)
    df.rename(columns={'SYNONYMS': 'synonyms'}, inplace=True)
#     Decanohydroxamic acid; caprinohydroxamic acid; n-Decanohydroxamic acid
    df.loc[:,'synonyms'] = [[ s.strip() for s in mystr.split(';')] for mystr in df['synonyms'].astype(str).tolist() ]
    df.rename(columns={'ID': 'lipidmaps_id'}, inplace=True)
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def get_hmdb(terms_to_keep):
    df = PandasTools.LoadSDF('/project/projectdirs/openmsi/projects/compound_data/hmdb/structures.sdf')
    df['source_database'] = 'hmdb'
    df.rename(columns={'GENERIC_NAME': 'common_name'}, inplace=True)
    df.rename(columns={'SYNONYMS': 'synonyms'}, inplace=True)
    df.loc[:,'synonyms'] = [[ s.strip() for s in mystr.split(';')] for mystr in df['synonyms'].astype(str).tolist() ]

#     2-(8S,9S,13S,14S)-3-Hydroxy-2-methoxy-13-methyl-7,8,9,11,12,14,15,16-octahydro-6H-cyclopenta[a]phenanthren-17-one; 2-Hydroxyestrone 2-methyl ether; 2-Methoxy-17-oxoestra-1,3,5(10)-trien-3-ol; 2-Methoxy-3-hydroxyestra-1,3,5(10)-trien-17-one; 3-Hydroxy-2-methoxy-Estra-1,3,5(10)-trien-17-one; 3-Hydroxy-2-methoxyestra-1,3,5(10)-trien-17-one; Methoxy-Estrone
    df.rename(columns={'HMDB_ID': 'hmdb_id'}, inplace=True)
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def get_chembl(terms_to_keep):
    sdf_file = '/project/projectdirs/openmsi/projects/compound_data/chembl/chembl_21.sdf.gz'
    df = PandasTools.LoadSDF(sdf_file)
    df['source_database'] = 'chembl'
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

def get_chebi(terms_to_keep):
    df = PandasTools.LoadSDF('/project/projectdirs/openmsi/projects/compound_data/chebi/ChEBI_complete.sdf.gz')
#     df = PandasTools.LoadSDF('/project/projectdirs/openmsi/projects/compound_data/chebi/ChEBI_complete_3star.sdf.gz')
    for index, row in df.iterrows():
        mol = row['ROMol']
        try:
            df.loc[index,'inchi'] = Chem.MolToInchi(mol)
        except:
            pass
    df['source_database'] = 'chebi'
    df.rename(columns={'KEGG COMPOUND Database Links': 'kegg_id'}, inplace=True)
    df.rename(columns={'ChEBI Name': 'common_name'}, inplace=True)
    df.rename(columns={'Synonyms': 'synonyms'}, inplace=True)
    df.loc[:,'synonyms'] = [[ s.strip() for s in mystr.split('\n')] for mystr in df['synonyms'].astype(str).tolist() ]

    # (-)-Epicatechin\n(-)-Epicatechol\n(2R,3R)-(-)-Epicatechin\n(2R,3R)-2-(3,4-dihydroxyphenyl)-3,4-dihydro-2H-1-benzopyran-3,5,7-triol\n3,3',4',5,7-Pentahydroxyflavane\nEpicatechol\nEpigallocatechin\nL(-)-Epicatechin\nL-Acacatechin\nL-Epicatechin\nL-Epicatechol\nalpha-Catechin
    df.rename(columns={'ChEBI ID': 'chebi_id'}, inplace=True)
    k = list(df.keys())
    for t in terms_to_keep:
        if not t in k:
            df[t] = ''
    return df

""" contribution from Hans de Winter """
def _InitialiseNeutralisationReactions():
    patts= (
        # Imidazoles
        ('[n+;H]','n'),
        # Amines
        ('[N+;!H0]','N'),
        # Carboxylic acids and alcohols
        ('[$([O-]);!$([O-][#7])]','O'),
        # Thiols
        ('[S-;X1]','S'),
        # Sulfonamides
        ('[$([N-;X2]S(=O)=O)]','N'),
        # Enamines
        ('[$([N-;X2][C,N]=C)]','N'),
        # Tetrazoles
        ('[n-]','[nH]'),
        # Sulfoxides
        ('[$([S-]=O)]','S'),
        # Amides
        ('[$([N-]C=O)]','N'),
        )
    return [(Chem.MolFromSmarts(x),Chem.MolFromSmiles(y)) for x,y in patts]

_reactions=None
def NeutraliseCharges(mol, reactions=None):
    global _reactions
    if reactions is None:
        if _reactions is None:
            _reactions=_InitialiseNeutralisationReactions()
        reactions=_reactions
#     mol = Chem.MolFromSmiles(smiles)
    replaced = False
    for i,(reactant, product) in enumerate(reactions):
        while mol.HasSubstructMatch(reactant):
            replaced = True
            # print Chem.MolToSmiles(mol,True)
            # print Chem.MolToSmiles(mol), Chem.MolToSmarts(reactant),Chem.MolToSmiles(product)
            rms = Chem.AllChem.ReplaceSubstructs(mol, reactant, product, replaceAll=True)
            # rms_smiles = Chem.MolToSmiles(rms[0],True)
            # mol = Chem.MolFromSmiles(rms_smiles)
            mol = rms[0]
            # print Chem.MolToSmiles(mol,True)
    if replaced:
        return (mol, True) #Chem.MolToSmiles(mol,True)
    else:
        return (mol, False)

# _reactions=None
# def NeutraliseCharges(smiles, reactions=None):
#     global _reactions
#     if reactions is None:
#         if _reactions is None:
#             _reactions=_InitialiseNeutralisationReactions()
#         reactions=_reactions
#     mol = Chem.MolFromSmiles(smiles)
#     replaced = False
#     for i,(reactant, product) in enumerate(reactions):
#         while mol.HasSubstructMatch(reactant):
#             replaced = True
#             rms = AllChem.ReplaceSubstructs(mol, reactant, product)
#             mol = rms[0]
#     if replaced:
#         return (Chem.MolToSmiles(mol,True), True)
#     else:
#         return (smiles, False)

max_stars_repo_name: biorack/metatlas

max_issues_repo_name: biorack/metatlas

max_forks_repo_name: biorack/metatlas


text:
from agents.informed_vae.main import ImageEncoder, ImageModelSequential

import torch
import torchvision
from tools.file_system_tools import FS
from tools.dataset_tools import Mnist
from tools.pytorch_tools import read_image, to_tensor
from torchvision import transforms
import torch.nn.functional as F
from tools.basics import product

#
#
# optimizer
#
#
import torch
import torch.nn.functional as F
from torch.optim.optimizer import Optimizer, required

class SGD(Optimizer):
    r"""Implements stochastic gradient descent (optionally with momentum).

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float): learning rate
        momentum (float, optional): momentum factor (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        dampening (float, optional): dampening for momentum (default: 0)

    Example:
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> optimizer.zero_grad()
        >>> loss_fn(model(input), target).backward()
        >>> optimizer.step()

    __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf

    .. note::
        The implementation of SGD with Momentum subtly differs from
        Sutskever et. al. and implementations in some other frameworks.

        Considering the specific case of Momentum, the update can be written as

        .. math::
            \begin{aligned}
                v_{t+1} & = \mu * v_{t} + g_{t+1}, \\
                p_{t+1} & = p_{t} - \text{lr} * v_{t+1},
            \end{aligned}

        where :math:`p`, :math:`g`, :math:`v` and :math:`\mu` denote the
        parameters, gradient, velocity, and momentum respectively.

        This is in contrast to Sutskever et. al. and
        other frameworks which employ an update of the form

        .. math::
            \begin{aligned}
                v_{t+1} & = \mu * v_{t} + \text{lr} * g_{t+1}, \\
                p_{t+1} & = p_{t} - v_{t+1}.
            \end{aligned}

    """

    @classmethod
    def sgd(
        cls,
        parameters,
        gradients,
        momentum_buffer_list,
        *,
        weight_decay: float,
        momentum: float,
        lr: float,
        dampening: float,
    ):
        r"""
        Functional API that performs SGD algorithm computation.
        See :class:`~torch.optim.SGD` for details.
        """
        for index, parameters in enumerate(parameters):

            gradient = gradients[index]
            if weight_decay != 0:
                gradient = gradient.add(parameters, alpha=weight_decay)

            if momentum != 0:
                buffer = momentum_buffer_list[index]

                if buffer is None:
                    buffer = torch.clone(gradient).detach()
                    momentum_buffer_list[index] = buffer
                else:
                    buffer.mul_(momentum).add_(gradient, alpha=1 - dampening)

                gradient = buffer

            parameters.add_(gradient, alpha=-lr)

    def __init__(self, model_parameters, lr=required, momentum=0, dampening=0, weight_decay=0):
        if lr is not required and lr < 0.0:
            raise ValueError("Invalid learning rate: {}".format(lr))
        if momentum < 0.0:
            raise ValueError("Invalid momentum value: {}".format(momentum))
        if weight_decay < 0.0:
            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))

        defaults = dict(lr=lr, momentum=momentum, dampening=dampening, weight_decay=weight_decay)
        super(SGD, self).__init__(model_parameters, defaults)

    def __setstate__(self, state):
        super(SGD, self).__setstate__(state)

    @torch.no_grad()
    def step(self, closure=None):
        """Performs a single optimization step.

        Args:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            with torch.enable_grad():
                loss = closure()

        for group in self.param_groups:
            params_with_grad = []
            gradients = []
            momentum_buffer_list = []
            weight_decay = group['weight_decay']
            momentum     = group['momentum']
            dampening    = group['dampening']
            lr           = group['lr']

            for parameter in group['params']:
                if parameter.grad is not None:
                    params_with_grad.append(parameter)
                    gradients.append(parameter.grad)

                    state = self.state[parameter]
                    if 'momentum_buffer' not in state:
                        momentum_buffer_list.append(None)
                    else:
                        momentum_buffer_list.append(state['momentum_buffer'])

            self.sgd(
                params_with_grad,
                gradients,
                momentum_buffer_list,
                weight_decay=weight_decay,
                momentum=momentum,
                lr=lr,
                dampening=dampening,
            )

            # update momentum_buffers in state
            for p, momentum_buffer in zip(params_with_grad, momentum_buffer_list):
                state = self.state[p]
                state['momentum_buffer'] = momentum_buffer

        return loss

#
# network
#
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from tools.basics import temp_folder
from tools.pytorch_tools import autoencoder_ify

class ImageEncoder(ImageModelSequential):
    def __init__(self, **config):
        self.input_shape   = config.get("input_shape", (1, 28, 28))
        self.output_shape  = config.get("output_shape", (10,))
        self.learning_rate = config.get("learning_rate", 0.01)
        self.momentum      = config.get("momentum", 0.5)
        self.log_interval  = config.get("log_interval", 10)

        with self.setup(input_shape=self.input_shape, output_shape=self.output_shape):
            self.add_module("conv1", nn.Conv2d(1, 10, kernel_size=5))
            self.add_module("conv1_pool", nn.MaxPool2d(2))
            self.add_module("conv1_activation", nn.ReLU())

            self.add_module("conv2", nn.Conv2d(10, 20, kernel_size=5))
            self.add_module("conv2_dropout", nn.Dropout2d())
            self.add_module("conv2_pool", nn.MaxPool2d(2))
            self.add_module("conv2_activation", nn.ReLU())

            self.add_module("flatten", nn.Flatten(1)) # 1 => skip the first dimension because thats the batch dimension
            self.add_module("fc1", nn.Linear(self.size_of_last_layer, 50))
            self.add_module("fc1_activation", nn.ReLU())
            self.add_module("fc1_dropout", nn.Dropout2d())

            self.add_module("fc2", nn.Linear(self.size_of_last_layer, product(self.output_shape)))
            self.add_module("fc2_activation", nn.LogSoftmax(dim=-1))

        def NLLLoss(batch_of_actual_outputs, batch_of_ideal_outputs):
            output = batch_of_actual_outputs[range(len(batch_of_ideal_outputs)), batch_of_ideal_outputs]
            return -output.sum()/len(output)

        self.loss_function = NLLLoss
        self.optimizer = SGD(self.parameters(), lr=self.learning_rate, momentum=self.momentum)

    def train_and_test_on_mnist(self):
        from tools.basics import temp_folder
        #
        # training dataset
        #
        train_loader = torch.utils.data.DataLoader(
            autoencoder_ify(torchvision.datasets.MNIST)(
                f"{temp_folder}/files/",
                train=True,
                download=True,
                transform=torchvision.transforms.Compose(
                    [
                        torchvision.transforms.ToTensor(),
                        torchvision.transforms.Normalize((0.1307,), (0.3081,)),
                    ]
                ),
            ),
            batch_size=64,
            shuffle=True,
        )

        #
        # testing dataset
        #
        test_loader = torch.utils.data.DataLoader(
            torchvision.datasets.MNIST(
                f"{temp_folder}/files/",
                train=False,
                download=True,
                transform=torchvision.transforms.Compose(
                    [
                        torchvision.transforms.ToTensor(),
                        torchvision.transforms.Normalize((0.1307,), (0.3081,)),
                    ]
                ),
            ),
            batch_size=1000,
            shuffle=True,
        )

        self.test(test_loader)
        self.fit(loader=train_loader, number_of_epochs=3)
        self.test(test_loader)

        return self

class ImageDecoder(ImageModelSequential):
    def __init__(self, **config):
        self.input_shape   = config.get("input_shape", (10,))
        self.output_shape  = config.get("output_shape", (1, 28, 28))
        self.learning_rate = config.get("learning_rate", 0.01)
        self.momentum      = config.get("momentum", 0.5)
        self.log_interval  = config.get("log_interval", 10)

        with self.setup(input_shape=self.input_shape, output_shape=self.output_shape):
            self.add_module("fn1", nn.Linear(self.size_of_last_layer, 400))
            self.add_module("fn1_activation", nn.ReLU(True))

            self.add_module("fn2", nn.Linear(self.size_of_last_layer, 4000))
            self.add_module("fn2_activation", nn.ReLU(True))

            conv1_shape = [ 10, 20, 20 ] # needs to mupltiply together to be the size of the previous layer (currently 4000)
            conv2_size = 10
            self.add_module("conv1_prep", nn.Unflatten(1, conv1_shape))
            self.add_module("conv1", nn.ConvTranspose2d(conv1_shape[0], conv2_size, kernel_size=5))
            self.add_module("conv2", nn.ConvTranspose2d(conv2_size, 1, kernel_size=5))
            self.add_module("conv2_activation", nn.Sigmoid())

            self.loss_function = nn.MSELoss()
            self.optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate, momentum=self.momentum)

class ImageAutoEncoder(ImageModelSequential):
    def __init__(self, **config):
        self.input_shape   = config.get("input_shape", (1, 28, 28))
        self.latent_shape  = config.get("latent_shape", (10,))
        self.output_shape  = config.get("output_shape", (1, 28, 28))
        self.learning_rate = config.get("learning_rate", 0.01)
        self.momentum      = config.get("momentum", 0.5)
        self.log_interval  = config.get("log_interval", 10)

        with self.setup(input_shape=self.input_shape, output_shape=self.output_shape):
            #
            # encoder
            #
            self.encoder = ImageEncoder(
                input_shape=self.input_shape,
                output_shape=self.latent_shape,
            )
            self.add_module("encoder", self.encoder)
            #
            # decoder
            #
            self.decoder = ImageDecoder(
                input_shape=self.latent_shape,
                output_shape=self.output_shape,
            )
            self.add_module("decoder", self.decoder)

        self.loss_function = nn.MSELoss()
        self.optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate, momentum=self.momentum)

    def update_weights(self, batch_of_inputs, batch_of_ideal_outputs, epoch_index, batch_index):
        self.optimizer.zero_grad()
        batch_of_actual_outputs = self.forward(batch_of_inputs)
        loss = self.loss_function(batch_of_actual_outputs, batch_of_inputs)
        loss.backward()
        self.optimizer.step()
        return loss

    # TODO: test(self) needs to be changed, but its a bit difficult to make it useful

    def train_and_test_on_mnist(self):
        #
        # modify Mnist so that the input and output are both the image
        #
        from tools.basics import temp_folder
        train_loader = torch.utils.data.DataLoader(
            autoencoder_ify(torchvision.datasets.MNIST)(
                f"{temp_folder}/files/",
                train=True,
                download=True,
                transform=torchvision.transforms.Compose(
                    [
                        torchvision.transforms.ToTensor(),
                        torchvision.transforms.Normalize((0.1307,), (0.3081,)),
                    ]
                ),
            ),
            batch_size=64,
            shuffle=True,
        )
        test_loader = torch.utils.data.DataLoader(
            torchvision.datasets.MNIST(
                f"{temp_folder}/files/",
                train=False,
                download=True,
                transform=torchvision.transforms.Compose(
                    [
                        torchvision.transforms.ToTensor(),
                        torchvision.transforms.Normalize((0.1307,), (0.3081,)),
                    ]
                ),
            ),
            batch_size=1000,
            shuffle=True,
        )

        # FIXME: add a testing method (probably related to confusion_matrix) for auto-encoder
        # self.test(test_loader)
        # TODO: autoencoder_ify the train loader inside the fit function
        self.fit(loader=train_loader, number_of_epochs=3)
        # self.test(test_loader)

    def generate_confusion_matrix(self, test_loader):
        from tools.basics import product
        number_of_outputs = product(self.latent_shape)
        confusion_matrix = torch.zeros(number_of_outputs, number_of_outputs)
        test_losses = []
        test_loss = 0
        correct = 0

        self.eval()
        with torch.no_grad():
            for batch_of_inputs, batch_of_ideal_outputs in test_loader:
                latent_space_activation_batch = self.encoder.forward(batch_of_inputs)
                for each_activation_space, each_ideal_output in zip(latent_space_activation_batch, batch_of_ideal_outputs):
                    # which index was chosen
                    predicted_index = numpy.argmax(each_activation_space)
                    actual_index    = numpy.argmax(each_ideal_output)
                    confusion_matrix[actual_index][predicted_index] += 1

        return confusion_matrix

    def importance_identification(self, train_dataset, test_dataset, training_size=10, testing_size=1):
        """
        Outputs:
            importance_values, shap_values

            the shap_values
                is A list
                The length of list is the size of the model's output (one element per output parameter)
                Every element in the list is a numpy array
                Every element has the shape of the model's input
                A value of 0 means no correlation
                negative values means negatively correlated (similar for positive values)

            the importance values
                is a numpy array
                The shape is the same shape as the input for the model
                The values are the relative importance of each latent parameter
                All values are positive
                Larger values = more important
        """
        import shap

        # TODO: improve me, these values are converted to and from numpy values basically as a means of copying to shead information such as gradient tracking info
        # note: the inputs are all encoded because we're trying to explain the latent/encoded space
        latent_spaces_for_training  = to_tensor( torch.from_numpy(b.encoder(train_dataset[index][0]).cpu().detach().numpy()) for index in range(len(train_dataset)) if index < training_size)
        latent_spaces_for_testing   = to_tensor( torch.from_numpy(b.encoder(test_dataset[index][0] ).cpu().detach().numpy()) for index in range(len(test_dataset )) if index < testing_size)

        # DeepExplainer needs the output to be flat for some reason
        # use only the decoder explains the latent space instead of going back to the image
        model = nn.Sequential(
            self.decoder,
            nn.Flatten(),
        )
        explainer = shap.DeepExplainer(model, latent_spaces_for_training)
        shap_values = explainer.shap_values(latent_spaces_for_testing)

        import numpy
        import functools
        # sum these up elementwise
        summed = numpy.squeeze(functools.reduce(
            lambda each_new, existing: numpy.add(each_new, existing),
            # take the absolute value because we just want impactful values, not just neg/pos correlated ones
            numpy.abs(shap_values),
            numpy.zeros_like(shap_values[0]),
        ))
        return summed, shap_values

class ImageClassifier(ImageAutoEncoder):
    def __init__(self, **config):
        self.input_shape   = config.get("input_shape", (1, 28, 28))
        self.latent_shape  = config.get("latent_shape", (20,))
        self.output_shape  = config.get("output_shape", (2,))
        self.learning_rate = config.get("learning_rate", 0.01)
        self.momentum      = config.get("momentum", 0.5)
        self.log_interval  = config.get("log_interval", 10)

        with self.setup(input_shape=self.input_shape, output_shape=self.output_shape):
            #
            # encoder
            #
            self.encoder       = config.get("encoder", ImageEncoder(
                input_shape=self.input_shape,
                output_shape=self.latent_shape,
            ))
            self.add_module("encoder", self.encoder)
            #
            # task (classifier)
            #
            self.task_network = nn.Sequential(
                nn.Linear(product(self.latent_shape), 2), # binary classification
                nn.Sigmoid(),
            )
            self.add_module("task_network", self.task_network)

        self.classifier_loss_function = self.loss_function = nn.BCELoss()
        self.optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate, momentum=self.momentum)

    def update_weights(self, batch_of_inputs, batch_of_ideal_outputs, epoch_index, batch_index):
        # set the gradient values to zero before accumulating them
        self.zero_grad()
        batch_of_latent_vectors = self.encoder.forward(batch_of_inputs)
        batch_of_classified_images = self.task_network.forward(batch_of_latent_vectors)
        task_loss = self.classifier_loss_function(batch_of_classified_images.type(torch.float), batch_of_ideal_outputs.type(torch.float))
        task_loss.backward()

        # call step after both losses have propogated backward
        self.optimizer.step()
        return task_loss

class ImageClassifier2(ImageAutoEncoder):
    def __init__(self, **config):
        self.input_shape         = config.get("input_shape"        , (1 , 28, 28))
        self.latent_shape        = config.get("latent_shape"       , (20, ))
        self.output_shape        = config.get("output_shape"       , (2 , ))
        self.decoded_shape       = config.get("decoded_shape"      , (1 , 28, 28))
        self.learning_rate       = config.get("learning_rate"      , 0.01)
        self.momentum            = config.get("momentum"           , 0.5)
        self.log_interval        = config.get("log_interval"       , 10)
        self.decoding_importance = config.get("decoding_importance", 0)

        with self.setup(input_shape=self.input_shape, output_shape=self.output_shape):
            #
            # encoder
            #
            self.encoder       = config.get("encoder", ImageEncoder(
                input_shape=self.input_shape,
                output_shape=self.latent_shape,
            ))
            self.add_module("encoder", self.encoder)
            #
            # task (classifier)
            #
            self.task_network = nn.Sequential(
                nn.Linear(product(self.latent_shape), 2), # binary classification
                nn.Sigmoid(),
            )
            self.add_module("task_network", self.task_network)
            #
            # task (decoder)
            #
            # self.decoder       = config.get("decoder", ImageDecoder(
            #     input_shape=self.latent_shape,
            #     output_shape=self.decoded_shape,
            # ))

        self.decoder_loss_function = nn.MSELoss()
        self.classifier_loss_function = self.loss_function = nn.BCELoss()
        self.optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate, momentum=self.momentum)

    def update_weights(self, batch_of_inputs, batch_of_ideal_outputs, epoch_index, batch_index):
        # set the gradient values to zero before accumulating them
        self.zero_grad()
        batch_of_latent_vectors = self.encoder.forward(batch_of_inputs)
        # loss #1
        # batch_of_decoded_images = self.decoder.forward(batch_of_latent_vectors)
        # decoder_loss = self.decoder_loss_function(batch_of_decoded_images, batch_of_inputs) * self.decoding_importance
        # decoder_loss.backward(retain_graph=True)
        # loss #2
        # FIXME: figure out how to make this loss relatively more important (maybe a scalar would work?)
        batch_of_classified_images = self.task_network.forward(batch_of_latent_vectors)
        task_loss = self.classifier_loss_function(batch_of_classified_images.type(torch.float), batch_of_ideal_outputs.type(torch.float))
        task_loss.backward()

        # call step after both losses have propogated backward
        self.optimizer.step()
        return task_loss

class SplitAutoEncoder(ImageAutoEncoder):
    def __init__(self, **config):
        self.input_shape   = config.get("input_shape", (1, 28, 28))
        self.latent_shape  = config.get("latent_shape", (20,))
        self.output_shape  = config.get("output_shape", (2,))
        self.decoded_shape = config.get("decoded_shape", (1, 28, 28))
        self.learning_rate = config.get("learning_rate", 0.01)
        self.momentum      = config.get("momentum", 0.5)
        self.log_interval  = config.get("log_interval", 10)
        self.decoding_importance = config.get("decoding_importance", 1)

        with self.setup(input_shape=self.input_shape, output_shape=self.output_shape):
            #
            # encoder
            #
            self.encoder       = config.get("encoder", ImageEncoder(
                input_shape=self.input_shape,
                output_shape=self.latent_shape,
            ))
            self.add_module("encoder", self.encoder)
            #
            # task (classifier)
            #
            self.task_network = nn.Sequential(
                nn.Linear(product(self.latent_shape), 2), # binary classification
                nn.Sigmoid(),
            )
            self.add_module("task_network", self.task_network)
            #
            # task (decoder)
            #
            # self.decoder       = config.get("decoder", ImageDecoder(
            #     input_shape=self.latent_shape,
            #     output_shape=self.decoded_shape,
            # ))

        self.decoder_loss_function = nn.MSELoss()
        self.classifier_loss_function = self.loss_function = nn.BCELoss()
        self.optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate, momentum=self.momentum)

    def update_weights(self, batch_of_inputs, batch_of_ideal_outputs, epoch_index, batch_index):
        # set the gradient values to zero before accumulating them
        self.zero_grad()
        batch_of_latent_vectors = self.encoder.forward(batch_of_inputs)
        # loss #1
        # batch_of_decoded_images = self.decoder.forward(batch_of_latent_vectors)
        # decoder_loss = self.decoder_loss_function(batch_of_decoded_images, batch_of_inputs) * self.decoding_importance
        # decoder_loss.backward(retain_graph=True)
        # loss #2
        # FIXME: figure out how to make this loss relatively more important (maybe a scalar would work?)
        batch_of_classified_images = self.task_network.forward(batch_of_latent_vectors)
        task_loss = self.classifier_loss_function(batch_of_classified_images.type(torch.float), batch_of_ideal_outputs.type(torch.float))
        task_loss.backward()

        # call step after both losses have propogated backward
        self.optimizer.step()
        return task_loss

reason: import numpy,from numpy