#!/usr/bin/env python
import matplotlib
import matplotlib.pyplot as plt
import random
import pandas
import numpy
import seaborn
from PIL import Image
import os

# plotting constants
PERCENTILE_CATEGORIES = [20, 50, 70, 90]
COLOR_CATEGORIES = ["#000033", "#000099", "#0000ff", "#6666ff"]
PERCENTILE_TICKS = [0, 20, 40, 60, 80, 100]
GRID_ALPHA = 0.2
LINE_ALPHA = 0.1
MARKER_ALPHA = 0.6
TITLE_FONT = 16
AXIS_FONT = 14
MAIN_FONT = 12
LABEL_FONT = 10
MAIN_COLOR = 'k'
SECONDARY_COLOR = "r"
MAIN_LINEWIDTH = 1
MARKER_SIZE = 2
BAR_WIDTH = 0.85
HISTOGRAM_BINS = 50
PNG_DPI = 300

def initialize_plot_style():
    """
    Set matplotlib visual style
    Returns: None

    """
    matplotlib.style.use('ggplot')
    plt.rcParams['lines.linewidth'] = MAIN_LINEWIDTH
    plt.rcParams['axes.facecolor'] = 'w'
    plt.rcParams['xtick.color'] = MAIN_COLOR
    plt.rc('xtick', labelsize=LABEL_FONT)
    plt.rc('ytick', labelsize=LABEL_FONT)

def load_mags(filename):
    print(f"Loading data from {filename}")
    data = dict()
    contig_ct = dict()
    with open(filename) as input:
        for line in input:
            cut = line.strip().split("\t")
            contig = cut[1] + ":" + cut[0]
            mag = cut[1]
            data[contig] = mag
            if mag not in contig_ct:
                contig_ct[mag] = 0
            contig_ct[mag] += 1
    final_data = dict()
    for name, mag in data.items():
        new_name = str(contig_ct[mag] / 10000) + ":" + name
        final_data[new_name] = mag
    return final_data

def load_host_links(filename, vmags):
    print(f"Loading data from {filename}")
    data = dict()
    with open(filename) as input:
        for i, line in enumerate(input):
            cut = line.strip().split("\t")
            if i == 0:
                continue
            virus = cut[0]
            for new_name in vmags.keys():
                if virus == new_name.split(":")[-1]:
                    virus = new_name
                    break
            bacteria = cut[5]
            copy_count = float(cut[14])
            if virus not in data:
                data[virus] = dict()
            data[virus][bacteria] = copy_count
    return data

def generate_column_color_labels(contig_mags, df):
    if contig_mags is None:
        col_colors = None
    else:
        bins = dict()
        colors = dict()
        mobile_clusters = dict()
        for contig, mag in contig_mags.items():
            if mag not in mobile_clusters:
                mobile_clusters[mag] = set()
            mobile_clusters[mag].add(contig)
        for cluster, contigs in mobile_clusters.items():
            represented_contigs = 0
            for contig in contigs:
                bins[contig] = cluster
                if contig in df:
                    represented_contigs += 1
            if represented_contigs > 1:
                random_color = (random.random(), random.random(), random.random())
                colors[cluster] = random_color
        col_colors = list()
        boring = True
        for mobile_element in df:
            if mobile_element in bins:
                cluster = bins[mobile_element]
                if cluster in colors:
                    col_colors.append(colors[cluster])
                    boring = False
                else:
                    col_colors.append((1, 1, 1))
            else:
                col_colors.append((1, 1, 1))
        if boring:
            col_colors = None
    return col_colors

def make_heatmap(data_dict, mobile_clusters=None, output_file="heatmap.png", x_label="Mobile elements",
                 y_label="MAG clusters"):
    """
    Make a clustered heatmap plot of mobile element to cluster HiC connectivity data
    Args:

        data_dict (dict[str[dict[str:float]]]): mobile elements pointing to clusters pointing to connectivity values
        output_file (str): path to output png file
        x_label (str): x-axix label string
        y_label (str): y-axix label string
        title (str): heatmap title
    Returns: None

    """
    print(f"Plotting heatmap {output_file}")
    for k in mobile_clusters.keys():
        if k not in data_dict:
            data_dict[k] = dict()

    df = pandas.DataFrame.from_dict(data_dict)
    df = df.fillna(0)
    # remove columns and rows with all 0s
    # df = df.loc[:, (df != 0).any(axis=0)]
    # df = df.loc[(df != 0).any(axis=1)]
    # log normalize
    df += 0.01
    #df = numpy.log(df)
    df = df.reindex(sorted(df.columns, reverse=True), axis=1)

    rows, columns = df.shape
    print(f"Matrix size: {rows} X {columns}")
    xticklabels = False
    yticklabels = False
    col_colors = generate_column_color_labels(mobile_clusters, df)
    if col_colors is not None:
        print(f"Generated {len(col_colors)} randomized color labels for mobile element clusters")
    seaborn.set(font_scale=0.5)
    g = seaborn.clustermap(df, figsize=(6, 6), vmin=0, vmax=1, cmap="Greys_r", col_colors=col_colors,
                           xticklabels=xticklabels, yticklabels=yticklabels, col_cluster=False,
                           dendrogram_ratio=[0.1, 0.1], cbar_pos=[0.02, 0.85, 0.03, 0.1], cbar_kws={"ticks": [0, 1]})

    for ax in [g.ax_row_dendrogram, g.ax_col_dendrogram]:
        ax.clear()
        ax.clear()
        ax.set_facecolor('w')
        ax.set_facecolor('w')
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

    g.ax_col_dendrogram.text(0.5, 0.5, x_label, rotation=0, horizontalalignment="center", fontsize=20)
    g.ax_row_dendrogram.text(0, 0.5, y_label, rotation=90, verticalalignment="center", fontsize=20)

    ax_color = g.ax_cbar
    ax_color.set_ylabel(f"copy count", fontsize=7, labelpad=-1)

    plt.savefig(output_file, bbox_inches='tight', dpi=300)
    plt.close()

def manual_png(figure, x, y, fig, resize=False):
    path = os.path.realpath(figure)
    print(f"Inserting {path}")
    im = Image.open(path)
    if resize:
        plot_h = fig.bbox.ymax * 1
        plot_w = fig.bbox.xmax * 0.8
        w, h = im.size
        resize_ratio = min(plot_h / h, plot_w / w)
        h *= resize_ratio
        im = im.resize((int(w), int(h)), Image.ANTIALIAS)
    im = numpy.array(im).astype(numpy.float) / 255
    plt.figimage(im, x, y)

def filter_binned_only(hosts, vmags):
    print(f"Filtering only binned contigs")
    filtered_hosts = dict()
    for virus, subdata in hosts.items():
        if virus in vmags:
            filtered_hosts[virus] = subdata
    print(f"Kept {len(filtered_hosts)} out of {len(hosts)} viruses")
    return filtered_hosts

def main():
    table_files = ["sheep_master_table.tsv", "human_master_table.tsv", "rumen_master_table.tsv", "wastewater_master_table.tsv"]
    vmag_tables = ["sheep_vmags.tsv", "human_vmags.tsv", "rumen_vmags.tsv", "wastewater_vmags.tsv"]
    names = ["Sheep fecal", "Human fecal", "Cow rumen", "Wastewater"]
    intermediate_plots = ["sheep_master_table.png", "human_master_table.png", "rumen_master_table.png", "wastewater_master_table.png"]

    for i, name in enumerate(names):
        vmag_table = vmag_tables[i]
        host_table = table_files[i]
        intermediate_plot = intermediate_plots[i]

        vmags = load_mags(vmag_table)
        hosts = load_host_links(host_table, vmags)
        hosts = filter_binned_only(hosts, vmags)
        initialize_plot_style()
        make_heatmap(hosts, mobile_clusters=vmags, output_file=intermediate_plot,
                     x_label=f"{name} viral contigs", y_label=f"{name} prokaryotic MAGs")
        plt.close()

    print("Make final merged figure")
    initialize_plot_style()
    fig = plt.figure(figsize=(13, 12))
    ax = fig.add_axes([0, 0, 1, 1])

    manual_png(intermediate_plots[0], 150, 1800, fig)
    manual_png(intermediate_plots[1], 2100, 1800, fig)
    manual_png(intermediate_plots[2], 150, 0, fig)
    manual_png(intermediate_plots[3], 2100, 0, fig)

    ax.text(0.015, 0.96, "A", fontsize=30, zorder=10)
    ax.text(0.515, 0.96, "B", fontsize=30, zorder=10)
    ax.text(0.015, 0.46, "C", fontsize=30, zorder=10)
    ax.text(0.515, 0.46, "D", fontsize=30, zorder=10)

    ax.axis("off")
    plt.savefig("figure.png", dpi=300)

main()


from datetime import datetime, timedelta

from django import template

from hack.models import Commit

register = template.Library()

@register.filter
def commits_over_52(hack):

    current = datetime.now()
    weeks = []
    commits = Commit.objects.filter(hack=hack).values_list('commit_date', flat=True)
    for week in range(52):
        weeks.append(len([x for x in commits if x < current and x > (current - timedelta(7))]))
        current -= timedelta(7)

    weeks.reverse()
    weeks = map(str, weeks)
    return ','.join(weeks)


#!env python3

import gitlab
import subprocess
import os,sys

gl = gitlab.Gitlab('https://clc-gitlab.cs.uiowa.edu:2443')

def clone_group(name,dir):
    """clone the group named name in directory dir"""
    group = gl.groups.get(name)
    projects = group.projects.list(all=True)
    os.makedirs(dir,exist_ok=True)
    n=0
    for project in projects:
        n=n+1
        print(project.name,str(n)+"/"+str(len(projects)))
        if project.name in ["QF_BV_legacy","Sage2_legacy"]:
            print(project.name,"skipped")
            continue
        oldpath=os.path.join(dir,project.path)
        path=os.path.join(dir,project.name)
        if oldpath != path and os.path.exists(oldpath):
            print("rename",oldpath,"to",path)
            os.rename(oldpath,path)
        if os.path.exists(path):
            subprocess.run(["git", "-C", path, "pull", "--depth=1"])
        else:
            subprocess.run(["git", "clone", "--depth=1", project.http_url_to_repo, path])

if len(sys.argv)<2:
    print("download.py <DIR>")
    exit(1)

dst=sys.argv[1]
clone_group("SMT-LIB-benchmarks",os.path.join(dst,"non-incremental"))
clone_group("SMT-LIB-benchmarks-inc",os.path.join(dst,"incremental"))


"""This module contains an end-to-end data workflow (AKA pipeline) of data tasks
from proprocessing, to modelling, and lastly post-processing.
"""

from prefect import Flow
from prefect import Parameter

from .tasks import sanitize_col_names
from .tasks import retrieve_data
from .tasks import clean_data
from .tasks import transform_data
from .tasks import encode_data
from .tasks import wrangle_na
from .tasks import gelman_standardize_data
from .tasks import run_model
from .tasks import plot_confidence_intervals

with Flow(name='e2e-pipeline') as e2e_pipeline:

    # Pipeline parameters
    url = Parameter('url', required=True)
    sep = Parameter('sep', default=',')
    cat_cols = Parameter('cat_cols', default=None)
    na_values = Parameter('na_values', default=None)
    na_strategy = Parameter('na_strategy', default='cc')
    transf_cols = Parameter('transf_cols', default=None)
    transf_func = Parameter('transf_func', default=None)
    endog = Parameter('endog', required=True)
    exog = Parameter('exog', required=True)

    # Sanitize column names
    cat_cols = sanitize_col_names(cat_cols)
    transformed_cols = sanitize_col_names(transf_cols)
    endog = sanitize_col_names(endog)
    exog = sanitize_col_names(exog)

    # Preprocessing
    data = retrieve_data(url, sep)
    cleaned_data = clean_data(data, na_values, cat_cols)
    encoded_data = encode_data(cleaned_data)
    wrangled_data = wrangle_na(encoded_data, na_strategy)
    transformed_data = transform_data(wrangled_data,
                                      transformed_cols,
                                      transf_func)
    standardized_data = gelman_standardize_data(transformed_data)

    # Modelling
    res = run_model(standardized_data, y=endog, X=exog)

    # Postprocessing
    conf_int_chart = plot_confidence_intervals(res)

if __name__ == "__main__":
    pass


import pyodbc
import pandas as pd
import numpy as np
import sklearn
from sklearn.ensemble import GradientBoostingRegressor
import concurrent.futures
from itertools import repeat

def procedure(j,month,threads):
    temptable = "reps.buf.storeforecast" + str(month)

    conn = pyodbc.connect('Driver={SQL Server};'
                          'Server=srv-dwh-lsn,15025;'
                          'Database=dwh;'
                          'Trusted_Connection=yes;')
    query1 = "declare @month as int = " + str(month) + " " \
             "declare @datefrom as date = concat(@month,'01') " \
             "declare @dateto as date = eomonth(@datefrom) " \
             "declare @date2 as date = iif(getdate()-1<@datefrom,getdate()-1,format(dateadd(m,-1,@datefrom),'yyyyMM15')) " \
             "select rs.*,a.store,b.avgsales,sales " \
             "FROM reps.Buf.CalendarForML rs " \
             "join dwh.buf.Push_DailyInfo_PBI a on rs.date=a.date and a.sales>10000 and a.date<=@date2 " \
             "join " + str(temptable) + " b on a.store=b.store and year(a.date)*100 + month(a.date)=mt " \
             "join (select store,ROW_NUMBER() over (order by store)%" + str(threads) + " as num from " + str(temptable) + " group by store) c on b.store=c.store " \
             "where num=" + str(j)

    data = (pd.read_sql(query1, conn)).set_index('Date')

    query2 = "declare @month as int = " + str(month) + " " \
             "declare @datefrom as date = concat(@month,'01') " \
             "declare @dateto as date = eomonth(@datefrom) " \
             "declare @date2 as date = iif(getdate()-1<@datefrom,getdate()-1,format(dateadd(m,-1,@datefrom),'yyyyMM15')) " \
             "declare @date1 as date = dateadd(d,-29,@date2) " \
             "select rs.*,b.store,b.avgsales " \
             "FROM reps.Buf.CalendarForML rs " \
             "join ( " \
             "select store,avg(sales) as avgsales,count(sales) as dayss " \
             "from dwh.buf.Push_DailyInfo_PBI " \
             "where Date between @date1 and @date2 and sales>=10000 " \
             "group by store " \
             ") b on 1=1 " \
             "join (select store,ROW_NUMBER() over (order by store)%" + str(threads) + " as num from " + str(temptable) + " group by store) c on b.store=c.store " \
             "where rs.date between @datefrom and @dateto and dayss>=29 and num=" + str(j)

    target = (pd.read_sql(query2, conn)).set_index('Date')
    stores = target['store'].drop_duplicates()
    result = pd.DataFrame(columns=['Date', 'store', 'plan'])
    for store in stores:
        T = target[target['store'] == store]
        D = data[data['store'] == store]
        X_Train = D.drop(['sales','avgsales', 'store'], 1)
        Y_Train = D['sales'] / D['avgsales']
        X_Test = T.drop(['avgsales', 'store'], 1)
        Y_Test = T[['store', 'avgsales']].reset_index()
        clf = GradientBoostingRegressor(n_estimators=200, learning_rate=0.1,max_depth=5)
        clf.fit(X_Train, Y_Train)
        Y_Test['pred'] = pd.DataFrame(clf.predict(X_Test))
        Y_Test['plan'] = Y_Test['avgsales'] * Y_Test['pred']
        Y_Test = Y_Test.drop(['avgsales', 'pred'], 1)
        result = result.append(other=Y_Test, ignore_index=True, sort=False)
        print(store)
    return result

def main():
    month = 201907
    threads = 7
    temptable = "reps.buf.storeforecast" + str(month)

    conn = pyodbc.connect('Driver={SQL Server};'
                          'Server=srv-dwh-lsn,15025;'
                          'Database=dwh;'
                          'Trusted_Connection=yes;')
    query = "declare @month as int = " + str(month) + " " \
             "declare @datefrom as date = concat(@month,'01') " \
             "declare @dateto as date = eomonth(@datefrom) " \
             "declare @date2 as date = iif(getdate()-1<@datefrom,getdate()-1,format(dateadd(m,-1,@datefrom),'yyyyMM15')) " \
             "declare @date1 as date = dateadd(d,-29,@date2) " \
             "declare @dif as int = datediff(m,@date2,@datefrom)+1 " \
             "declare @day as int = day(@date2) " \
             "select store,mt,avgsales " \
             "into " + str(temptable) + " " \
             "from ( " \
             "select *,count(mt) over (partition by store) as num " \
             "from ( " \
             "select a.store,mt,avg(b.sales) as avgsales,count(b.sales) as qty " \
             "from ( " \
             "select store,year(date)*100 + month(date) as mt,count(*) as qty, " \
             "day(iif(eomonth(date)<=@date2,eomonth(date),@date2)) as maxqty, " \
             "dateadd(d,@day-29,eomonth(date,-@dif)) as begdate, " \
             "dateadd(d,@day,eomonth(date,-@dif)) as enddate " \
             "from dwh.buf.Push_DailyInfo_PBI " \
             "where sales>10000 and date<=@date2 " \
             "group by store,year(date)*100 + month(date),day(iif(eomonth(date)<=@date2,eomonth(date),@date2)),dateadd(d,@day-29,eomonth(date,-@dif)),dateadd(d,@day,eomonth(date,-@dif)) " \
             ") a " \
             "left join dwh.buf.Push_DailyInfo_PBI b on a.store=b.store and b.date between begdate and enddate and b.sales>10000 " \
             "where qty>=maxqty-1 " \
             "group by a.store,mt " \
             ") a " \
             "where qty>=29 " \
             ") a " \
             "where num>=25 "
    cur = conn.cursor()
    cur.execute(query)
    conn.commit()
    cur.close()
    conn.close()

    forecast = pd.DataFrame(columns=['Date', 'store', 'plan'])

    with concurrent.futures.ProcessPoolExecutor() as executor:
        for i in executor.map(procedure, range(0, threads),repeat(month),repeat(threads)):
            forecast = forecast.append(other=i, ignore_index=True, sort=False)

    conn = pyodbc.connect('Driver={SQL Server};'
                          'Server=srv-dwh-lsn,15025;'
                          'Database=dwh;'
                          'Trusted_Connection=yes;')
    query = "declare @month as int = " + str(month) + " " \
            "declare @datefrom as date = concat(@month,'01') " \
            "declare @dateto as date = eomonth(@datefrom) " \
            "declare @date2 as date = iif(getdate()-1<@datefrom,getdate()-1,format(dateadd(m,-1,@datefrom),'yyyyMM15')) " \
            "declare @date1 as date = dateadd(d,-29,@date2) " \
            ";with stores as ( " \
            "select b.store,avgsales,dayss,iif(c.store is null,0,1) as num " \
            "from ( " \
            "select store,avg(sales) as avgsales,count(sales) as dayss " \
            "from dwh.buf.Push_DailyInfo_PBI " \
            "where Date between @date1 and @date2 and sales>=10000 " \
            "group by store " \
            ") b " \
            "left join (select distinct store from " + str(temptable) + ") c on b.store=c.store " \
            ") " \
            "select store,store2,coef " \
            "from ( " \
            "select a.*,ROW_NUMBER() over (partition by store order by delta) as num " \
            "from ( " \
            "select a.store,b.store as store2,sum(power(a.sales-isnull(b.sales,0),2)) as delta,avg(a.sales)/avg(b.sales) as coef " \
            "from ( " \
            "select a.store,date,sales,ROW_NUMBER() over (partition by a.store order by date desc) as N " \
            "from stores a " \
            "join dwh.buf.Push_DailyInfo_PBI b on a.store=b.store and b.sales>=10000 and b.date<=@date2 " \
            "where not (num=1 and dayss>=29) " \
            ") a " \
            "left join dwh.buf.Push_DailyInfo_PBI b on a.Date=b.date " \
            "join (select * from stores where num=1 and dayss>=29) c on c.store=b.store " \
            "where N<=30 " \
            "group by a.store,b.store " \
            ") a " \
            ") b where num<=10 "
    data = (pd.read_sql(query, conn))
    target = pd.merge(data, forecast, how='left', left_on='store2', right_on='store')

    target = target[['Date','store_x', 'coef', 'plan']]
    target['plan'] = target['coef'] * target['plan']
    target['store'] = target['store_x']
    target = target.drop(['coef','store_x'], 1)
    target = target.groupby(['Date', 'store']).mean().reset_index()
    print(forecast)
    print(target)
    forecast = forecast.append(other=target, ignore_index=True, sort=False)
    forecast.to_csv(r"\\SRV-DC02-DCA\Departments-HQ\Finance\УКАиО\Для заливки в хранилище\ForecastML\forecast" + str(month) + ".csv", sep=";")

    query = "drop table " + str(temptable)
    cur = conn.cursor()
    cur.execute(query)
    conn.commit()
    cur.close()
    conn.close()

if __name__ == '__main__':
    main()


from __future__ import division
import os
import numpy as np
from scipy import misc as ms
import sys
import re
import cv2

###################################################

def write_kitti_png(path, flow, valid=None):
    temp = np.ones((flow.shape[0], flow.shape[1], 3), dtype=np.float64)
    temp[:, :, :2] = flow.astype(np.float64) * 64.0 + 2**15
    if valid is not None:
        temp[:, :, 2] = valid
    temp = temp.astype('uint16')
    write_PNG_u16(path, temp)

def write_PNG_u16(path, flow):
    """ Does not check if input flow is multichannel. """
    print(flow.shape)
    ret = cv2.imwrite(path, flow[..., ::-1])
    if not ret:
        print('Flow not written')

def read_flow_flo(filename):
    """ Read flo file and return flow array. """
    f = open(filename, 'rb')
    magic = np.fromfile(f, np.float32, count=1)
    data2d = None

    if magic != 202021.25:
        print('Magic number incorrect. Invalid .flo file.')
    else:
        w = np.fromfile(f, np.int32, count=1)
        h = np.fromfile(f, np.int32, count=1)
        # print("Reading %d x %d flo file" % (h, w))
        # data2d = np.fromfile(f, np.float32, count=2 * w * h)
        # data2d = np.resize(data2d, (h, w, 2))
        # Numpy bullshit adendum
        data2d = np.fromfile(f, np.float32, count=int(2 * w * h))
        # reshape data into 3D array (columns, rows, channels)
        data2d = np.resize(data2d, (int(h), int(w), 2))
    f.close()
    return data2d

def crop_center(img,cropx,cropy):
    y,x,s = img.shape
    startx = x//2-(cropx//2)
    starty = y//2-(cropy//2)
    return img[starty:starty+cropy,startx:startx+cropx]

###################################################

filenames = os.listdir('flow_noc_test')

print('############ ground truth renaming.... ############')
filenames = np.sort(filenames)
print(filenames)

for i,name in enumerate (filenames):
    print(i, name)
    print ('%06d'%i)
    tempImage = ms.imread('flow_noc_test/'+name)
    tempImage = crop_center(tempImage, 1216, 320)
    print(tempImage.shape)
    ms.imsave('GT_test/'+('%06d'%i)+'.png', tempImage)

filenames = os.listdir('predicted')

print('############ predicted conversion.... ############')

filenames = np.sort(filenames)
print(filenames)

for i,name in enumerate (filenames):
    print(i, name)
    data2d = read_flow_flo('predicted/'+name)
    print(data2d.shape)
    name = name.split('.')[0]
    name = name + '.png'
    write_kitti_png('converted/'+name, data2d)


from __future__ import absolute_import

import hashlib
import json
import shutil
from cStringIO import StringIO

from .factoriosave import FactorioSave
from .runner import run_game
from .mod import Mod
from .sprite import pack_sprites
from .locale import apply_locales

FACTORIO_PATH = "/home/des/code/factorios/factorio-0.16.51"

class ProcessingError(Exception):
    pass

def process_save(working_dir):
    sf = FactorioSave("%s/save.zip" % working_dir)

    yield "Loaded save for factorio version %s\n" % sf.version

    mod_map = {}
    for mod_name, mod_version in sorted(sf.mods.items()):
        yield "Detected mod %s_%s\n" % (mod_name, mod_version)

        if mod_name in ('base'):
            continue

        mod = Mod(mod_name, mod_version)
        if not mod.is_downloaded():
            yield "Downloading %s_%s\n" % (mod.name, mod.version)
            mod.download()

        mod_map[mod.name] = mod

    yield "Extracting data. This may take a couple minutes\n"
    data = run_game(FACTORIO_PATH, working_dir, mod_map)

    mod_map['base'] = Mod('base', '0.16.51')
    mod_map['core'] = Mod('core', '0.16.51')

    yield "Applying locale data\n"
    for msg in apply_locales(data, mod_map):
        yield msg

    yield "Packing sprites. This may take a minute.\n"
    sheet = pack_sprites(data, mod_map)

    yield "Saving sprite sheet\n"
    sheetstr = StringIO()
    sheet.save(sheetstr, format='png')
    sheetstr.seek(0)
    h = hashlib.md5()
    for block in iter(lambda: sheetstr.read(4096), b""):
        h.update(block)
    sheet_hash = h.hexdigest()
    sheetstr.seek(0)
    with open("static/calc/images/sprite-sheet-%s.png" % sheet_hash, 'w') as f:
        shutil.copyfileobj(sheetstr, f)

    data['sprites']['hash'] = sheet_hash
    yield 'wrote images/sprite-sheet-%s.png\n' % sheet_hash

    data_str = json.dumps(data, indent=4, sort_keys=True, separators=(',', ': '))

    h = hashlib.md5()
    h.update(data_str)
    data_hash = h.hexdigest()

    with open("static/calc/data/%s.json" % data_hash, 'w') as f:
        f.write(data_str)

    yield "wrote data/%s.json\n" % data_hash
    yield "Your calculator is ready\n"
    yield "/calc/calc-%s.html\n" % data_hash


# -*- coding: utf-8 -*-
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import phd.viz
import phd.thermo
constants = phd.thermo.load_constants()
constants['Oid'] = -17.3
colors = phd.viz.phd_style()

data = pd.read_csv('../data/Garcia2011_Brewster2014_data.csv')

FIT_STRAIN = 1
ALL_DATA = 0

fig, ax = plt.subplots(1, 1, figsize=(4, 3))
ax.set_xscale('log')
ax.set_yscale('log')
ax.set_xlabel('repressors per cell')
ax.set_ylabel('fold-change')

R = np.logspace(0, 4, 500)

op_colors = {'Oid': colors['black'], 'O1':colors['purple'], 'O2':colors['orange'],
            'O3':colors['blue']}
face_colors = {'Oid':colors['light_grey'], 'O1':colors['light_purple'],
               'O2':colors['light_orange'], 'O3':colors['light_blue']}
glyphs = {'garcia':'o', 'brewster': 'd'}

# Theory curves

for o in ['O1', 'O2', 'O3', 'Oid']:
    arch = phd.thermo.SimpleRepression(R=R, ep_r=constants[o],
                                       ka=constants['Ka'], ki=constants['Ki'],
                                       ep_ai=constants['ep_AI'],
                                       effector_conc=0).fold_change()
    ax.plot(R, arch, '-', lw=1, label=o, color=op_colors[o])

ax.plot([], [], 'ko', markerfacecolor='w', label='Garcia & Phillips 2011', ms=2)
ax.plot([], [], 'kD', markerfacecolor='w', label='Brewster et al. 2014', ms=2)
for g, d in data.groupby(['operator', 'author']):
    ax.plot(d['repressor'], d['fold_change'], marker=glyphs[g[1]], markerfacecolor=face_colors[g[0]],
            color=op_colors[g[0]], markeredgewidth=0.75, label='__nolegend__',
            linestyle='none', ms=4)

ax.legend(loc='lower left')
plt.tight_layout()
plt.savefig('../figs/old_gods_titration.pdf', bbox_inches='tight')


import numpy as np
import pylab as p
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
arch=open("potencial.txt",'r')

X=np.zeros((512,512))
Y=np.zeros((512,512))

pot=np.zeros((512,512))

for i in range(512):
	for j in range(512):
		line=arch.readline()
		l=line.split(',')
		X[j][i]=float(l[0])
		Y[j][i]=float(l[1])
		pot[j][i]=float(l[2])
arch.close()

camp=open("campo.txt",'r')
X2=np.zeros((512,512))
Y2=np.zeros((512,512))
Ex=np.zeros((512,512))
Ey=np.zeros((512,512))
for i in range(512):
	for j in range(512):
		lin=camp.readline()
		l=lin.split(',')
		X2[j][i]=float(l[0])
		Y2[j][i]=float(l[1])
		Ex[j][i]=float(l[2])
		Ey[j][i]=float(l[3])

camp.close()
fig=plt.figure()

#ax = fig.add_subplot(211, projection='3d')

#ax.plot_wireframe(X,Y,pot,color='r')
ax=fig.add_subplot(211)
ax.imshow(pot,cmap='hot')

ax2 = fig.add_subplot(212)
t=np.linspace(1.5,3.5,100)
s=np.linspace(2,2,100)
s2=np.linspace(3,3,100)
ax2.streamplot(X2, Y2, Ex, Ey,cmap=plt.cm.inferno, density=2, arrowstyle='->', arrowsize=1.5)
ax2.plot(t,s,color='r',linewidth=5)
ax2.plot(t,s2,color='black',linewidth=5)
plt.savefig('placas.pdf')


# -*- coding: utf-8 -*-
"""
Created on Thu Sep 24 21:13:34 2020
# =============================================================================
# Coin change problem: Maximum number of ways
# =============================================================================
Problem is basically combination of Unbounded Knapsack and cound the no of subset
Recursion+Momonisation
Time complexity- n**2
@author: RAVI
"""

from prettytable import PrettyTable
def CountWay(arr,n,s):
    if l[n][s]!=-1:
        return l[n][s]
    else:
        if n==0 and s!=0:
            l[n][s]=0
            return 0
        elif n!=0 and s==0:
            l[n][s]=1
            return 1
        elif n==0 and s==0:
            l[n][s]=1
            return 1
        elif arr[n-1]<=s:
            l[n][s]=CountWay(arr,n,s-arr[n-1])+CountWay(arr,n-1,s)
            return l[n][s]
        else:
            l[n][s]=CountWay(arr,n-1,s)
            return l[n][s]

n=int(input("Enter No of Coin type:"))
arr=sorted(list(map(int,input("Enter Coin Value:").split())))
s=int(input("Enter sum to get:"))
l=[[-1]*(s+1) for i in range(n+1)]
ans=CountWay(arr,n,s)
print(ans)

x=PrettyTable()
x.field_names=[i for i in range(s+1)]
for i in l:
    x.add_row(i)
print(x)

Coin change problem_Maximum number of ways.py


import datetime

from django.utils.timezone import make_aware
from rest_framework import serializers
from rest_framework.exceptions import ValidationError

from care.facility.api.serializers import TIMESTAMP_FIELDS
from care.facility.models.patient_sample import PatientSample, PatientSampleFlow
from config.serializers import ChoiceField

class PatientSampleFlowSerializer(serializers.ModelSerializer):
    status = ChoiceField(choices=PatientSample.SAMPLE_TEST_FLOW_CHOICES, required=False)

    class Meta:
        model = PatientSampleFlow
        fields = "__all__"

class PatientSampleSerializer(serializers.ModelSerializer):
    status = serializers.CharField(read_only=True)
    result = serializers.CharField(read_only=True)
    notes = serializers.CharField(required=False)
    date_of_sample = serializers.DateTimeField(read_only=True)
    date_of_result = serializers.DateTimeField(read_only=True)
    patient_id = serializers.IntegerField(required=False)
    consultation_id = serializers.IntegerField(required=False)

    class Meta:
        model = PatientSample
        exclude = TIMESTAMP_FIELDS + ("patient", "consultation")

    def create(self, validated_data):
        return super(PatientSampleSerializer, self).create(validated_data)

class PatientSamplePatchSerializer(PatientSampleSerializer):
    status = ChoiceField(choices=PatientSample.SAMPLE_TEST_FLOW_CHOICES)
    result = ChoiceField(choices=PatientSample.SAMPLE_TEST_RESULT_CHOICES, required=False)

    def update(self, instance, validated_data):
        try:
            choice = PatientSample.SAMPLE_TEST_FLOW_CHOICES[validated_data["status"] - 1][1]
        except KeyError:
            raise ValidationError({"status": ["is required"]})
        valid_choices = PatientSample.SAMPLE_FLOW_RULES[PatientSample.SAMPLE_TEST_FLOW_CHOICES[instance.status - 1][1]]
        if choice not in valid_choices:
            raise ValidationError({"status": [f"Next valid choices are: {', '.join(valid_choices)}"]})
        if choice != "COMPLETED" and validated_data.get("result"):
            raise ValidationError({"result": [f"Result can't be updated unless test is complete"]})
        if choice == "COMPLETED" and not validated_data.get("result"):
            raise ValidationError({"result": [f"is required as the test is complete"]})

        if validated_data.get("status") == PatientSample.SAMPLE_TEST_FLOW_MAP["SENT_TO_COLLECTON_CENTRE"]:
            validated_data["date_of_sample"] = make_aware(datetime.datetime.now())
        elif validated_data.get("status") == PatientSample.SAMPLE_TEST_FLOW_MAP["DENIED"]:
            validated_data["result"] = PatientSample.SAMPLE_TEST_RESULT_MAP["INVALID"]
        elif validated_data.get("status") == PatientSample.SAMPLE_TEST_FLOW_MAP["REQUEST_SUBMITTED"]:
            validated_data["result"] = PatientSample.SAMPLE_TEST_RESULT_MAP["AWAITING"]
        elif validated_data.get("result") is not None:
            validated_data["date_of_result"] = make_aware(datetime.datetime.now())

        return super().update(instance, validated_data)

class PatientSampleReadSerializer(PatientSamplePatchSerializer):
    flow = serializers.ListSerializer(child=PatientSampleFlowSerializer())
    patient_name = serializers.CharField(source="patient.name")


import torch
import torch.nn as nn
from .ResNet import ResNet, BasicBlock, Bottleneck

__all__ = ['scan_resnet18']

class SCAN(nn.Module):
    def __init__(self, channels, b_stride=2, final_channels=512, num_classes=100, selection=False):
        super(SCAN, self).__init__()
        self.attention = nn.Sequential(
                    nn.Conv2d(channels, channels, kernel_size=3, padding=1, stride=2),
                    nn.BatchNorm2d(channels),
                    nn.ReLU(),
                    nn.ConvTranspose2d(channels, channels, kernel_size=4, padding=1, stride=2),
                    nn.BatchNorm2d(channels),
                    nn.Sigmoid()
                )
        self.bottleneck = nn.Sequential(
            nn.Conv2d(channels, 128, kernel_size=1, stride=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=b_stride, stride=b_stride),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.Conv2d(128, final_channels, kernel_size=1, stride=1),
            nn.BatchNorm2d(final_channels),
            nn.ReLU(),
            nn.AvgPool2d(4,4)
        )
        self.shallow_classifier = nn.Linear(final_channels, num_classes)

    def forward(self, x):
        x = self.attention(x)
        features = self.bottleneck(x)
        features = features.view(x.size(0), -1)
        x = self.shallow_classifier(features)

        return x, features

class SCAN_ResNet(ResNet):
    def __init__(self, block, layers, num_classes=100):
        super(SCAN_ResNet, self).__init__(block, layers, num_classes)

        self.scan1 = SCAN(
            channels=64,
            final_channels=512,
            b_stride=8
        )
        self.scan2 = SCAN(
            channels=128,
            final_channels=512,
            b_stride=4
        )
        self.scan3 = SCAN(
            channels=256,
            final_channels=512,
            b_stride=2
        )
        self.avgpool = nn.AvgPool2d(4, 4)

    def forward(self, x):
        feature_list = []
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.layer1(x)

        feature1 = self.scan1(x)

        x = self.layer2(x)

        fea2 = self.scan2(x)
        fea2 = fea2 * x
        feature_list.append(fea2)

        x = self.layer3(x)

        fea3 = self.scan3(x)
        fea3 = fea3 * x
        feature_list.append(fea3)

        x = self.layer4(x)
        x = self.avgpool(x)
        feature_list.append(x)

        feature1 = self.scala1(feature_list[0]).view(x.size(0), -1)
        feature2 = self.scala2(feature_list[1]).view(x.size(0), -1)
        feature3 = self.scala3(feature_list[2]).view(x.size(0), -1)
        feature4 = self.scala4(feature_list[3]).view(x.size(0), -1)

        exit1 = self.fc1(feature1)
        exit2 = self.fc2(feature2)
        exit3 = self.fc3(feature3)
        exit4 = self.fc(feature4)

        return [exit1, exit2, exit3, exit4], [feature1, feature2, feature3, feature4]

def scan_resnet18(pretrained=False, **kwargs):
    model = SCAN_ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)

    return model


'''
Tool Name: Find Closest Facilities
Source Name: FindClosestFacilities.py
Version: ArcGIS 10.2.1
Author: ESRI
This script finds closest facilities from one or more incidents based on a specified network cost.
It is intended to be used in a geoprocessing service.
'''

#Import the modules
import arcpy
import os
import sys
import traceback
import time
import fnmatch
#from xml.etree import ElementTree as ET
import uuid
import NAUtils as nau

#set some constants
DEBUG = False
NA_LAYER = "FindClosestFacilities_ClosestFacility"
#POLYGONS_SUBLAYER = NA_LAYER + os.sep + "Polygons"
ROUND_PRECISION = 5
TIME_UNITS = ('minutes','hours','days', 'seconds')
INFINITY = sys.maxint
ID_FIELD_NAME = "ID"
#Field names from input features sets. These need to be updated if we change schema for inputs
FACILITIES_FIELDS = (u'Name', u'ID', u'AdditionalTime', u'AdditionalDistance', u'CurbApproach')
INCIDENTS_FIELDS = FACILITIES_FIELDS
POINT_BARRIER_FIELDS = (u'Name', u'BarrierType', u'Additional_Time', u'Additional_Distance')
LINE_BARRIER_FIELDS = (u'Name')
POLYGON_BARRIER_FIELDS = (u'Name', u'BarrierType', u'ScaledTimeFactor', u'ScaledDistanceFactor')
ATTRIBUTE_PARAMETER_FIELDS = (u'AttributeName', u'ParameterName', u'ParameterValue')
TOOL_NAME = "FindClosestFacilities_na"

#functions
def max_distance_between_points(point_feature_sets):
    '''Return the maximum straight line distance in meters between the point features. point_feature_sets is
    the two value list of feature sets containing input points. All feature classes are assumed to be in same spatial
    reference. The spatial reference of first feature class is used for calculations. The maximum distance is
    calculated as the maximum of the width or height for the combined extent of all the point feature classes.
    In case of an exception, return sys.maxint as we want to use hierarchy if we cannot figure out how to enforce hierarchy. '''
    max_distance = INFINITY
    try:
        #convert the feature sets into feature classes so that we get all the properties like extent and spatial ref
        point_feature_classes = []
        point_array = arcpy.Array()
        for i,fs in enumerate(point_feature_sets):
            point_fc = "in_memory{0}pfc{1}".format(os.sep,i)
            fs.save(point_fc)
            point_feature_classes.append(point_fc)

        #Convert each feature class extents to polygons
        extent_polygons = []
        extent_lower_left_points = []
        for fc in point_feature_classes:
            desc_fc = arcpy.Describe(fc)
            fc_extent = desc_fc.extent
            extent_lower_left_points.append(fc_extent.lowerLeft)
            point_array.add(fc_extent.lowerLeft)
            point_array.add(fc_extent.lowerRight)
            point_array.add(fc_extent.upperRight)
            point_array.add(fc_extent.upperLeft)
            point_array.add(fc_extent.lowerLeft)
            extent_polygons.append(arcpy.Polygon(point_array, desc_fc.spatialReference))
            point_array.removeAll()
        #union the two polygons to get the merged extent
        combined_extent_polygon = extent_polygons[0].union(extent_polygons[1])

        #Get the width and height of combined extent
        #Get the diagonal line from the combined extent
        combined_extent = combined_extent_polygon.extent
        #For feature classes with single point, the combined extent after union comes out to be null
        #In such cases just construct a line using the lower left extent point for feature classes.
        point_array.removeAll()
        if combined_extent.width > 0:
            #point_array.add(combined_extent.lowerLeft)
            #point_array.add(combined_extent.upperRight)
            #combined_extent_diagonal = arcpy.Polyline(point_array,combined_extent.spatialReference)
            point_array.add(combined_extent.lowerLeft)
            point_array.add(combined_extent.lowerRight)
            combined_extent_width = arcpy.Polyline(point_array,combined_extent.spatialReference).getLength("GEODESIC")
            point_array.removeAll()
            point_array.add(combined_extent.lowerLeft)
            point_array.add(combined_extent.upperLeft)
            combined_extent_height = arcpy.Polyline(point_array,combined_extent.spatialReference).getLength("GEODESIC")
            point_array.removeAll()
            max_distance =  max((combined_extent_height, combined_extent_width))
        else:
            point_array.add(extent_lower_left_points[0])
            point_array.add(extent_lower_left_points[1])
            max_distance = arcpy.Polyline(point_array,combined_extent.spatialReference).getLength("GEODESIC")
            point_array.removeAll()
    except Exception as ex:
        max_distance = INFINITY

    return max_distance

#ParameterName to Parameter Index mapping. If the parameter index changes, make sure that this mapping
#is upto date.

parameter_names = ('incidents', 'facilities', 'measurement_units','network_dataset',
                   'out_gdb_workspace','out_routes_name', 'out_directions_name',
                   'out_facilities_name', 'num_facilities_to_find', 'default_cutoff', 'travel_direction',
                   'time_of_day', 'time_of_day_usage', 'time_zone_usage','uturn_policy',
                   'point_barriers', 'line_barriers', 'polygon_barriers', 'time_attribute',
                   'time_attribute_units', 'distance_attribute', 'distance_attribute_units',
                   'use_hierarchy', 'restrictions', 'attribute_parameters', 'accumulate_attributes',
                   'max_snap_tolerance', 'feature_locator_where_clause', 'route_shape',
                   'route_line_simplification_tolerance', 'populate_directions',
                   'directions_language', 'directions_distance_units','directions_style_name',
                   'max_features_point_barriers', 'max_features_line_barriers',
                   'max_features_polygon_barriers', 'max_facilities', 'max_facilities_to_find',
                   'max_incidents', 'force_hierarchy_beyond_distance', 'save_output_layer',
                   'solve_succeeded', 'out_routes', 'out_directions', 'out_facilities')

parameter_index = {value:index for index,value in enumerate(parameter_names)}
parameter_info = arcpy.GetParameterInfo(TOOL_NAME)

#Get all the input parameter values
facilities = arcpy.GetParameter(parameter_index['facilities'])
incidents = arcpy.GetParameter(parameter_index['incidents'])
network_dataset = arcpy.GetParameterAsText(parameter_index['network_dataset'])
measurement_units = arcpy.GetParameterAsText(parameter_index['measurement_units'])
out_gdb_workspace = arcpy.GetParameterAsText(parameter_index['out_gdb_workspace'])
out_routes_name = arcpy.GetParameterAsText(parameter_index['out_routes_name'])
out_directions_name = arcpy.GetParameterAsText(parameter_index['out_directions_name'])
out_facilities_name = arcpy.GetParameterAsText(parameter_index['out_facilities_name'])
num_facilities_to_find = arcpy.GetParameter(parameter_index['num_facilities_to_find'])
str_default_cutoff = arcpy.GetParameterAsText(parameter_index['default_cutoff'])
default_cutoff = float(str_default_cutoff) if str_default_cutoff else None
travel_direction = arcpy.GetParameterAsText(parameter_index['travel_direction'])
time_of_day = arcpy.GetParameter(parameter_index['time_of_day'])
time_of_day_usage = arcpy.GetParameterAsText(parameter_index['time_of_day_usage'])
time_zone_usage = arcpy.GetParameterAsText(parameter_index['time_zone_usage'])
uturn_policy = arcpy.GetParameterAsText(parameter_index['uturn_policy'])
point_barriers = arcpy.GetParameter(parameter_index['point_barriers'])
line_barriers = arcpy.GetParameter(parameter_index['line_barriers'])
polygon_barriers = arcpy.GetParameter(parameter_index['polygon_barriers'])
time_attribute = arcpy.GetParameterAsText(parameter_index['time_attribute'])
time_attribute_units = arcpy.GetParameterAsText(parameter_index['time_attribute_units'])
distance_attribute = arcpy.GetParameterAsText(parameter_index['distance_attribute'])
distance_attribute_units = arcpy.GetParameterAsText(parameter_index['distance_attribute_units'])
use_hierarchy = arcpy.GetParameter(parameter_index['use_hierarchy'])
restrictions = arcpy.GetParameterAsText(parameter_index['restrictions'])
attribute_parameters = arcpy.GetParameter(parameter_index['attribute_parameters'])
accumulate_attributes = arcpy.GetParameterAsText(parameter_index['accumulate_attributes'])
max_snap_tolerance = arcpy.GetParameterAsText(parameter_index['max_snap_tolerance'])
feature_locator_where_clause = arcpy.GetParameterAsText(parameter_index['feature_locator_where_clause'])
route_shape = arcpy.GetParameterAsText(parameter_index['route_shape'])
route_line_simplification_tolerance = arcpy.GetParameterAsText(parameter_index['route_line_simplification_tolerance'])
populate_directions = arcpy.GetParameter(parameter_index['populate_directions'])
directions_language = arcpy.GetParameterAsText(parameter_index['directions_language'])
directions_distance_units = arcpy.GetParameterAsText(parameter_index['directions_distance_units'])
directions_style_name = arcpy.GetParameterAsText(parameter_index['directions_style_name'])
str_max_features_point_barriers = arcpy.GetParameterAsText(parameter_index['max_features_point_barriers'])
str_max_features_line_barriers = arcpy.GetParameterAsText(parameter_index['max_features_line_barriers'])
str_max_features_polygon_barriers = arcpy.GetParameterAsText(parameter_index['max_features_polygon_barriers'])
str_max_facilities = arcpy.GetParameterAsText(parameter_index['max_facilities'])
str_max_facilities_to_find = arcpy.GetParameterAsText(parameter_index['max_facilities_to_find'])
str_max_incidents = arcpy.GetParameterAsText(parameter_index['max_incidents'])
str_force_hierarchy_beyond_distance = arcpy.GetParameterAsText(parameter_index['force_hierarchy_beyond_distance'])
save_output_layer = arcpy.GetParameter(parameter_index['save_output_layer'])

#Derived outputs values. These are set to parameter values in finally block.
out_routes_fc = os.path.join(out_gdb_workspace, out_routes_name)
out_directions_fc = os.path.join(out_gdb_workspace, out_directions_name)
out_facilities_fc = os.path.join(out_gdb_workspace, out_facilities_name)
solve_succeeded = False
cf_layer_exists = False

#These are deleted in finally block if datasets referenced by objects exists
input_copies = []

try:
    #Check out network analyst extension
    arcpy.CheckOutExtension("network")

    measurement_method = "TRAVEL_TIME" if measurement_units.lower() in TIME_UNITS else "TRAVEL_DISTANCE"
    #Convert constraint values from strings to number. If empty string use max Int
    max_features_point_barriers = int(str_max_features_point_barriers) if str_max_features_point_barriers else INFINITY
    max_features_line_barriers = int(str_max_features_line_barriers) if str_max_features_line_barriers else INFINITY
    max_features_polygon_barriers = int(str_max_features_polygon_barriers) if str_max_features_polygon_barriers else INFINITY
    max_facilities = int(str_max_facilities) if str_max_facilities else INFINITY
    max_facilities_to_find = int(str_max_facilities_to_find) if str_max_facilities_to_find else INFINITY
    max_incidents = int(str_max_incidents) if str_max_incidents else INFINITY
    force_hierarchy_beyond_distance = float(str_force_hierarchy_beyond_distance) if str_force_hierarchy_beyond_distance else INFINITY

    #Check if the output feature class names are valid. Fail with first invalid name
    nau.check_valid_table_name(out_routes_name, out_gdb_workspace, 30101,
                               parameter_info[parameter_index['out_routes_name']].displayName)
    nau.check_valid_table_name(out_directions_name, out_gdb_workspace, 30101,
                               parameter_info[parameter_index['out_directions_name']].displayName)
    nau.check_valid_table_name(out_facilities_name, out_gdb_workspace, 30101,
                               parameter_info[parameter_index['out_facilities_name']].displayName)

    desc_nds = arcpy.Describe(network_dataset)
    desc_nds_attributes = desc_nds.attributes

    #Convert all input features to feature sets or recordsets if they are not
    #This is required as if input is passed a feature layer or feature class
    #We will end up directly modifying the inputs

    facilities_obj = nau.InputFeatureClass(facilities)
    #Store the OBJECTID field for facilities as it will used later when exporting output facilities
    orig_input_facilities_oid = facilities_obj.origOIDFieldName
    #Store all the fields names from input facilities to be used later when exporting output facilities
    orig_input_facilities_fld_names = facilities_obj.fieldNames
    incidents_obj = nau.InputFeatureClass(incidents)
    point_barriers_obj = nau.InputFeatureClass(point_barriers)
    line_barriers_obj = nau.InputFeatureClass(line_barriers)
    polygon_barriers_obj = nau.InputFeatureClass(polygon_barriers)
    attribute_parameters_obj = nau.InputTable(attribute_parameters)
    #Keep a list of input copies so we can delete them just before exit
    input_copies = (facilities_obj, incidents_obj, point_barriers_obj, line_barriers_obj,
                    polygon_barriers_obj, attribute_parameters_obj)

    #If the network dataset does not support hierarchy, set the useHierarchy parameter to false.
    nds_has_hierarchy = nau.nds_supports_hierarchy(desc_nds_attributes)
    if not nds_has_hierarchy:
        use_hierarchy = False

    #determine whether we should use time based or distance based impedance attribute based on measurement method
    impedance_attribute = time_attribute if measurement_method == "TRAVEL_TIME" else distance_attribute
    impedance_units = nau.verify_impedance_units(time_attribute, time_attribute_units, distance_attribute,
                                                 distance_attribute_units, desc_nds_attributes, False)[impedance_attribute]

    #If the Cutoff is specified, convert the cutoff value from user specified unit to impedance unit
    if default_cutoff:
        converted_cutoff = nau.convert_units(default_cutoff, measurement_units, impedance_units)
    else:
        converted_cutoff = default_cutoff

    #Get counts for facilities, incidents, barrier features and attribute parameters
    facility_count = facilities_obj.count
    incident_count = incidents_obj.count

    #Convert inputs from record sets to feature classes if they are not empty.
    if facility_count:
        facilities_obj.copyFeatures(out_gdb_workspace, FACILITIES_FIELDS)
        facilities = facilities_obj.catalogPath

    if incident_count:
        incidents_obj.copyFeatures(out_gdb_workspace, INCIDENTS_FIELDS)
        incidents = incidents_obj.catalogPath

    if point_barriers_obj.count:
        point_barriers_obj.copyFeatures(out_gdb_workspace, POINT_BARRIER_FIELDS)
        point_barriers = point_barriers_obj.catalogPath

    if line_barriers_obj.count:
        line_barriers_obj.copyFeatures(out_gdb_workspace, LINE_BARRIER_FIELDS)
        line_barriers = line_barriers_obj.catalogPath

    if polygon_barriers_obj.count:
        polygon_barriers_obj.copyFeatures(out_gdb_workspace, POLYGON_BARRIER_FIELDS)
        polygon_barriers = polygon_barriers_obj.catalogPath

    if attribute_parameters_obj.count:
        attribute_parameters_obj.copyFeatures(out_gdb_workspace, ATTRIBUTE_PARAMETER_FIELDS)
        attribute_parameters = attribute_parameters_obj.catalogPath

    ##Determine if the throttling conditions are met. If not raise an exception and quit
    ##If throttling parameters have zero value, then do not perform throttling checks.
    # Throttling Check 1: Check for number of facilities
    if facility_count == 0:
        arcpy.AddIDMessage("ERROR",30125)
        raise nau.InputError()
    if str_max_facilities and facility_count > max_facilities:
        arcpy.AddIDMessage("ERROR", 30096,"Facilities", max_facilities)
        raise nau.InputError()
    # Throttling Check 2: Check for number of incidents
    if incident_count == 0:
        arcpy.AddIDMessage("ERROR",30125)
        raise nau.InputError()
    if str_max_incidents and incident_count > max_incidents:
        arcpy.AddIDMessage("ERROR", 30096,"Incidents", max_incidents)
        raise nau.InputError()

    #Throttling Check 3: Check for number of facilities to find
    if str_max_facilities_to_find and num_facilities_to_find > max_facilities_to_find:
        arcpy.AddIDMessage("ERROR",30126, num_facilities_to_find, max_facilities_to_find)
        raise nau.InputError()

    #Throttling Check 4: Check if hierarchy needs to be forced
    if str_force_hierarchy_beyond_distance and use_hierarchy == False:
        #force to use hierarchy. If the NDS does not support hierarchy raise an error and quit.
        if nds_has_hierarchy:
            max_distance_between_locations = max_distance_between_points((facilities, incidents))
            converted_max_distance = float(nau.convert_units(max_distance_between_locations, "Meters", distance_attribute_units))
            if converted_max_distance > force_hierarchy_beyond_distance:
                use_hierarchy = True
                #Report the exceeded value in measurement units if using TRAVEL_DISTANCE. Otherwise report the
                #exceeded value in distance attribute units which are input units for force hierarchy constraint
                if measurement_method == "TRAVEL_DISTANCE":
                    converted_force_hierarchy_beyond_distance = nau.convert_units(force_hierarchy_beyond_distance,
                                                                                distance_attribute_units,
                                                                                measurement_units)
                    report_value_with_units = "{0} {1}".format(converted_force_hierarchy_beyond_distance, measurement_units)
                else:
                    report_value_with_units = "{0} {1}".format(force_hierarchy_beyond_distance, distance_attribute_units)
                arcpy.AddIDMessage("WARNING", 30109)
                arcpy.AddIDMessage("WARNING", 30127, report_value_with_units)
        else:
            arcpy.AddIDMessage("ERROR", 30119, "Force Hierarchy beyond Distance")
            raise nau.InputError()

    #Throttling Check 5: Check if the number of barrier features (point, line and polygon) are within maximum allowed
    load_point_barriers, load_line_barriers, load_polygon_barriers = nau.check_barriers(point_barriers_obj, line_barriers_obj,
                                                                                polygon_barriers_obj, max_features_point_barriers,
                                                                                max_features_line_barriers,
                                                                                max_features_polygon_barriers, desc_nds)
    ##Perform the closest facility analysis as all throttling conditions are met.
    #Get the restrictions and accumulate attributes that are valid for the network dataset
    if restrictions:
        restrictions_to_use = nau.get_valid_attributes(desc_nds_attributes, restrictions)
    else:
        restrictions_to_use = []

    #Determine the accumulation attributes to use
    #Get only the attributes that are valid for the current network dataset.
    if accumulate_attributes:
        accumulate_attributes = nau.get_valid_attributes(desc_nds_attributes, accumulate_attributes,
                                                                "Cost", 30128)
        #remove time attribute and distance attribute from the list of accumulate attributes as we will manage
        #these attributes seperately.
        if time_attribute in accumulate_attributes:
            accumulate_attributes.remove(time_attribute)
        if distance_attribute in accumulate_attributes:
            accumulate_attributes.remove(distance_attribute)

    else:
        accumulate_attributes = []
    #Always accumate time or distance attribute based on measurement method.
    if measurement_method == "TRAVEL_TIME":
        system_accumulate_attribute = [distance_attribute]
    else:
        system_accumulate_attribute = [time_attribute]
    accumulate_attributes_to_use = accumulate_attributes + system_accumulate_attribute

    #Make a new closest facility layer
    cf_layer = arcpy.na.MakeClosestFacilityLayer(network_dataset, NA_LAYER, impedance_attribute, travel_direction,
                                                 converted_cutoff, num_facilities_to_find, accumulate_attributes_to_use,
                                                 uturn_policy, restrictions_to_use,use_hierarchy,None, route_shape,
                                                 time_of_day, time_of_day_usage).getOutput(0)
    cf_layer_exists = True
    na_class_names = arcpy.na.GetNAClassNames(cf_layer)
    solver_props = arcpy.na.GetSolverProperties(cf_layer)

    #Set time zone usage if time of day is specified
    if time_of_day:
        solver_props.timeZoneUsage = time_zone_usage

    #Add attribute parameters if specified
    nau.update_attribute_parameters(cf_layer, attribute_parameters,
                                    restrictions_to_use + accumulate_attributes_to_use + [impedance_attribute], desc_nds_attributes)
    #Add Barriers before loading facilities and incidents as we want to exclude restricted portions
    if load_point_barriers:
        #point_barrier_fields = arcpy.ListFields(point_barriers)
        nau.add_locations(cf_layer, "Barriers", point_barriers_obj, impedance_attribute, impedance_units,
                          measurement_units, max_snap_tolerance, feature_locator_where_clause, measurement_method)
    if load_line_barriers:
        #line_barrier_fields = arcpy.ListFields(line_barriers)
        nau.add_locations(cf_layer, "PolylineBarriers", line_barriers_obj, impedance_attribute, impedance_units,
                          measurement_units, max_snap_tolerance, feature_locator_where_clause, measurement_method)
    if load_polygon_barriers:
        #polygon_barrier_fields = arcpy.ListFields(polygon_barriers)
        nau.add_locations(cf_layer, "PolygonBarriers", polygon_barriers_obj, impedance_attribute,
                          impedance_units, measurement_units, max_snap_tolerance, feature_locator_where_clause,
                          measurement_method)

    #Add facilities
    #facility_fields = arcpy.ListFields(facilities)
    facility_fields = facilities_obj.describeObject.fields
    facility_id_field = nau.find_field(facilities, facility_fields, ID_FIELD_NAME)
    propogate_facility_ids = True if facility_id_field else False
    nau.add_locations(cf_layer, "Facilities", facilities_obj, impedance_attribute, impedance_units,
                      measurement_units, max_snap_tolerance, feature_locator_where_clause, measurement_method,
                      propogate_facility_ids, facility_id_field)

    #Add Incidents
    #incident_fields = arcpy.ListFields(incidents)
    incident_fields = incidents_obj.describeObject.fields
    incident_id_field = nau.find_field(incidents, incident_fields, ID_FIELD_NAME)
    propogate_incident_ids = True if incident_id_field else False
    nau.add_locations(cf_layer, "Incidents", incidents_obj, impedance_attribute, impedance_units,
                      measurement_units, max_snap_tolerance, feature_locator_where_clause, measurement_method,
                      propogate_incident_ids, incident_id_field)

    #Solve
    solve_result = arcpy.na.Solve(cf_layer,"SKIP","TERMINATE", route_line_simplification_tolerance)
    cf_sub_layers = {k.datasetName:k for k in arcpy.mapping.ListLayers(cf_layer)[1:]}
    routes_sub_layer = cf_sub_layers["CFRoutes"]
    routes_sub_layer_name = na_class_names["CFRoutes"]
    if solve_result.getOutput(1).lower() == 'true':
        solve_succeeded = True
    else:
        solve_succeeded = False
    if solve_result.maxSeverity == 1:
        nau.print_message(solve_result.getMessages(1), 1)

    #Get a list of cost attributes and their units to check if we need to calculate new fields when reporting
    #accumulated attribute values.
    cost_attributes = {}
    for attr in desc_nds_attributes:
        if attr.usageType == "Cost":
            cost_attributes[attr.name] = attr.units
    #Get a list of Total_ fields from the routes sub layer
    routes_sub_layer_field_names = [f.name for f in arcpy.Describe(routes_sub_layer).fields]
    cost_field_names = fnmatch.filter(routes_sub_layer_field_names, "Total_*")

    initial_facility_id_field_name = "Facilities" + ID_FIELD_NAME
    initial_incident_id_field_name = "Incidents" + ID_FIELD_NAME

    #Transfer the FacdilityOID and IncidentOID fields from incidents and facilities to routes
    #Propogate incident or facilities ID to routes layer if required.
    facility_join_fields = ["FacilityOID"]
    if propogate_facility_ids:
        facility_join_fields.append(initial_facility_id_field_name)
    arcpy.management.JoinField(routes_sub_layer, "FacilityID", cf_sub_layers["Facilities"], "ObjectID",
                               facility_join_fields)
    incident_join_fields = ["IncidentOID"]
    if propogate_incident_ids:
        incident_join_fields.append(initial_incident_id_field_name)
    arcpy.management.JoinField(routes_sub_layer, "IncidentID", cf_sub_layers["Incidents"], "ObjectID",
                               incident_join_fields)

    #Save the output layer. The layer name is based on random guid
    if save_output_layer:
        scratch_folder = arcpy.env.scratchFolder
        uid = str(uuid.uuid4()).replace("-","")
        na_layer_file_name = "_ags_gpna{0}.lyr".format(uid)
        output_layer_file = os.path.join(scratch_folder, na_layer_file_name)
        arcpy.management.SaveToLayerFile(cf_layer,output_layer_file)
        arcpy.AddIDMessage("INFORMATIVE", 30124, na_layer_file_name)

    #Export the selected facilities as a new feature class
    #Get the original facilities features before they were copied
    orig_input_facilities = facilities_obj.inputFeatures
    #Make a layer so we can use AddJoin
    orig_input_facilities_lyr = "OrigInputFacilitiesLayer"
    arcpy.management.MakeFeatureLayer(orig_input_facilities, orig_input_facilities_lyr)
    #Make a join based on FacilityOID from routes sublayer and OID from orig_input_facilities
    arcpy.management.AddJoin(orig_input_facilities_lyr, orig_input_facilities_oid, routes_sub_layer, "FacilityOID")
    where_clause = "CFRoutes.FacilityOID IS NOT NULL"
    arcpy.management.SelectLayerByAttribute(orig_input_facilities_lyr, "NEW_SELECTION", where_clause)
    arcpy.management.RemoveJoin(orig_input_facilities_lyr, "CFRoutes")
    #Transfer all attributes and the OID as ORIG_ID.
    #If ORIG_ID already exists, get a unique field name such as ORIG_ID_1
    fac_fms = arcpy.FieldMappings()
    fac_fms.addTable(orig_input_facilities_lyr)
    fac_fm = arcpy.FieldMap()
    fac_fm.addInputField(orig_input_facilities_lyr, orig_input_facilities_oid)
    out_fld = fac_fm.outputField
    unique_fld_name = nau.get_unique_field_name("ORIG_FID", orig_input_facilities_fld_names)
    out_fld.name = unique_fld_name
    out_fld.aliasName = unique_fld_name
    fac_fm.outputField = out_fld
    fac_fms.addFieldMap(fac_fm)
    arcpy.conversion.FeatureClassToFeatureClass(orig_input_facilities_lyr, out_gdb_workspace,
                                                out_facilities_name,field_mapping=fac_fms)
    #arcpy.management.CopyFeatures(orig_input_facilities_lyr, out_facilities_fc)

    #Prepare the field mappings for the routes layer before exporting it to a feature class.
    routes_feature_layer = "CFRoutes_Layer"
    arcpy.management.MakeFeatureLayer(routes_sub_layer, routes_feature_layer)
    field_mappings = arcpy.FieldMappings()
    field_mappings.addTable(routes_feature_layer)
    #Change the data type of FacilityID and IncidentID fields from Integer to Text
    type_update = {"type" : "TEXT", "length" : 50}
    nau.update_field_map_output_field(field_mappings, "FacilityID", type_update)
    nau.update_field_map_output_field(field_mappings, "IncidentID", type_update)

    name_update = dict.fromkeys(("name", "aliasName"), "")
    #Rename all accumate_attribute fields to Total_attributename_units
    for attr in accumulate_attributes:
        orig_name = "Total_" + attr
        new_name = "Total_{0}_{1}".format(attr, cost_attributes.get(attr, ""))
        name_update["name"] = new_name
        name_update["aliasName"] = new_name
        nau.update_field_map_output_field(field_mappings, orig_name, name_update)
    #Rename the time attribute and distance attribute as Total_Units
    time_cost_field = "Total_{0}".format(time_attribute_units)
    distance_cost_field = "Total_{0}".format(distance_attribute_units)
    name_update["name"] = time_cost_field
    name_update["aliasName"] = time_cost_field
    nau.update_field_map_output_field(field_mappings, "Total_" + time_attribute, name_update)
    name_update["name"] = distance_cost_field
    name_update["aliasName"] = distance_cost_field
    nau.update_field_map_output_field(field_mappings, "Total_" + distance_attribute, name_update)

    out_routes_fc_fld_names = [f.name for f in field_mappings.fields]
    #save routes to the feature class
    arcpy.conversion.FeatureClassToFeatureClass(routes_sub_layer, out_gdb_workspace, out_routes_name,"",
                                                field_mappings)

    #Make sure the cost is reported in expected units
    #values in the dict are the field from which to derive the value, units for field value, units to convert the value into.
    system_unit_cost_field_names = {"Total_Minutes" : (0, time_attribute_units, "Minutes"),
                                    "Total_Miles" : (1, distance_attribute_units, "Miles"),
                                    "Total_Kilometers": (1, distance_attribute_units, "Kilometers")
                                    }
    measurement_unit_cost_field = "Total_" + measurement_units.replace(" ", "")
    if not measurement_unit_cost_field in system_unit_cost_field_names:
        if measurement_method == "TRAVEL_TIME":
            system_unit_cost_field_names[measurement_unit_cost_field] = (0, time_attribute_units, measurement_units)
        else:
            system_unit_cost_field_names[measurement_unit_cost_field] = (1, distance_attribute_units, measurement_units)
    #determine the cost fields we need to convert units and store how the value should be converted

    cost_fields_to_calc = []
    for system_unit in system_unit_cost_field_names:
        if not system_unit in out_routes_fc_fld_names:
            arcpy.management.AddField(out_routes_fc, system_unit, "DOUBLE")
            cost_fields_to_calc.append(system_unit)

    if cost_fields_to_calc:

        with arcpy.da.UpdateCursor(out_routes_fc, [time_cost_field, distance_cost_field] + cost_fields_to_calc) as cursor:
            cursor_fld_names = cursor.fields
            for row in cursor:
                for i in range(2,len(cost_fields_to_calc) + 2):
                    field_name = cursor_fld_names[i]
                    src_value, from_unit, to_unit = system_unit_cost_field_names[field_name]
                    row[i] = nau.convert_units(row[src_value], from_unit, to_unit)
                cursor.updateRow(row)

    #Update IncidentID and FacilityID fields if ids were propogated
    if propogate_facility_ids:
        arcpy.management.CalculateField(out_routes_fc, "FacilityID",
                                        "!{0}!".format(initial_facility_id_field_name),"PYTHON")
        arcpy.management.DeleteField(out_routes_fc, initial_facility_id_field_name)
    if propogate_incident_ids:
        arcpy.management.CalculateField(out_routes_fc, "IncidentID",
                                        "!{0}!".format(initial_incident_id_field_name),"PYTHON")
        arcpy.management.DeleteField(out_routes_fc, initial_incident_id_field_name)

    #Save directions
    streetdirprops = None
    if populate_directions:

        streetdirprops = solver_props.streetDirectionsProperties
        if not streetdirprops:
            arcpy.AddIDMessage("WARNING",30129)
            populate_directions = False

    if populate_directions:
        statemgr = nau.StreetDirPropsStateResetManager(streetdirprops)

        streetdirprops.timeAttribute = time_attribute
        streetdirprops.lengthUnits = directions_distance_units
        try:
            if directions_language:
                streetdirprops.language = directions_language
        except Exception as ex:
            default_language = streetdirprops.language
            if default_language != directions_language:
                arcpy.AddIDMessage("WARNING",30099, default_language, directions_language)
        try:
            if directions_style_name:
                streetdirprops.styleName = directions_style_name
        except Exception as ex:
            default_style_name = streetdirprops.styleName
            if default_style_name != directions_style_name:
                arcpy.AddIDMessage("WARNING", 30100, default_style_name, directions_style_name)

        streetdirprops.outputSpatialReference = arcpy.env.outputCoordinateSystem

        arcpy.na.GenerateDirectionsFeatures(cf_layer, out_directions_fc, False)

        del statemgr
    else:
        arcpy.na.GenerateDirectionsFeatures(cf_layer, out_directions_fc, True)

    streetdirprops = None

except nau.InputError as ex:
    #Handle errors due to throttling conditions
    solve_succeeded = False
    if ex.message:
        nau.print_message(ex.message)
except arcpy.ExecuteError:
    #Handle GP exceptions
    solve_succeeded = False
    if DEBUG:
        #Get the line number at which the GP error occurred
        tb = sys.exc_info()[2]
        nau.print_message("A geoprocessing error occurred in File %s, line %s" % (__file__, tb.tb_lineno))
    else:
        nau.print_message("A geoprocessing error occurred.")
    warning_messages = arcpy.GetMessages(1)
    if warning_messages:
        nau.print_message(warning_messages, 1)
    nau.print_message(arcpy.GetMessages(2))
except:
    #Handle python errors
    solve_succeeded = False
    if DEBUG:
        #Get a nicely formatted traceback object except the first line.
        msgs = traceback.format_exception(*sys.exc_info())[1:]
        msgs[0] = "A python error occurred in " + msgs[0].lstrip()
        for msg in msgs:
            nau.print_message(msg.strip())
    else:
        nau.print_message("A python error occurred.")

finally:
    #Delete the in-memory na layer
    if cf_layer_exists:
        try:
            arcpy.management.Delete(cf_layer)
        except:
            pass
    #Delete copies of inputs
    for obj in input_copies:
        if obj:
            obj.deleteCopy()

    arcpy.SetParameter(parameter_index['solve_succeeded'], solve_succeeded)
    arcpy.SetParameterAsText(parameter_index['out_routes'], out_routes_fc)
    arcpy.SetParameterAsText(parameter_index['out_directions'], out_directions_fc)
    arcpy.SetParameterAsText(parameter_index['out_facilities'], out_facilities_fc)


from collections import Counter
import operator
import numpy as np
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity as cs

# vectors_path = "data/vectors_filtered_by_vet/"
vectors_path = "data/vectors_vet_as_label/"

f_label = open("data/ads_category.txt","r")

labels = []
for line in f_label:
	labels.append(line.strip())

f_label.close()

n = len(labels)
label_frequency = Counter(labels)
sorted_label_frequency = sorted(label_frequency.items(), key=operator.itemgetter(1), reverse = True)

filter_label_with_most_frequency = [label for label,count in sorted_label_frequency][:20]

# 'BusinessAndManagement', 'Banking,FinanceAndRelatedFields', 'Building', 'ElectricalAndElectronicEngineeringAndTechnology', 'SalesAndMarketing', 'FoodAndHospitality', 'ProcessAndResourcesEngineering', 'HumanWelfareStudiesAndServices', 'OfficeStudies', 'Nursing'

sample_size = 5000
ind_lists = []
for label in filter_label_with_most_frequency:
	ind_list = []
	for i in range(n):
		if labels[i] == label:
			ind_list.append(i)
		if len(ind_list) == sample_size:
			break
	ind_lists.extend(ind_list)

ads_vectors = np.zeros((n,100))
# f = open("data/sentence_vectors_new_with_punctuation.txt")
f = open(vectors_path + "ads_vectors.txt","r")
for i,line in enumerate(f):
	ads_vectors[i] = line.split()

ads = ads_vectors[ind_lists]
# similarity_matrix = cs(test_vectors, test_vectors)

# for i in range(10 - 1):
# 	print np.mean(similarity_matrix[170][ sample_size * i : sample_size * (i+1) ])

def read_txt(filename, filetype):
	l = []
	f = open(filename, "r")
	if filetype == "vectors":
		for line in f:
			l.append(line.strip().split())
	elif filetype == "label":
		for line in f:
			l.append(line.strip())
	return l

vet_label = read_txt("data/vet_category.txt", "label")
vet_vectors = read_txt("data/vet_vectors.txt", "vectors")
vets = np.array(vet_vectors, dtype = np.float)

similarity_matrix = cs(ads, vets)

mean_matrix = np.zeros((len(filter_label_with_most_frequency),len(vet_label)))
for i in range(len(vet_label)):
	c = similarity_matrix[:,i]
	for j in range(len(filter_label_with_most_frequency)):
		mean_matrix[j,i] = np.mean(c[j*sample_size:(j+1)*sample_size])

df = pd.DataFrame(mean_matrix.T, index = vet_label, columns = filter_label_with_most_frequency).to_csv(vectors_path + "similarity_result.csv")


'''
Created on May 14, 2014

@author: bernard
'''
import maya.cmds as cmds

def getMaxAndMinAxis (objectName, preview=False):
    """
    send a object to get the Max and Min X,Y and Z
    @param objectName: Name of Object
    @type objectName: String
    --------------------------------------------
    @param preview: to see the preview
    @type preview: Bool
    --------------------------------------------
    @return: 0-maxX, 1-minX, 2-maxY, 3-minY, 4-maxZ, 5-minZ
    """
    maxX,maxY,maxZ,minX,minY,minZ = 0,0,0,0,0,0
    vertexList = cmds.getAttr("%s.vrts"%objectName, multiIndices=True )
    i = 0
    for i in vertexList:
        cmds.select("%s.pnts[%s]"%(objectName,i))
        vertexPositionX, vertexPositionY, vertexPositionZ = cmds.xform("%s.pnts[%s]"%(objectName,i), query=True, translation=True, worldSpace=True )
        if vertexPositionX > maxX or i == 0:
            maxX = vertexPositionX
        if vertexPositionY > maxY or i == 0:
            maxY = vertexPositionY
        if vertexPositionZ > maxZ or i == 0:
            maxZ = vertexPositionZ
        if vertexPositionX < minX or i == 0:
            minX = vertexPositionX
        if vertexPositionY < minY or i == 0:
            minY = vertexPositionY
        if vertexPositionZ < minZ or i == 0:
            minZ = vertexPositionZ

    if preview == True:
        cmds.spaceLocator(name="minX", p=(0, 0, 0))
        cmds.spaceLocator(name="minY", p=(0, 0, 0))
        cmds.spaceLocator(name="minZ", p=(0, 0, 0))
        cmds.spaceLocator(name="maxX", p=(0, 0, 0))
        cmds.spaceLocator(name="maxY", p=(0, 0, 0))
        cmds.spaceLocator(name="maxZ", p=(0, 0, 0))
        meshX,meshY,meshZ = cmds.xform(objectName, query=True, translation=True, worldSpace=True )
        cmds.move( minX, meshY, meshZ, 'minX')
        cmds.move( maxX, meshY, meshZ, 'maxX')
        cmds.move( meshX, minY, meshZ, 'minY')
        cmds.move( meshX, maxY, meshZ, 'maxY')
        cmds.move( meshX, meshY, minZ, 'minZ')
        cmds.move( meshX, meshY, maxZ, 'maxZ')

    return maxX,minX,maxY,minY,maxZ,minZ

def renameShape (currentName='', changNameTo=''):
    """
    send a shapeName and change the name and transform
    @param currentName: The name of shape
    @type currentName: String
    --------------------------------------------
    @param changeNameTo: The new name
    @type changeNameTo: String
    --------------------------------------------
    @return: 0-Shape, 1-Transform
    """
    returnNewName=[]
    getParentTransform = cmds.listRelatives(currentName, parent = True)
    returnNewName.append(cmds.rename(currentName, '%sShape' % changNameTo))
    returnNewName.append(cmds.rename(getParentTransform, '%s' % changNameTo))
    return returnNewName

def renameTransform (currentName='', changNameTo=''):
    """
    send a transformName and change the name and shape
    @param currentName: The name of transform
    @type currentName: String
    --------------------------------------------
    @param changeNameTo: The new name
    @type changeNameTo: String
    --------------------------------------------
    @return: 0-Shape , 1-Transform
    """
    returnNewName=[]
    if cmds.listRelatives(currentName, parent = False, type='shape'):
        getChild = cmds.listRelatives(currentName, parent = False)
        returnNewName.append(cmds.rename(getChild[0], '%sShape' % changNameTo))
    returnNewName.append(cmds.rename(currentName, '%s' % changNameTo))
    return returnNewName

def oceanBuilder(projectName):
    import BBmaya.tools_fx.modules.BB_SplashGenerator as generator
    reload(generator)
    objectName = "Apple"

    # create Node
    objectOceanShader_S = cmds.createNode("oceanShader",name="%s_OceanShader"%objectName)
    objectHeightField = cmds.createNode("heightField",name="%s_OceanPreviewPlane"%objectName)
    objectShadingEngine_S = cmds.createNode("shadingEngine",name="%s_OceanShader"%objectName)
    objectMakeNurbPlane_S = cmds.createNode("makeNurbPlane",name="%s_MakeNurbPlane"%objectName)
    objectNurbSurface = cmds.createNode("nurbsSurface",name="%s_OceanPlane"%objectName)

    # Renaming Shape and Transform
    objectHeightField_S, objectHeightField_T = generator.renameShape(objectHeightField,"%s_OceanPreviewPlane"%objectName)
    objectNurbSurface_S, objectNurbSurface_T = generator.renameShape(objectNurbSurface,"%s_OceanPlane"%objectName)

    # Connecting Ocean Shaders
    cmds.connectAttr("%s.outColor"%objectOceanShader_S,"%s.color"%objectHeightField_S)
    cmds.connectAttr("%s.displacement"%objectOceanShader_S,"%s.displacement"%objectHeightField_S)
    cmds.connectAttr("%s.outColor"%objectOceanShader_S,"%s.surfaceShader"%objectShadingEngine_S)
    cmds.connectAttr("%s.displacement"%objectOceanShader_S,"%s.displacementShader"%objectShadingEngine_S)
    cmds.connectAttr("time1.outTime","%s.time"%objectOceanShader_S)
    cmds.connectAttr("%s.instObjGroups"%objectNurbSurface_S,"%s.dagSetMembers[0]"%objectShadingEngine_S)
    cmds.connectAttr("%s.outputSurface"%objectMakeNurbPlane_S,"%s.create"%objectNurbSurface_S)
    cmds.connectAttr("%s.message"%objectShadingEngine_S,"lightLinker1.link[2].object")
    cmds.connectAttr("%s.message"%objectShadingEngine_S,"lightLinker1.shadowLink[2].shadowObject")
    cmds.connectAttr("%s.partition"%objectShadingEngine_S,"renderPartition.sets[2]")

    # HeightField preset
    cmds.setAttr("%s.resolution"%objectHeightField_S,20)
    cmds.setAttr("%s.scaleX"%objectHeightField_T,10)
    cmds.setAttr("%s.scaleZ"%objectHeightField_T,10)

def expressionVeriableBuilder():
    '''float $particleSprayRate = 3000;
        float $particleBubblesRate = 100;
        float $fluidDisplacement = 6.0;
        float $fluidFoam = 2.0;
        float $u = .I[0];
        float $v = .I[1];
        float $disp[] = `colorAtPoint -u $u -v $v oceanShader1`;
        float $lastY = `getAttr -time (frame - 2) %s.translateY`;
        float $curY = %s.translateY;
        float $ydiff = $lastY - $curY;'''


import multiprocessing

import numpy as np
from numpy import savetxt
import torch

from torch.utils.data import DataLoader
from torch.optim.lr_scheduler import ExponentialLR
# from torch.utils.tensorboard import SummaryWriter

import argparse
import sys
import utils.config as config
from rdkit import Chem
import json
import os
import csv
import pickle
import time
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from sklearn.model_selection import KFold
import numpy as np

import torch.nn as nn
from torch.nn.utils.rnn import pack_padded_sequence

from src.utils.build_vocab import Vocabulary
from src.datasets.data_loader import Pdb_Dataset
from src.evaluation.Contrib.statistics import analysis_to_csv, analysis_to_csv_test
from src.training.utils import save_checkpoint_sampling

class Sampler():
    def __init__(self, cfg, sampling, Feature_Loader):
        self.cfg = cfg
        self.Feature_Loader = Feature_Loader
        self.path_root = cfg['preprocessing']['path_root']
        self.init_refined = self.path_root + "/data/new_refined/"
        self.files_refined = os.listdir(self.init_refined)
        self.files_refined = [file for file in self.files_refined if file[0].isdigit()]
        self.files_refined.sort()

        self.attention = self.cfg['training_params']['mode']
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        # self.device = torch.device("cpu")
        self.sampling = sampling
        self.model_encoder =  cfg['model']['encoder']
        print(self.model_encoder)
        self.model_decoder =  cfg['model']['decoder']
        self.sampling_data = cfg['sampling_params']['sampling_data']
        self.protein_dir = cfg["training_params"]["image_dir"]
        # self.number_smiles = cfg["sampling_params"]["number_smiles"]
        # if (self.sampling == "max"):
        #     self.number_smiles = 1
        self.time_waiting = cfg["sampling_params"]["time_waiting"]
        self.type_fold = cfg["sampling_params"]["type_fold"]
        # model params
        self.model_name = cfg['model_params']['model_name']
        self.num_epochs = cfg['model_params']['num_epochs']
        self.batch_size = cfg['model_params']['batch_size']
        self.learning_rate = cfg['model_params']['learning_rate']
        self.num_workers = cfg['model_params']['num_workers']

        # training params
        self.protein_dir = cfg['training_params']['image_dir']
        self.caption_path = cfg['training_params']['caption_path']
        self.log_step = cfg['training_params']['log_step']
        self.save_step = cfg['training_params']['save_step']
        self.vocab_path = cfg['preprocessing']['vocab_path']
        #output files
        self.savedir = os.path.join(cfg['output_parameters']['savedir'], self.model_name)
        self.save_dir_smiles = os.path.join(self.savedir, "statistics")
        self.tesnorboard_path = self.savedir
        self.log_path = os.path.join(self.savedir, "logs")
        self.idx_file = os.path.join(self.log_path, "idxs")
        #encoder/decoder path
        # self.encoder_path = os.path.join(self.savedir, "models", cfg['training_params']['encoder_name'])
        # self.decoder_path = os.path.join(self.savedir, "models", cfg['training_params']['decoder_name'])
        self.save_dir_encodings = os.path.join(cfg['output_parameters']['savedir'], "encodings", self.model_name)
        #sampling params
        os.makedirs(self.save_dir_smiles, exist_ok=True)
        os.makedirs(self.save_dir_encodings, exist_ok=True)

        with open(self.vocab_path, "rb") as f:
            self.vocab = pickle.load(f)

        self.dataset = Pdb_Dataset(cfg, self.vocab)
        self.path_checkpoint_evaluator = os.path.join(self.savedir, "checkpoints", "checkpoint_evaluator.csv")
        if os.path.exists(self.path_checkpoint_evaluator):
            self.data_checkpoint = pd.read_csv(self.path_checkpoint_evaluator)

    def analysis_cluster(self, split_no, epoch_no, type_fold, encoder_path, decoder_path):
        # encoder, decoder = self._get_model_path(idx_fold)
        self.idx_fold = split_no
        self.type_fold = type_fold
        self.epoch_no = epoch_no
        self.name_file_stat = self.sampling + "_" + str(self.type_fold) + "_" + str(self.idx_fold) + ".csv"
        self.path_to_file_stat = os.path.join(self.save_dir_smiles, self.name_file_stat)
        self.file_statistics = open(self.path_to_file_stat, "a+")
        self.checkpoint_sampling_path = os.path.join(self.savedir, "checkpoints", str(split_no) + '_sample.pkl')
        #the file of the whole stat
        if (len(open(self.path_to_file_stat).readlines()) == 0):
            self.file_statistics.write("name,fold,type_fold,epoch_no,orig_smile,gen_smile,gen_NP,gen_logP,gen_sa,gen_qed,gen_weight,gen_similarity,orig_NP,orig_logP,orig_sa,orig_qed,orig_weight,frequency,sampling,encoder,decoder" +  "\n")
            self.file_statistics.flush()
        # checkpoint_sampling = torch.load(self.checkpoint_sampling_path)
        print("loading start_ind_protein...")
        # start_ind_protein = checkpoint_sampling['idx_sample_start']
        start_ind_protein = self.data_checkpoint.loc[(self.data_checkpoint['type_fold'] == self.type_fold) & (self.data_checkpoint['sampling'] == self.sampling), 'start_pdb']
        # idx_sample = checkpoint_sampling['idx_sample_regime_start']
        self.encoder, self.decoder = config.eval_model_captioning(self.cfg, encoder_path, decoder_path, device = self.device)
        # self.file_folds = os.path.join(self.idx_file, "test_idx_" + str(self.idx_fold))
        self.file_folds = os.path.join(self.idx_file, self.type_fold)
        with (open(self.file_folds, "rb")) as openfile:
            idx_proteins = pickle.load(openfile)
            train_idx, test_idx = idx_proteins[self.idx_fold]
            print("train idx , - ", train_idx)
            print("test idx , - ", test_idx)
        # idx_proteins = [1,2,3,4]
        files_refined = os.listdir(self.protein_dir)
        idx_all = [i for i in range(len(files_refined) - 3)]
        #take indx of proteins in the training set
        if (self.sampling_data == "train"):
            # idx_to_generate = np.setdiff1d(idx_all, idx_proteins)
            idx_to_generate = train_idx
        else:
            # idx_to_generate = idx_proteins
            idx_to_generate = test_idx

        #sampling checkpoint
        end_idx = len(idx_to_generate)
        for idx in range(int(start_ind_protein), end_idx):
            id_abs_protein = idx_to_generate[idx]
            self.generate_smiles(id_abs_protein)
            next_idx = (idx + 1) % end_idx
            # print("next_ind!!!! - ", next_idx)
            # print("end_ind!! - ", end_idx)
            # print("ind!!! - ", idx)

            self.data_checkpoint.loc[(self.data_checkpoint['type_fold'] == self.type_fold) & (self.data_checkpoint['sampling'] == self.sampling), 'start_pdb'] = next_idx
            # save_checkpoint_sampling(self.checkpoint_sampling_path, next_idx, idx_sample)
            if (next_idx == 0):
                self.data_checkpoint.loc[(self.data_checkpoint['type_fold'] == self.type_fold) & (self.data_checkpoint['sampling'] == self.sampling), 'start_rec_epoch'] = epoch_no + 1
                # save_checkpoint_sampling(self.checkpoint_sampling_path, next_idx, idx_sample + 1)
            self.data_checkpoint.to_csv(self.path_checkpoint_evaluator, index=False)

    def _get_models(self, idx_fold):
        encoder_path, decoder_path = self._get_model_path(idx_fold)
        encoder, decoder = config.eval_model_captioning(cfg, encoder_path, decoder_path, device = self.device)
        return encoder, decoder

    def _get_model_path(self):
        encoder_name = "encoder-" + str(self.idx_fold) + "-1-2.ckpt"
        decoder_name = "decoder-" + str(self.idx_fold) + "-1-2.ckpt"
        encoder_path = os.path.join(self.savedir, "models", encoder_name)
        decoder_path = os.path.join(self.savedir, "models", decoder_name)
        return encoder_path, decoder_path

    def load_pocket(self, id_protein, transform=None):
        print("loading data of a protein", self.dataset._get_name_protein(id_protein))
        # features, masks = self.dataset._get_features_complex(id_protein)
        # geometry = self.dataset._get_geometry_complex(id_protein)
        # features = features.to(self.device).unsqueeze(0)
        # geometry = geometry.to(self.device).unsqueeze(0)
        # masks = masks.to(self.device).unsqueeze(0)
        features, masks, geometry = self.Feature_Loader._get_feat_geo_from_file(id_protein)
        features = features.to(self.device).unsqueeze(0)
        geometry = geometry.to(self.device).unsqueeze(0)
        masks = masks.to(self.device).unsqueeze(0)
        return features, geometry, masks

    def generate_encodings(self, id):
        #generate features of encoder and writes it to files
        protein_name =  self.dataset._get_name_protein(id)
        features, geometry, masks = self.load_pocket(id)
        # Generate a caption from the image
        feature = self.encoder(features, geometry, masks)
        torch.save(feature, os.path.join(self.folder_save, protein_name + "_feature_encoding.pt"))

    def printing_smiles(self, sampled_ids, list_smiles_all):
        sampled_caption = []
       # print("sampled_id", sampled_ids)
        for word_id in sampled_ids:
            word = self.vocab.idx2word[word_id]
            sampled_caption.append(word)
            if word == "<end>":
                break
        sentence = "".join(sampled_caption)
        sentence = sentence[7:-5]
        print(sentence)
        m = Chem.MolFromSmiles(sentence)
        if m is None or sentence == '' or sentence.isspace() == True:
            print('invalid')
            # list_smiles_all.append(sentence)
            return 1
        else:
            print(sentence)
            # smiles.append(sentence)
            list_smiles_all.append(sentence)
            return 1

    def smiles_all_txt(self):
        file_all_smiles = open(os.path.join(self.save_dir_smiles, "all_smiles_lig.txt"), "w")
        files_refined =  os.listdir(self.caption_path)
        files_refined.remove(".DS_Store")
        for protein_name in files_refined:
            init_path_smile =  os.path.join(
                self.caption_path, protein_name, protein_name + "_ligand.smi"
            )
            with open(init_path_smile) as fp:
                initial_smile = fp.readlines()[0]
                file_all_smiles.write(initial_smile + "\n")
                file_all_smiles.flush()

    def generate_smiles(self, id):
        #original + gen smiles
        print("current id - ", id)
        smiles = []
        protein_name =  self.dataset._get_name_protein(id)
        print("current protein ", protein_name)
        #path of the real smile
        init_path_smile =  os.path.join(
                self.caption_path, protein_name, protein_name + "_ligand.smi"
            )

        with open(init_path_smile) as fp:
            initial_smile = fp.readlines()[0] #write a true initial smile
        smiles.append(initial_smile)
        amount_val_smiles = 0

        iter = 0
        start = time.time()
        if (self.sampling == "beam_1"):
            self.number_smiles = 1
        else:
            self.number_smiles = self.cfg["sampling_params"]["number_smiles"]
        if (self.sampling.startswith('beam') == False):
            while (amount_val_smiles < self.number_smiles):
                end = time.time()
                # print("time elapsed", end - start)
                if((end - start) > self.time_waiting):
                    #stop generating if we wait for too long till 50 ligands
                    self.file_long_proteins = open(os.path.join(self.save_dir_smiles, "exceptions_long.txt"), "w")
                    self.file_long_proteins.write(protein_name + "\n") #write a protein with long time of generating
                    self.file_long_proteins.flush()
                    break
                iter += 1
                # Build models
                # Load the trained model parameters
                # # Prepare features and geometry from pocket
                features, geometry, masks = self.load_pocket(id)

                # Generate a caption from the image
                feature = self.encoder(features, geometry, masks)
                #print("feature", feature)

                if (self.sampling == "probabilistic"):
                    sampled_ids = self.decoder.sample_prob(feature)
                    # if self.cfg["training_params"]["mode"] != "attention":
                    #     sampled_ids = self.decoder.sample_prob(feature)
                    # else:
                elif (self.sampling == "max"):
                    sampled_ids = self.decoder.sample_max(feature)
                    self.number_smiles = 1
                elif (self.sampling == "simple_probabilistic"):
                    sampled_ids = self.decoder.simple_prob(feature)
                elif (self.sampling.startswith("simple_probabilistic_topk") == True):
                    k = int(self.sampling.split("_")[-1])
                    sampled_ids = self.decoder.simple_prob_topk(feature, k)
                elif (self.sampling.startswith("temp_sampling")):
                    temperature = float(self.sampling.split("_")[-1])
                    sampled_ids = self.decoder.sample_temp(feature, temperature)
                sampled_ids = ( sampled_ids[0].cpu().numpy())
                if(type(sampled_ids[0]) != list):
                    idx =  self.printing_smiles(sampled_ids, smiles)
                    amount_val_smiles += idx
                else:
                    amount_val_smiles = 0

        elif (self.sampling.startswith('beam') == True):
            number_beams = int(self.sampling.split("_")[1])
            features, geometry, masks = self.load_pocket(id)
            feature = self.encoder(features, geometry, masks)
            # self.decoder = self.decoder.float()
            if (self.attention == "attention"):
                sampled_ids, alphas  = self.decoder.sample_beam_search(feature, number_beams)
            else:
                sampled_ids  = self.decoder.sample_beam_search(feature, number_beams)
            # print("sampled-ind", sampled_ids)
            if(sampled_ids == 120):
                amount_val_smiles = 0
            else:
                for sentence in sampled_ids:
                    print("sentence", sentence[1:])
                    iter += 1
                    idx =  self.printing_smiles(np.asarray(sentence[1:]), smiles)
                    amount_val_smiles += idx
        else:
            raise ValueError("Unknown sampling...")

        if (amount_val_smiles > 0):
            # print("stat write!!!")
            # save_dir_analysis = os.path.join(save_dir_smiles, str(self.idx_fold), protein_name)
            stat_protein = analysis_to_csv(smiles,  protein_name, self.idx_fold, self.type_fold, self.epoch_no) #get the list of lists of statistics
            # stat_protein = np.transpose(np.vstack((stat_protein, np.asarray(amount_val_smiles * [amount_val_smiles /iter]))))
            stat_protein.append(amount_val_smiles * [amount_val_smiles /iter])
            stat_protein.append(amount_val_smiles * [self.sampling])
            stat_protein.append(amount_val_smiles * [self.model_encoder])
            stat_protein.append(amount_val_smiles * [self.model_decoder])
            # file_statistics.write(str(list(map(list, zip(*stat_protein)))) + "\n")
            wr = csv.writer(self.file_statistics)
            wr.writerows(list(map(list, zip(*stat_protein))))
            self.file_statistics.flush()
        # else:
        #     length = self.number_smiles
        #     # print("length, - ", length)
        #     stat_protein = [length * ['a'], length * ['a'], length * [str(self.epoch_no)], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'],
        #           length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a'], length * ['a']]
        #     wr = csv.writer(self.file_statistics)
        #     wr.writerows(list(map(list, zip(*stat_protein))))
        #     self.file_statistics.flush()
            # print("end of stat!")

    def analysis_all(self):
        #for every fold takes indicies for the test, generates smiles and builds statistics
        num_folds = 3
        # all_stat = np.empty((1, 8))
        for id_fold in range(num_folds):
            file_freq = open(os.path.join(save_dir_smiles, str(id_fold), str(id_fold) + "_freq.txt"), "w")
            file_idx = os.path.join(save_dir_folds, "test_idx_" + str(id_fold))
            with (open(file_idx, "rb")) as openfile:
                idx_proteins = pickle.load(openfile)
            for id_protein in idx_proteins:
                self.generate_smiles(id_protein)

    def test_analysis_all(self):
        #for every fold takes indicies for the test, generates smiles and builds statistics
        num_folds = 3
        all_stat = []
        # idx_array = [[11,12], [14, 15]]
        idx_array = [[11], [14]]
        for id_fold in range(2):
            file_freq = open(os.path.join(save_dir_smiles, str(id_fold), str(id_fold) + "_freq.txt"), "w")
            idx_proteins = idx_array[id_fold]
            for id_protein in idx_proteins:
                self.generate_smiles(id_protein)

        # all_stat = np.array(all_stat)
        # print("shape all_stat", len(all_stat))
        # print("all_stat", all_stat)
        df = pd.DataFrame(all_stat, columns = ['name', 'fold', 'logP','sa','qed','weight','similarity', 'orig_logP', 'orig_sa', 'orig_qed', 'orig_weight','frequency'])
        df.to_csv(os.path.join(save_dir_smiles, "all_stat_new.csv"))

    def save_encodings_all(self, mode, split_no, encoder_path, decoder_path):
        r'''For every protein id in rain/test generates feature and saves it
        '''
        self.mode_split = mode
        self.type_fold = self.cfg["sampling_params"]["type_fold"]
        self.folder_save = os.path.join(self.save_dir_encodings, mode)
        if not os.path.exists(self.folder_save):
            os.makedirs(self.folder_save )

        self.encoder, self.decoder = config.eval_model_captioning(self.cfg, encoder_path, decoder_path, device = self.device)

        idx_folds = pickle.load(open(os.path.join(self.idx_file, self.type_fold), "rb" ) )
        train_id, test_id = idx_folds[split_no]
        if (mode == "test"):
            idx_proteins_gen = test_id
        else:
            idx_proteins_gen = train_id

        for id_protein in idx_proteins_gen:
            self.generate_encodings(id_protein)

        files_encodings =  os.listdir(self.folder_save)
        all_encodings = []
        for file_enc in files_encodings:
            if(file_enc[0].isdigit()):
                path_to_enc = os.path.join(self.folder_save, file_enc)
                enc_from_torch = torch.load(path_to_enc, map_location=torch.device('cpu')).view(-1).detach().numpy()
                # print(type(enc_from_torch))
                all_encodings.append(enc_from_torch)
        all_encodings = np.asarray(all_encodings)
        name = str(self.mode_split) + "_" + str(split_no) + "_" + str(self.type_fold)+ '_' + self.model_name + "_all_encodings.csv"
        np.savetxt(os.path.join(self.save_dir_encodings, name), all_encodings, delimiter=',')

    def collect_all_encodings(self):
        r''' Writes all saved features to 1 file
        '''
        files_encodings =  os.listdir(self.folder_save)
        all_encodings = []
        for file_enc in files_encodings:
            if(file_enc[0].isdigit()):
                path_to_enc = os.path.join(self.folder_save, file_enc)
                enc_from_torch = torch.load(path_to_enc, map_location=torch.device('cpu')).view(-1).detach().numpy()
                # print(type(enc_from_torch))
                all_encodings.append(enc_from_torch)
        all_encodings = np.asarray(all_encodings)
        name = str(self.mode_split) + "_all_encodings.csv"
        np.savetxt(os.path.join(self.save_dir_encodings, name), all_encodings, delimiter=',')

        # df = pd.DataFrame(all_stat, columns = ['name', 'fold', 'logP','sa','qed','weight','similarity', 'frequency'])
        # df = pd.DataFrame(all_stat, columns = ['name', 'fold', 'logP','sa','qed','weight','similarity', 'orig_logP', 'orig_sa', 'orig_qed', 'orig_weight','frequency'])
        # df.to_csv(os.path.join(save_dir_smiles, "all_stat_new.csv"))

        # all_stat = np.vstack((all_stat, stat_protein))
        # all_stat += map(list, zip(*stat_protein))