text:
from utilities.util import (generate_weighted_connected_graph,
                            generate_random_cluster,
                            generate_cover)
import networkx as nx

#############################################
# CHECKERS ##################################
#############################################

# assert that g is a connected graph
def check_graph_connectivity(g):
    assert nx.is_connected(g)

# assert that g is a weighted graph
def check_graph_weights(g, max_weight=100):
    for u, v in g.edges():
        assert 0 <= g[u][v]['weight'] <= max_weight

# assert that cluster is connected
def check_cluster_connectivity(g, c):
    check_graph_connectivity(nx.subgraph(g, list(c)))

# assert that cluster is in the requested size
def check_cluster_size(c, min_size, max_size):
    assert min_size <= len(c) <= max_size

# assert cover is in the requested size
def check_cover_size(cover, requested_size):
    assert len(cover) >= requested_size

# assert cover is a correct coverage of graph nodes
def check_cover_correctness(cover, graph_nodes):
    assert frozenset.union(*([c for c in cover])) == set(graph_nodes)

# assert each cluster in the cover is in the requested size
def check_cover_clusters_size(cover, min_cluster_size, max_cluster_size):
    for c in cover:
        check_cluster_size(c, min_cluster_size, max_cluster_size)

#############################################
# TESTS #####################################
#############################################

def test_generate_weighted_connected_graph_connectivity():
    for size in range(1, 100):
        g = generate_weighted_connected_graph(size)
        check_graph_connectivity(g)

def test_generate_weighted_connected_graph_weights():
    for size in range(1, 101, 20):
        for max_weight in range(50):
            g = generate_weighted_connected_graph(size, max_weight=max_weight)
            check_graph_weights(g, max_weight)

def test_generate_random_cluster_connectivity():
    for size in range(1, 100):
        g = generate_weighted_connected_graph(size)
        for _ in range(10):
            c = generate_random_cluster(g)
            check_cluster_connectivity(g, c)

def test_generate_random_cluster_min_max_size():
    for size in range(1, 30):
        g = generate_weighted_connected_graph(size)
        for i in range(1, size):
            for j in range(i, size):
                c = generate_random_cluster(g, min_cluster_size=i, max_cluster_size=j)
                check_cluster_connectivity(g, c)

def test_generate_cover_coverage():
    for graph_size in range(1, 40):
        g = generate_weighted_connected_graph(graph_size)
        for requested_cover_size in range(1, graph_size):
            cover = generate_cover(g, requested_cover_size)
            check_cover_size(cover, requested_cover_size)
            check_cover_correctness(cover, g.nodes())

def test_generate_cover_min_max_cluster_size():
    for graph_size in range(1, 20):
        g = generate_weighted_connected_graph(graph_size)
        for i in range(1, graph_size):
            for j in range(i, graph_size):
                cover = generate_cover(g, 1, min_cluster_size=i, max_cluster_size=j)
                check_cover_size(cover, 1)
                check_cover_correctness(cover, g.nodes())
                check_cover_clusters_size(cover, i, j)

max_stars_repo_name: eyal-shagrir/Graph-Cluster-Covers

max_issues_repo_name: eyal-shagrir/Graph-Cluster-Covers

max_forks_repo_name: eyal-shagrir/Graph-Cluster-Covers

reason: import networkx


text:
"""
Authors : Aditya Jain
Contact : https://adityajain.me
"""

import numpy as np
from ..tree import DecisionTreeClassifier
from ..tree import DecisionTreeRegressor

class RandomForestClassifier():
	"""
	Random Forest fits number of decision tree on various sub-samples of the dataset and uses averaging to improve the predictive accuracy and control over-fitting.
	The sub-sample size is always the same as the original input sample size but the samples are drawn with replacement if bootstrap=True.

	Parameter
	---------
	n_estimators : integer (Default 50), number of trees in forest

	max_depth : integer (Default 'inf'), maximum depth allowed for each tree

	min_samples_split : integer (Default 2), The minimum number of samples required to split an internal node

	max_features : ( 'auto', 'sqrt', 'log2', 'max_features' )
		The number of features to consider when looking for the best split:
		'auto' is same as 'sqrt'
		'sqrt' is sqrt(number of features)
		'log2' is log2(number of features)
		'max_features' is all features

	bootstrap : If False, the whole datset is used to build each tree.

	random_state : random seed

	"""
	def __init__(self, n_estimators=50, max_depth = None, min_samples_split=2, max_features="auto",
				 bootstrap=True, random_state=None):
		np.random.seed( random_state if random_state!=None else np.random.randint(100)  )
		self.__n_estimators = n_estimators
		self.__max_depth = float('inf') if max_depth==None else max_depth
		self.__min_samples_split = min_samples_split
		self.__max_features = {
			'auto': lambda x: int(np.sqrt(x)), 'sqrt': lambda x: int(np.sqrt(x)),
			'log2': lambda x: int(np.log2(x)), 'max_features': lambda x: x  }[max_features]
		self.__bootstrap = bootstrap
		self.__n_samples = None
		self.__n_features = None
		self.__n_classes = None
		self.__trees = [  ]

	def __bootstrapX(self,X):
		indexes = np.random.choice( np.arange(0,len(X),1), size=self.__n_samples, replace=self.__bootstrap )
		return X[indexes,:]

	def __get_feature_index(self):
		return np.random.choice( np.arange(0,self.__n_features,1),
								size=self.__max_features(self.__n_features), replace=False)

	def fit(self,X,y):
		"""
		Fit the X and y to estimators

		Parameters
		----------
		X : numpy array, independent variables

		y : numpy array, target variable

		"""
		self.__n_samples, self.__n_features = X.shape
		self.__n_classes = len(np.unique(y))
		X_y = np.c_[X,y]
		for _ in range(self.__n_estimators):
			dt = DecisionTreeClassifier( max_depth=self.__max_depth,
										min_samples_split=self.__min_samples_split,
									   n_classes = self.__n_classes)
			data = self.__bootstrapX(X_y)
			features = self.__get_feature_index()
			dt.fit( data[:,features], data[:,-1] )
			self.__trees.append( (dt.tree_, features) )

	def predict(self,X):
		"""
		Predict labels using all estimators

		Parameters
		----------
		X : numpy array, independent variables

		Returns
		-------
		predicted labels

		"""
		return np.argmax( self.predict_proba(X), axis=1 )

	def predict_proba(self,X):
		"""
		Predict probaibilty of each class using all estimators

		Parameters
		----------
		X : numpy array, independent variablesss

		Returns
		-------
		probability of each class [ n_samples, n_classes ]

		"""
		probs = np.zeros( (len(X),self.__n_classes) )
		for root, features in self.__trees:
			probs += np.array([ self.__predict_row(row,root)[1] for row in X[:,features] ])
		return probs/self.__n_estimators

	def __predict_row(self,row,node):
		if row[node['index']] < node['value']:
			if isinstance(node['left'], dict):  return self.__predict_row(row,node['left'])
			else: return node['left']
		else:
			if isinstance(node['right'], dict): return self.__predict_row(row,node['right'])
			else: return node['right']

	def score(self,X,y):
		"""
		Calculate accuracy from independent variables

		Parameters
		----------
		X : numpy array, independent variables

		y : numpy array, dependent variable

		Returns
		-------
		accuracy score

		"""
		return (y==self.predict(X)).sum()/len(y)

class RandomForestRegressor():
	"""
	Random Forest fits number of decision tree on various sub-samples of the dataset and uses averaging to improve the predictive accuracy and control over-fitting.
	The sub-sample size is always the same as the original input sample size but the samples are drawn with replacement if bootstrap=True.

	Parameter
	---------
	n_estimators : integer (Default 50), number of trees in forest

	criterion : ('mse', 'mae', 'std') ( Default 'mse' )
		The function to measure the quality of a split.
		'mse' is mean squared error
		'mae' is mean absolute error
		'std' is standard deviation

	max_depth : integer (Default 'inf'), maximum depth allowed for each tree

	min_samples_split : integer (Default 2), The minimum number of samples required to split an internal node

	max_features : ( 'auto', 'sqrt', 'log2', 'max_features' ) ( Default 'auto' )
		The number of features to consider when looking for the best split:
		'auto' is same as 'sqrt'
		'sqrt' is sqrt(number of features)
		'log2' is log2(number of features)
		'max_features' is all features

	bootstrap : If False, the whole datset is used to build each tree.

	random_state : random seed

	"""
	def __init__(self,  n_estimators=10, criterion='mse', max_depth = None, min_samples_split=2, max_features="auto",
				 bootstrap=True, random_state=None):
		np.random.seed( random_state if random_state!=None else np.random.randint(100)  )
		self.__n_estimators = n_estimators
		self.__criterion = criterion
		self.__max_depth = float('inf') if max_depth==None else max_depth
		self.__min_samples_split = min_samples_split
		self.__max_features = {
			'auto': lambda x: int(np.sqrt(x))+1, 'sqrt': lambda x: int(np.sqrt(x))+1,
			'log2': lambda x: int(np.log2(x))+1, 'max_features': lambda x: x  }[max_features]
		self.__bootstrap = bootstrap
		self.__n_samples = None
		self.__n_features = None
		self.__trees = [  ]

	def __bootstrapX(self,X):
		indexes = np.random.choice( np.arange(0,len(X),1), size=self.__n_samples, replace=self.__bootstrap )
		return X[indexes,:]

	def __get_feature_index(self):
		return np.random.choice( np.arange(0,self.__n_features,1),
								size=self.__max_features(self.__n_features), replace=False)

	def fit(self,X,y):
		"""
		Fit the X and y to estimators

		Parameters
		----------

		X : numpy array, independent variables

		y : numpy array, target variable

		"""
		self.__n_samples, self.__n_features = X.shape
		X_y = np.c_[X,y]
		for _ in range(self.__n_estimators):
			dt = DecisionTreeRegressor( criterion=self.__criterion, max_depth=self.__max_depth,
										min_samples_split=self.__min_samples_split)
			data = self.__bootstrapX(X_y)
			features = self.__get_feature_index()
			dt.fit( data[:,features], data[:,-1] )
			self.__trees.append( (dt.tree_, features) )

	def predict(self,X):
		"""
		Predict values using all estimators

		Parameters
		----------
		X : numpy array, independent variables

		Returns
		-------
		predicted values

		"""
		predictions = np.zeros( (len(X)) )
		for root, features in self.__trees:
			predictions += np.array([ self.__predict_row(row,root) for row in X[:,features] ])
		return predictions/self.__n_estimators

	def __predict_row(self,row,node):
		if row[node['index']] < node['value']:
			if isinstance(node['left'], dict): return self.__predict_row(row,node['left'])
			else: return node['left']
		else:
			if isinstance(node['right'], dict): return self.__predict_row(row,node['right'])
			else: return node['right']

	def score(self,X,y):
		"""
		Computer Coefficient of Determination (rsquare)

		Parameters
		----------
		X : 2D numpy array, independent variables

		y : numpy array, dependent variables

		Returns
		-------
		r2 values

		"""
		y_pred = self.predict(X)
		return 1-( np.sum( (y-y_pred)**2 )/np.sum( (y-y.mean())**2 ) )

ext: py

reason: import numpy


text:
import pandas as pd
import numpy as np
np.random.seed(42)
import random
random.seed(42)

import pickle
import gzip
import glob
import os
import re
from copy import deepcopy
import json

from tqdm import tqdm
from pathlib import Path
import shutil
from urllib.parse import urlparse
import statistics
import seaborn as sns
import matplotlib.pyplot as plt
sns.set_theme()

import concurrent.futures
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor

from pdb import set_trace

## Calculates some statistics which are necessary for sorting by global attribute count in step 05

def process_domain(domain_path):
    domain_name = os.path.basename(domain_path)
    domain_name = domain_name[:-7]

    with gzip.open(domain_path, "rb") as f:
        domain_objects_dict = pickle.load(f)

    assert len(domain_objects_dict) != 0

    if len(domain_objects_dict) > 2:
        domain_tup = (domain_name, len(domain_objects_dict), True)
    else:
        domain_tup = (domain_name, len(domain_objects_dict), False)

    return domain_tup

def calc_stats(domains, name, main_class, output_path, step):

    object_per_domain_dict = {}
    number_domains = 0
    objects_overall = 0

    gt2_object_per_domain_dict = {}
    gt2_number_domains = 0
    gt2_objects_overall = 0

    arguments = list(domains)
    results = []

    with ThreadPoolExecutor() as executor:
        futures = []
        for domain in arguments:
            futures.append(executor.submit(process_domain, domain))

        for future in concurrent.futures.as_completed(futures):
            results.append(future.result())

    for result in results:
        is_gt2 = result[2]
        if is_gt2:
            gt2_object_per_domain_dict[result[0]] = result[1]
        object_per_domain_dict[result[0]] = result[1]

    objects_overall = sum([v for k,v in object_per_domain_dict.items()])
    gt2_objects_overall = sum([v for k,v in gt2_object_per_domain_dict.items()])

    number_domains= len(object_per_domain_dict)
    gt2_number_domains= len(gt2_object_per_domain_dict)

    object_per_domain_sorted = [(k,v) for k, v in sorted(object_per_domain_dict.items(), key=lambda item: item[1], reverse=True)]
    top100_domains = object_per_domain_sorted[:100]

    low = []
    rest = []

    for tup in object_per_domain_sorted[100:]:
        if tup[1] > 2:
            rest.append(tup)
        else:
            low.append(tup)

    top100_domains_dict = {k:v for (k, v) in top100_domains}
    top100_domains_series = pd.Series(top100_domains_dict)

    low_domains_dict = {k:v for (k, v) in low}
    low_domains_series = pd.Series(low_domains_dict)

    rest_domains_dict = {k:v for (k, v) in rest}
    rest_domains_series = pd.Series(rest_domains_dict)

    sorted_object_per_domain_list = [v for (k,v) in object_per_domain_sorted]

    gt2_sorted_object_per_domain_list = sorted([v for k, v in gt2_object_per_domain_dict.items()], reverse=True)

    median_objects_per_domain = statistics.median(sorted_object_per_domain_list)

    gt2_median_objects_per_domain = statistics.median(gt2_sorted_object_per_domain_list)

    with open(f'{output_path}{name}_{step}_statistics.txt', 'w') as f:
        f.write(f'Number of domains\t{number_domains}\nNumber of objects\t{objects_overall}\nMedian objects per domain\t{median_objects_per_domain}\n')

    with open(f'{output_path}gt2_{name}_{step}_statistics.txt', 'w') as f:
        f.write(f'Number of domains\t{gt2_number_domains}\nNumber of objects\t{gt2_objects_overall}\nMedian objects per domain\t{gt2_median_objects_per_domain}\n')

    sns.lineplot(x=list(range(0, len(sorted_object_per_domain_list))), y=sorted_object_per_domain_list)
    plt.xlabel("Domain ID")
    plt.ylabel("# of objects")
    plt.title("Objects per domain distribution")
    plt.savefig(f'{output_path}{name}_{step}_distribution.png')
    plt.clf()

    sns.lineplot(x=list(range(0, len(gt2_sorted_object_per_domain_list))), y=gt2_sorted_object_per_domain_list)
    plt.xlabel("Domain ID")
    plt.ylabel("# of objects")
    plt.title("Objects per domain distribution")
    plt.savefig(f'{output_path}gt2_{name}_{step}_distribution.png')
    plt.clf()

    top100_domains_series.to_json(f'{output_path}{name}_{step}_top100_domains.json.gz')
    if len(low_domains_series) != 0:
        low_domains_series.to_json(f'{output_path}{name}_{step}_low_domains.json.gz')
    if len(rest_domains_series) != 0:
        rest_domains_series.to_json(f'{output_path}{name}_{step}_rest_domains.json.gz')

with open('../../schemaorg_classes.json', 'r') as f:
    class_dict = json.load(f)

classes = [k for k in class_dict.keys()]

for name in tqdm(classes):
    for k in class_dict.keys():
        if k == name:
            main_class = class_dict[k]

    print(main_class)

    output_path = f"../../data/interim/tablecorpus/domain_statistics/{name}/"
    shutil.rmtree(output_path, ignore_errors=True)
    Path(output_path).mkdir(parents=True, exist_ok=True)

    domains_all = glob.glob(f'../../data/interim/tablecorpus/domain_objects_extracted/all/{name}/*.pkl.gz')
    domains_delist = glob.glob(f'../../data/interim/tablecorpus/domain_objects_extracted/delist/{name}/*.pkl.gz')
    domains_delist_dedup = glob.glob(f'../../data/interim/tablecorpus/domain_objects_extracted/delist-dedup/{name}/*.pkl.gz')

    calc_stats(domains_all, name, main_class, output_path, 'all')
    calc_stats(domains_delist, name, main_class, output_path, 'delist')
    calc_stats(domains_delist_dedup, name, main_class, output_path, 'delist_dedup')


text:
import numba as nb
import numpy as np
import scipy.stats

@nb.njit(parallel=True)
def tiecorrect(rankvals):
    """
    parallelized version of scipy.stats.tiecorrect

    :param rankvals: p x n array of ranked data (output of rankdata function)
    """
    tc = np.ones(rankvals.shape[1], dtype=np.float64)
    for j in nb.prange(rankvals.shape[1]):
        arr = np.sort(np.ravel(rankvals[:, j]))
        idx = np.nonzero(
            np.concatenate((np.array([True]), arr[1:] != arr[:-1], np.array([True])))
        )[0]
        t_k = np.diff(idx).astype(np.float64)

        size = np.float64(arr.size)
        if size >= 2:
            tc[j] = 1.0 - (t_k ** 3 - t_k).sum() / (size ** 3 - size)

    return tc

@nb.njit(parallel=True)
def rankdata(data):
    """
    parallelized version of scipy.stats.rankdata

    :param data: p x n array of data to rank, column-wise
    """
    ranked = np.empty(data.shape, dtype=np.float64)
    for j in nb.prange(data.shape[1]):
        arr = np.ravel(data[:, j])
        sorter = np.argsort(arr)

        arr = arr[sorter]
        obs = np.concatenate((np.array([True]), arr[1:] != arr[:-1]))

        dense = np.empty(obs.size, dtype=np.int64)
        dense[sorter] = obs.cumsum()

        # cumulative counts of each unique value
        count = np.concatenate((np.nonzero(obs)[0], np.array([len(obs)])))
        ranked[:, j] = 0.5 * (count[dense] + count[dense - 1] + 1)

    return ranked

def mannwhitneyu(x, y, use_continuity=True):
    """Version of Mann-Whitney U-test that runs in parallel on 2d arrays

    This is the two-sided test, asymptotic algo only. Returns log p-values
    """
    x = np.asarray(x)
    y = np.asarray(y)
    assert x.shape[1] == y.shape[1]

    n1 = x.shape[0]
    n2 = y.shape[0]

    ranked = rankdata(np.concatenate((x, y)))
    rankx = ranked[:n1, :]  # get the x-ranks
    u1 = n1 * n2 + (n1 * (n1 + 1)) / 2.0 - np.sum(rankx, axis=0)  # calc U for x
    u2 = n1 * n2 - u1  # remainder is U for y
    T = tiecorrect(ranked)

    # if *everything* is identical we'll raise an error, not otherwise
    if np.all(T == 0):
        raise ValueError("All numbers are identical in mannwhitneyu")
    sd = np.sqrt(T * n1 * n2 * (n1 + n2 + 1) / 12.0)

    meanrank = n1 * n2 / 2.0 + 0.5 * use_continuity
    bigu = np.maximum(u1, u2)

    with np.errstate(divide="ignore", invalid="ignore"):
        z = (bigu - meanrank) / sd

    logp = np.minimum(scipy.stats.norm.logsf(z) + np.log(2), 0)

    return u2, logp

max_stars_repo_name: MacoskoLab/brain-atlas

max_issues_repo_name: MacoskoLab/brain-atlas

max_forks_repo_name: MacoskoLab/brain-atlas

reason: import numpy,import scipy,import numba


text:
import sys
import os

if __name__ == "__main__":
    here = sys.path[0]
    sys.path.insert(0, os.path.join(here, '..', '..', 'sol'))

#============================ imports =========================================

import math
import influxdb
import Sol
import matplotlib.pyplot as plt
import numpy

#============================ defines =========================================

MAC_LONG_RANGE      = [ "00-17-0d-00-00-b0-00-aa",
                        "00-17-0d-00-00-b0-00-cc",
                        "00-17-0d-00-00-b0-00-87",
                    ]
MAC_MEDIUM_RANGE    = [ "00-17-0d-00-00-30-60-ef",
                        "00-17-0d-00-00-58-32-36"
                    ]

#============================ main    =========================================

def main():
        sol            = Sol.Sol()
        influxClient   = influxdb.client.InfluxDBClient(
                host            = 'localhost',
                port            = '8086',
                database        = 'realms'
                )

        # get motes ids
        query = "SELECT * FROM SOL_TYPE_DUST_EVENTMOTECREATE \
                WHERE site='ARG_junin'"
        influx_json = influxClient.query(query).raw
        res = sol.influxdb_to_json(influx_json)

        # populate motes list
        motes   = [None]*100
        for m in res:
                motes[m['value']['moteId']] = m
                motes[m['value']['moteId']]['neighbors'] = [None]*100

        # get motes neighbors
        query = "SELECT * FROM SOL_TYPE_DUST_NOTIF_HRNEIGHBORS \
                WHERE site='ARG_junin'"
        influx_json = influxClient.query(query).raw
        res = sol.influxdb_to_json(influx_json)

        # update motes location
        for r in res:
            moteId = _getMoteIdFromMac(motes, r['mac'])
            if moteId != -1:
                for n in r['value']['neighbors']:
                    nbrId = n['neighborId']
                    if motes[nbrId] != None:
                        motes[nbrId]['value']['latitude'] = r['value']['latitude']
                        motes[nbrId]['value']['longitude'] = r['value']['longitude']

        # compute distance
        fw = open('pdr_dist.out', 'w')
        for r in res:
            moteId = _getMoteIdFromMac(motes, r['mac'])
            if moteId != -1:
                for n in r['value']['neighbors']:
                    nbrId = n['neighborId']
                    if motes[nbrId] != None:
                        # compute distance with neighbor
                        dist    = _distance_on_unit_sphere(
                                    float(r['value']['latitude']),
                                    float(r['value']['longitude']),
                                    float(motes[nbrId]['value']['latitude']),
                                    float(motes[nbrId]['value']['longitude'])
                                )

                        # compute pdr with neighbor
                        if int(n['numTxPackets']) != 0:
                            pdr         = (int(n['numTxPackets'])-int(n['numTxFailures'])
                                        )/float(n['numTxPackets'])*100

                            # find mote device type
                            mote1_type   = 0
                            if r['mac'] in MAC_LONG_RANGE:
                                mote1_type = 2
                            if r['mac'] in MAC_MEDIUM_RANGE:
                                mote1_type = 1
                            mote2_type   = 0
                            if motes[nbrId]['value']['macAddress'] in MAC_LONG_RANGE:
                                mote2_type = 2
                            if motes[nbrId]['value']['macAddress'] in MAC_MEDIUM_RANGE:
                                mote2_type = 1

                            # write stat
                            fw.write(str(r['mac']) + \
                                    " " + str(motes[nbrId]['value']['macAddress']) + \
                                    " " + str(dist) + \
                                    " " + str(pdr) +
                                    " " + str(mote1_type) +
                                    " " + str(mote2_type) + "\n"
                                    )
        fw.close()

        # create graph
        plt.figure()
        with open('pdr_dist.out') as f:
            x_series    = [[[],[],[]],[[],[],[]],[[],[],[]]]
            y_series    = [[[],[],[]],[[],[],[]],[[],[],[]]]
            for line in f:
                line_list = line.split(' ')
                mote1_type  = int(line_list[4])
                mote2_type  = int(line_list[5])
                pdr         = float(line_list[2])
                dist        = float(line_list[3])

                x_series[mote1_type][mote2_type].append(pdr)
                y_series[mote1_type][mote2_type].append(dist)
        fw.close()
        plt.xlabel('Distance (m)')
        plt.ylabel('PDR')
        colors  = iter(['r', 'b', 'g', 'c', 'm', 'y', 'k', 'w', '0.50'])
        markers = iter(['^', '.', 'o', '*', 'p', 's', 'x', 'D', 'h'])
        mote_types=['DC9003', 'DC9018', 'LongRange']
        for i in range(0, len(x_series)):
            for j in range(0, len(y_series)):
                plt.plot(x_series[i][j],
                        y_series[i][j],
                        marker=next(markers),
                        linestyle='None',
                        color=next(colors),
                        label=mote_types[i]+' to '+mote_types[j]
                        )
        plt.legend()
        plt.show()

#============================ helpers =========================================

def _getMoteIdFromMac(motes, mac):
        for index, m in enumerate(motes):
                if m != None:
                        if m['value']['macAddress'] == mac:
                                return index
        return -1

def _distance_on_unit_sphere(lat1, long1, lat2, long2):
    # Convert latitude and longitude to
    # spherical coordinates in radians.
    degrees_to_radians = math.pi/180.0

    # phi = 90 - latitude
    phi1 = (90.0 - lat1)*degrees_to_radians
    phi2 = (90.0 - lat2)*degrees_to_radians

    # theta = longitude
    theta1 = long1*degrees_to_radians
    theta2 = long2*degrees_to_radians

    # Compute spherical distance from spherical coordinates.

    # For two locations in spherical coordinates
    # (1, theta, phi) and (1, theta', phi')
    # cosine( arc length ) =
    # sin phi sin phi' cos(theta-theta') + cos phi cos phi'
    # distance = rho * arc length

    cos = (math.sin(phi1)*math.sin(phi2)*math.cos(theta1 - theta2) +
        math.cos(phi1)*math.cos(phi2))
    arc = math.acos( cos )

    # Remember to multiply arc by the radius of the earth
    # in your favorite set of units to get length.
    return arc*6371*1000 # in meters

main()

max_stars_repo_name: realms-team/server-sw

max_issues_repo_name: realms-team/solserver

max_forks_repo_name: realms-team/server-sw


text:
""" A neural chatbot using sequence to sequence model with
attentional decoder.

This is based on Google Translate Tensorflow model
https://github.com/tensorflow/models/blob/master/tutorials/rnn/translate/

Sequence to sequence model by Cho et al.(2014)

Created by Chip Huyen as the starter code for assignment 3,
class CS 20SI: "TensorFlow for Deep Learning Research"
cs20si.stanford.edu

This file contains the code to run the model.

See readme.md for instruction on how to run the starter code.

This implementation learns NUMBER SORTING via seq2seq. Number range: 0,1,2,3,4,5,EOS

https://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf

See README.md to learn what this code has done!

Also SEE https://stackoverflow.com/questions/38241410/tensorflow-remember-lstm-state-for-next-batch-stateful-lstm
for special treatment for this code

Belief Tracker
"""

import numpy as np
import uuid

class Node:

    API_NODE = "category"
    NORMAL_NODE = "normal"
    PROPERTY_NODE = "property"

    RANGE = "range"
    KEY = "key"

    def __init__(self, slot, value, fields, node_type, id):
        '''
        For now only api_node has more than one field
        Args:
            slot: slot serves as edge
            fields: a map indicating fields that are required or not
            node_type: api_node, property_node, normal
        '''
        self.value = value  # value aka slot-value filling
        self.slot = slot  # slot aks slot value filling
        self.ins = set([slot])
        self.parent_node = []
        self.fields = fields  # is a dict, valued as prob.
        self.outs = set(fields.keys())
        self.field_type = dict() # RANGE, KEY,...
        self.children = dict()  # value to chidren nodes
        self.node_type = node_type
        self.slot_to_values_mapper = dict()
        self.value_to_slot_mapper = dict()
        self.fields_trans = dict()
        self.id = id  # globally uniq
        self.level = 0

    def get_node_by_value(self, value):
        return self.children[value]

    def set_node_slot_trans(self, field, trans):
        if field not in self.fields:
            raise ValueError(
                'translation field error: node must have field which is included in self.fields.'
                ' Fix the graph file before continue;', field, self.fields)
        self.fields_trans[field] = trans

    def get_node_slot_trans(self, field):
        if field in self.fields_trans:
            return self.fields_trans[field]
        return None

    def set_field_type(self, field, field_type):
        self.field_type[field] = field_type

    def get_field_type(self, field):
        if field not in self.field_type:
            return Node.KEY
        return self.field_type[field]

    def has_field(self, field):
        return field in self.fields

    def add_node(self, node, edge=None, ignore_field_conflict=False):
        """
        Args:
            node: child_node to append to the current node, there will be a lot other information brought in
            ignore_field_conflict:
            edge: str
        Raises:
            ValueError: child_node slot must be in self.fields.
        """
        node_value = node.value
        node_field = node.slot
        if not edge:
            edge = node.slot
        if node_field not in self.fields and not ignore_field_conflict:
            raise ValueError(
                'error: node must have field which is included in self.fields.'
                ' Fix the graph file before continue;', node_field, self.fields)
        # self.outs.add(node_field)
        self.children[node_value] = node
        node.ins.add(edge)
        node.level = self.level + 1
        node.parent_node.append(self)

        # supply other information
        if node_field not in self.slot_to_values_mapper:
            self.slot_to_values_mapper[node_field] = []
        self.slot_to_values_mapper[node_field].append(node_value)

        if node_value not in self.value_to_slot_mapper:
            self.value_to_slot_mapper[node_value] = node_field

    def has_ancestor_by_value(self, value):
        for node in self.parent_node:
            # print(self.value, '***', node.value, '---', value)
            if node.value == value:
                return True
            if node.has_ancestor_by_value(value):
                return True
        return False

    def is_api_node(self):
        return self.node_type == self.API_NODE

    def is_property_node(self):
        return self.node_type == self.PROPERTY_NODE

    def get_slot_by_value(self, value):
        if value not in self.value_to_slot_mapper:
            return None
        return self.value_to_slot_mapper[value]

    def gen_required_slot_fields(self):
        required_fields = []
        for field, prob in self.fields.items():
            rnd = np.random.uniform(0, 1)
            if rnd < prob:
                required_fields.append(field)
        return required_fields

    def has_child(self, value):
        """
        has direct child
        """
        return value in self.children

    def get_child(self, value):
        return self.children[value]

    def has_posterity(self, value, belief_graph):
        """
        We use belief_graph to achieve faster bottom to up search
        """
        candidate_nodes = belief_graph.get_nodes_by_value(value)
        if self.value == "ROOT".lower():
            return len(candidate_nodes) > 0

        for node in candidate_nodes:
            id = node.id
            a = node
            while a.value != "ROOT".lower():
                if a.value == self.value:
                    return True
                a = a.parent_node
        return False

    def get_posterity_nodes_by_value(self, value, belief_graph):
        candidate_nodes = belief_graph.get_nodes_by_value(value)
        if self.value == "ROOT".lower():
            return candidate_nodes

        posterity = []
        for node in candidate_nodes:
            id = node.id
            a = node
            while a.value != "ROOT".lower():
                if a.value == self.value:
                    posterity.append(belief_graph.get_node_by_id(id))
                    break
                a = a.parent_node
        return posterity

    # def get_siblings(self, keep_slot):
    #     """
    #     This function might be dangerous to be used
    #     Args:
    #         keep_slot: True or False, return only siblings with same slot or not
    #     """
    #     if not self.parent_node:
    #         return []
    #
    #     siblings = []
    #     children = self.parent_node.chidren
    #     for key, value in children.items():
    #         if key != self.value:
    #             sibling = value
    #             if sibling.slot == self.slot or keep_slot == False:
    #                 siblings.append(sibling)
    #     return siblings

    def get_ancestry_values(self):
        anc_values = set()
        for node in self.parent_node:
            while node.value != "ROOT".lower():
                anc_values.add(node.value)
                anc_values.add(node.get_ancestry_values())
        return anc_values

    # def get_sibling_names(self, value, keep_slot):
    #     siblings = self.get_siblings(keep_slot)
    #     sibling_names = []
    #     for node in siblings:
    #         sibling_names.append(node.value)
    #     return sibling_names

    # def get_parent_node(self):
    #     return self.parent_node

    def get_children_names_by_slot(self, slot):
        children_names = []
        for key, value in self.children.items():
            if self.value_to_slot_mapper[key] == slot:
                children_names.append(key)
        return children_names

    def get_children_names(self, max_num=-1, required_only=False):
        children_names = []
        for key, value in self.children.items():
            if required_only == False or self.fields[key] == 1:
                children_names.append(key)
        children_names = np.array(children_names)
        if max_num == -1:
            return children_names
        children_names = np.random.choice(children_names, max_num)
        return children_names

    def is_leaf(self):
        return len(self.children) == 0

max_stars_repo_name: aquadrop/memory

max_issues_repo_name: aquadrop/memory

max_forks_repo_name: aquadrop/memory


text:
# -*- coding: utf-8 -*-
# @Time    : 2019/10/24 20:12
# @Author  : Esbiya
# @Email   : 18829040039@163.com
# @File    : img_locate.py
# @Software: PyCharm

import os
import cv2
import numpy as np
import requests
from PIL import Image
from eastmoney.chaojiying import image_to_text

def _pic_download(url, type):
    """
    图片下载
    :param url:
    :param type:
    :return:
    """
    save_path = os.path.abspath('...') + '\\' + 'images'
    if not os.path.exists(save_path):
        os.mkdir(save_path)
    img_path = save_path + '\\' + '{}.jpg'.format(type)
    img_data = requests.get(url).content
    with open(img_path, 'wb') as f:
        f.write(img_data)
    return img_path

def merge_img(url, type):
    """
    还原图片
    :param url: 乱序验证码 url
    :param type: 类型
    :return:
    """
    merge_array = [{"left": 157, "top": 0}, {"left": 145, "top": 0}, {"left": 265, "top": 0}, {"left": 277, "top": 0},
                   {"left": 181, "top": 0}, {"left": 169, "top": 0}, {"left": 241, "top": 0}, {"left": 253, "top": 0},
                   {"left": 109, "top": 0}, {"left": 97, "top": 0}, {"left": 289, "top": 0}, {"left": 301, "top": 0},
                   {"left": 85, "top": 0}, {"left": 73, "top": 0}, {"left": 25, "top": 0}, {"left": 37, "top": 0},
                   {"left": 13, "top": 0}, {"left": 1, "top": 0}, {"left": 121, "top": 0}, {"left": 133, "top": 0},
                   {"left": 61, "top": 0}, {"left": 49, "top": 0}, {"left": 217, "top": 0}, {"left": 229, "top": 0},
                   {"left": 205, "top": 0}, {"left": 193, "top": 0}, {"left": 145, "top": 80}, {"left": 157, "top": 80},
                   {"left": 277, "top": 80}, {"left": 265, "top": 80}, {"left": 169, "top": 80},
                   {"left": 181, "top": 80}, {"left": 253, "top": 80}, {"left": 241, "top": 80},
                   {"left": 97, "top": 80}, {"left": 109, "top": 80}, {"left": 301, "top": 80},
                   {"left": 289, "top": 80}, {"left": 73, "top": 80}, {"left": 85, "top": 80}, {"left": 37, "top": 80},
                   {"left": 25, "top": 80}, {"left": 1, "top": 80}, {"left": 13, "top": 80}, {"left": 133, "top": 80},
                   {"left": 121, "top": 80}, {"left": 49, "top": 80}, {"left": 61, "top": 80}, {"left": 229, "top": 80},
                   {"left": 217, "top": 80}, {"left": 193, "top": 80}, {"left": 205, "top": 80}]

    captcha_path = _pic_download(url, type)
    captcha = Image.open(captcha_path)
    new_captcha = Image.new('RGB', (260, 160))

    upper_list = merge_array[:26]
    lower_list = merge_array[26:]

    for index, location in enumerate(upper_list):
        imgcrop = captcha.crop((location['left'], location['top'], location['left'] + 10, location['top'] + 80))
        new_captcha.paste(imgcrop, (index * 10, 0))

    for index, location in enumerate(lower_list):
        imgcrop = captcha.crop((location['left'], location['top'], location['left'] + 10, location['top'] + 80))
        new_captcha.paste(imgcrop, (index * 10, 80))

    new_captcha.save(captcha_path)
    return captcha_path

def _cut_slider(path):
    """
    滑块切割
    :return:
    """
    save_path = os.path.abspath('...') + '\\' + 'images'
    if not os.path.exists(save_path):
        os.mkdir(save_path)

    image = Image.open(path)
    x = []
    y = []
    for i in range(image.size[0]):
        for j in range(image.size[1]):
            pix = image.load()[i, j]
            if pix != 255:
                x.append(i)
                y.append(j)
    z = (np.min(x), np.min(y), np.max(x), np.max(y))
    result = image.crop(z)
    result.convert('RGB').save(save_path + '\\' + 'temp.jpg')
    # result.show()
    return result.size[0], result.size[1]

def get_distance(captcha_url, slider_url):
    """
    获取滑块缺口距离
    :param captcha_url:
    :param slider_url:
    :return:
    """
    save_path = os.path.abspath('...') + '\\' + 'images'
    if not os.path.exists(save_path):
        os.mkdir(save_path)

    # 完整图片还原
    captcha_path = merge_img(captcha_url, 'slide_full')
    # 缺口图片还原
    slider_path = _pic_download(slider_url, 'slide_gap')
    # 滑块切割
    w, h = _cut_slider(slider_path)

    # # 计算拼图还原距离
    target = cv2.imread(slider_path, 0)
    template = cv2.imread(captcha_path, 0)
    temp = save_path + '\\' + 'temp.jpg'
    targ = save_path + '\\' + 'targ.jpg'
    cv2.imwrite(targ, target)
    cv2.imwrite(temp, template)

    target = cv2.imread(targ)
    template = cv2.imread(temp)
    result = cv2.matchTemplate(target, template, cv2.TM_CCOEFF_NORMED)
    x, y = np.unravel_index(result.argmax(), result.shape)

    # 调用PIL Image 做测试
    image = Image.open(captcha_path)

    xy = (y + 5, x, y + w, x + h)
    # 切割
    imagecrop = image.crop(xy)
    # 保存切割的缺口
    imagecrop.convert('RGB').save(save_path + '\\' + "new_image.jpg")
    imagecrop.show()
    return int(round(y)) + 2

def get_position(full, words):
    """
    获取点选验证位置
    :param full:
    :param words:
    :return:
    """
    # 还原点选原图
    merge_img(full, 'click_full')
    full_path = os.path.abspath('...') + '\\' + 'images' + '\\' + 'click_full.jpg'
    full_bg = Image.open(full_path)

    # 下载点选文字图片
    words_path = _pic_download(words, 'click_words')
    text_bg = Image.open(words_path)

    # 合并
    click_img = Image.new('RGB', (260, 190))
    click_img.paste(full_bg, (0, 0))
    click_img.paste(text_bg, (0, 160))

    click_img.show()
    click_path = os.path.abspath('...') + '\\' + 'images' + '\\' + 'click_merge.jpg'
    click_img.save(click_path)

    with open(click_path, 'rb') as f:
        img_data = f.read()
    # 超级鹰识别
    ok, position = image_to_text(img_data, img_kind=9004)
    if ok:
        return position
    return None

if __name__ == '__main__':
    # position = get_position('https://captcha2.eastmoney.com/14/resources/a8af_160/3/83/83565224e1e1b9e992d0feee4d0dea43.jpg',
    #                         'https://captcha2.eastmoney.com/14/resources/a8af_160/3/83/83565224e1e1b9e992d0feee4d0dea43_words.jpg')
    # print(position)

    distance = get_distance('https://captcha2.eastmoney.com/13/resources/e02b_160/3/73/73bc88b9cb453cb71713c4694137c5f2/bg/473a0b2e.jpg',
                            'https://captcha2.eastmoney.com/13/resources/e02b_160/3/73/73bc88b9cb453cb71713c4694137c5f2/slice/473a0b2e.png')
    print(distance)
    # _cut_slider(os.path.abspath('...') + '\\' + 'images' + '\\' + 'slide_gap.jpg')


text:
#! python
"""
Tutorial 1 - Pandas Library
"""

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import plotly.graph_objects as go

def get_data(filename, field_x, field_y, field_z, field_s, field_c):
    """ Import data into an array """
    data = pd.read_csv(filename)
    print(data.head())
    sub_data = data[[field_y, field_x, field_z, field_c]]
    # sub_data.set_index(field_x, inplace=True)
    print(sub_data.head())
    sub_data.plot(linestyle=None, marker='o')
    # plt.show()

    return d_scatter(sub_data, field_x, field_y, field_z, field_s, field_c)

def d_scatter(data, field_x, field_y, field_z, field_s, field_c):
    """ creates a 3d plot using matplotlib """
    x = data[field_x]
    y = data[field_y]
    z = data[field_z]
    c = data[field_c]

    fig = go.Figure(data=[go.Scatter3d(x=x, y=y, z=z, mode='markers', marker=dict(size=8, color=c,
                                       colorscale='viridis', opacity=0.8) )])
    return fig.show()

if __name__ == "__main__":
    """dimuon dataset: http://opendata.cern.ch/record/303 """
    filepath = "../Datasets/Zee.csv"
    field_z = "py1"
    field_y = "pz1"
    field_x = "pt1"
    field_s = "E1"
    field_c = "M"
    get_data(str(filepath), field_x, field_y, field_z, field_s, field_c)

max_stars_repo_name: adamdadd/DatSci

max_issues_repo_name: adamdadd/DatSci

max_forks_repo_name: adamdadd/DatSci


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 28 09:54:19 2015

@author: ibackus
"""
# External imports
from matplotlib.colors import LogNorm, Normalize
from matplotlib.cm import get_cmap
import numpy as np
import pynbody as pb
SimArray = pb.array.SimArray

# Internal imports
import cubehelix
import pbmov_utils

# setup colormaps
ch=cubehelix.cmap()
cx4 = cubehelix.cmap(reverse=False, start=0., rot=0.5)  #mostly reds
cx3 = cubehelix.cmap(reverse=False, start=0.3, rot=-0.5)# mostly blues
cx_default = ch

def renderFrame(frameSettings, params, sim, pos0=None, preview=False):
    """
    Renders a frame of a movie and returns it as an RGB array.  Optionally,
    if preview is True, it will plot the image instead of returning it.

    NOTE: when not previewing, the sim['pos'] will be altered by the view space
    transformation.

    Parameters
    ----------

    frameSettings : dict
        A dictionary containing settings/parameters pertinent only to the
        current frame of the movie (see movieSettings.getFrame)
    params : dict
        A dictionary containing settings for the whole movie
    sim : snapshot
        pynbody snapshot to render.  IF a family-level quantity is being plotted,
        e.g. 'rho', then just that family should be given.  ie:
            snapshot = pynbody.load('filename')
            sim = snapshot.gas
    pos0 : array
        Initial particle positions.  useful if the particle positions in
        sim were altered by a previously rendered frame
    preview : bool
        If True, the image is plotted and nothing is returned
        If False (default) the image is rendered and returned without plotting

    Returns
    -------

    im : array
        If preview=False, an RGB array of the image is returned.  Otherwise,
        None is returned
    """
    # Load settings for this frame
    cam = frameSettings['cam']
    target = frameSettings['target']
    camera_rot = frameSettings['camOrient']
    vmax = frameSettings['vmax']
    vmin = frameSettings['vmin']
    simRot = frameSettings['simRot']
    simRotAxis = np.zeros(3)
    simRotAxis[0:2] = simRot[0:2]
    simRotAngle = simRot[2]

    # Load movie-wide (all frames) settings
    cmap = params['cmap']
    kwargs = params['pbkwargs']
    res = params['res']
    if kwargs is None:

        kwargs = {}

    # Get positions
    if preview:

        original_pos = sim['pos'].copy()

    if pos0 is None:

        pos0 = sim['pos']

    # Default target
    if target is None:

        target = np.zeros(3)

    # Get frame width (physical size)
    width = pbmov_utils.frame_width(cam, target)
    d = np.sqrt( ((cam - target)**2).sum())
    # Perform view space transformation on position
    pos = pbmov_utils.vsm_transform(pos0, cam, target, camera_rot*(np.pi/180))
    # perform an additional rotation of the simulation
    R = pbmov_utils.rotation_matrix(simRotAxis, simRotAngle*(np.pi/180),
                                    fourD=False)
    pos = np.dot(R, pos.T).T
    sim['pos'] = pos

    if preview:

        # Plot
        pb.plot.sph.image(sim.g, width=width, z_camera=d, noplot=False, \
        resolution=res, vmin=vmin, vmax=vmax, cmap=cmap, **kwargs);

        # Revert simulation position back to original
        sim['pos'] = original_pos
        color_im = None

    else:

        # Render image in pynbody
        im = pb.plot.sph.image(sim.g, width=width, z_camera=d, noplot=True, \
        resolution=res, **kwargs)
        # Flip along y axis
        im = np.flipud(im)
        # Now make a color, RGB image
        log = kwargs.get('log', True)
        color_im = rgbify(im, vmin, vmax, cmap, log)
        return color_im

def rgbify(im, vmin=None, vmax=None, cmap=cx_default, log=True):
    """
    Converts an image made by pynbody.plot.sph.image to an RGB array of ints,
    dtype uint8.  A logarithmic image will be produced

    **ARGUMENTS**

    im : SimArray or numpy array
        2D image array.
    vmin, vmax : float or SimArray
        Normalization limits.  Values outside these bounds will be mapped to
        0, 1.

    **RETURNS**

    color_im_array : numpy array, dtype uint8
        Logarithmic, normalized RGB image of integers
    """
    if isinstance(cmap, str):

        cmap = get_cmap(cmap)

    im = im.copy()
    im[im==0] = np.nan

    # Set up the min/max values
    if vmin is not None:

        # Filter min values
        im[im < vmin] = vmin

    else:

        vmin = np.nanmin(im)

    if vmax is not None:

        # Filter max values
        im[im > vmax] = vmax

    else:

        vmax = np.nanmax(im)

    # Assume all nans were zeros originally and replace them
    im[np.isnan(im)] = vmin

    # Run a log normalization
    if log:
        a = LogNorm(vmin=vmin, vmax=vmax)
    else:
        a = Normalize(vmin=vmin, vmax=vmax)
    try:
        im = a(im)
    except ValueError:
        print vmin, vmax
        raise

    # Apply color map
    color_im = cmap(im)

    # Convert to 0-255 integer values
    color_im_int = (255*color_im[:,:,0:-1] + 0.5).astype(np.uint8)

    return color_im_int

max_stars_repo_name: ibackus/pbmov

max_issues_repo_name: ibackus/pbmov

max_forks_repo_name: ibackus/pbmov


text:
""" Script to visualize a realization """

import json
import os
from typing import Any, Dict

import imageio
import geopandas as gpd
import matplotlib.pyplot as plt
import numpy as np
from shapely.geometry import Polygon

def dist_between_2_states(state1: Dict, state2: Dict) -> float:
    """
    Return the distance between 2 states
    """
    dx = state1["x_meters"] - state2["x_meters"]
    dy = state1["y_meters"] - state2["y_meters"]

    return np.sqrt(dx ** 2 + dy ** 2)

def visualize_realization(
    traj_path: os.path,
    map_path: os.path,
    save_dir: os.path,
    dist_threshold_from_ego: float = 30,
    dt_refresh_fig: float = 0.25,
) -> None:
    """
    A demo function showing how to use the plotting functions
    :param traj_path: Path of trajectory to update.
    :param map_path: Path to map file
    :param save_dir: Where to save the plots.
    :param dist_threshold_from_ego: If an agent's distance to ego is greater than this threshold,
        then the agent is not plotted. Units: Meters.
    :param dt_refresh_fig: how often to refresh the figure and create a new one. Units: seconds.
    """
    with open(traj_path, "r") as j_file:
        traj_states = json.load(j_file)
    map_df = gpd.read_file(map_path)

    save_dir = os.path.join(save_dir, os.path.basename(traj_path.split(".")[0]))
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    # Plotting parameters
    ego_color = "black"
    ped_color, veh_color = "red", "blue"

    n_states = len(traj_states)
    ind = 0
    curr_ego = traj_states[0]
    figs = []  # keep track of figs to make into a movie
    fig = None
    # keep track of which objects have been plotted: want to plot each obj at most once per fig
    plotted_obj_ids = set()
    # make greater than dt to trigger condition below in while loop
    time_elapsed_since_fig_created = 2 * dt_refresh_fig
    # from above while loop we are guarranteed that curr state is ego
    while ind < n_states:
        state = traj_states[ind]
        time_elapsed = state["timestamp"] / 1e6  # in seconds

        if time_elapsed_since_fig_created > dt_refresh_fig:
            if fig is not None:
                plt.savefig(os.path.join(save_dir, f"{ind}.png"))
                plt.close(fig)

            # create a new figure
            fig = plt.figure()
            ax = fig.add_subplot(111)
            plt.axis("off")

            map_df.plot(ax=ax)

            # update relevant variables
            # index of object when figure created
            ind_obj_fig_created = ind
            time_elapsed_since_fig_created = 0
            plotted_obj_ids = set()

            ax.set_title(f"Time elapsed: {time_elapsed: .2f} seconds")

            # center around ego
            fig_half_range = 0.7 * dist_threshold_from_ego
            ego_x, ego_y = curr_ego["x_meters"], curr_ego["y_meters"]
            xlimits = [ego_x - fig_half_range, ego_x + fig_half_range]
            ylimits = [ego_y - fig_half_range, ego_y + fig_half_range]
            ax.set_xlim(xlimits), ax.set_ylim(ylimits)

            ax.set_aspect("equal")

        if state["type"] == "vehicle":
            color = veh_color
        elif state["type"] == "pedestrian":
            color = ped_color
        elif state["type"] == "ego":
            curr_ego = state
            color = ego_color

        if dist_between_2_states(curr_ego, state) > dist_threshold_from_ego:
            ind += 1
            continue

        if state["id"] not in plotted_obj_ids:
            plotted_obj_ids.add(state["id"])
            footprint = Polygon(state["footprint"])
            ax.plot(*footprint.exterior.xy, color=color)

        ind += 1

        # update time since we created a figure
        time_elapsed_since_fig_created = (
            state["timestamp"] - traj_states[ind_obj_fig_created]["timestamp"]
        ) / 1e6

    plt.close("all")

    # convert to video
    images = []
    sorted_filenames = []
    for filename in os.listdir(save_dir):
        sorted_filenames.append(int(filename.split(".")[0]))
    sorted_filenames.sort()
    sorted_filenames = [os.path.join(save_dir, f"{f}.png") for f in sorted_filenames]
    for filename in sorted_filenames:
        images.append(imageio.imread(os.path.join(save_dir, filename)))
    imageio.mimsave(os.path.join(save_dir, "movie.mp4"), images, fps=1 / dt_refresh_fig)

max_stars_repo_name: bassam-motional/Reasonable-Crowd

max_issues_repo_name: bassam-motional/Reasonable-Crowd

max_forks_repo_name: bassam-motional/Reasonable-Crowd


text:
from neural_networks.aliaser import Sequence, Tokenizer, to_categorical
import numpy as np
from dataset_loader.dataset_helper import Dataset_Helper

class TextGenerator(Sequence):
    def __init__(self, filename, batch_size, num_of_texts, num_of_words, tokenizer: Tokenizer, delimeter,
                 dataset_helper:Dataset_Helper, max_len=None, start_point=0, preprocess=False,preload_dataset=True, is_predicting=False, tokenizer_mode='binary'):
        self.filename = filename
        self.batch_size = batch_size
        self.num_of_texts = num_of_texts
        self.tokenizer = tokenizer
        self.delimeter = delimeter
        self.num_of_words = num_of_words
        self.num_of_classes = dataset_helper.get_num_of_topics() #num_of_classes
        self.start_point = start_point
        self.max_len = max_len
        self.preprocess = preprocess
        self.preload_dataset = preload_dataset
        self.is_predicting = is_predicting
        self.dataset_helper = dataset_helper
        self.tokenizer_mode = tokenizer_mode
        self.labels = []
        self.tmp_articles = None
        self.articles = []
        if preload_dataset:
            self.load_dataset()

    def __len__(self):
        return np.ceil(self.num_of_texts / self.batch_size).astype(np.int)

    def load_dataset(self):
        """with open(self.filename, encoding='utf-8', errors='ignore') as csvfile:
            for row in csv.reader(csvfile, delimiter=self.delimeter):
                self.articles.append([int(row[0]),preprocess_sentence(row[1]) if self.preprocess else row[1]])"""
        self.articles = self.dataset_helper.get_dataset(path=self.filename)

    def get_dataset(self):
        self.tmp_articles = np.array(self.articles)
        if self.is_predicting:
            self.labels.extend(list(map(int,self.tmp_articles[:,0])))

    def get_labels(self):
        if self.dataset_helper.vectorized_labels:
            return np.array(self.tmp_articles[:,0].tolist(),dtype=np.uint8)
        else:
            return to_categorical(self.tmp_articles[:,0], num_classes=self.num_of_classes, dtype=np.uint8)

    def get_basic_tokenizer_matrix(self):
        return self.tokenizer.texts_to_matrix(self.tmp_articles[:, 1], mode=self.tokenizer_mode)

    def __getitem__(self, item):
        self.tmp_articles = []
        if self.preload_dataset:
            self.tmp_articles = self.articles[self.start_point+item*self.batch_size:self.start_point+(item+1)*self.batch_size]
        else:
            """with open(self.filename, encoding='utf-8', errors='ignore') as csvfile:
                for row in islice(csv.reader(csvfile, delimiter=self.delimeter), self.start_point+item*self.batch_size,None):
                    self.tmp_articles.append([int(row[0]),preprocess_sentence(row[1])])
                    if len(self.tmp_articles) >= self.batch_size:
                        break"""
            self.tmp_articles = self.dataset_helper.get_dataset_slice(self.start_point+item*self.batch_size,self.batch_size,path=self.filename)

        self.tmp_articles = np.array(self.tmp_articles)
        if len(self.tmp_articles.shape) < 2:
            print("Working around...")
            print(self.start_point)
            self.tmp_articles = np.array([[0, "fgdssdgdsfgdsfgdsfg"]])
        if self.is_predicting:
            self.labels.extend(list(map(int,self.tmp_articles[:,0])))

max_stars_repo_name: MarekUlip/TextAnalysis

max_issues_repo_name: MarekUlip/TextAnalysis

max_forks_repo_name: MarekUlip/TextAnalysis


text:
import numpy as np
import nltk
import sklearn
import tokenizers
import multiprocessing
import itertools
import functools

def get_dataset():
    files = ['./analysis/input/negative_tweets.txt', './analysis/input/neutral_tweets.txt', './analysis/input/positive_tweets.txt']

    x = []
    for file in files:
        s = []
        with open(file, 'r') as f:
            for line in f:
                s.append(line.strip())

        assert len(s) == 1367
        x.extend(s)

    y = np.array([-1] * 1367 + [0] * 1367 + [1] * 1367)
    return x, y

def test_tokenizer(X, y, tokenizer, train_size, k_best):
    ss = sklearn.model_selection.ShuffleSplit(n_splits=10, train_size=(train_size / 100), test_size=None,
                                              random_state=42)
    total_train_error = 0.0
    total_test_error = 0.0
    total_f1 = 0.0
    runs = 0
    for train, test in ss.split(X, y):
        X_train = np.array(X)[train]
        y_train = y[train]

        X_test = np.array(X)[test]
        y_test = y[test]

        vect = sklearn.feature_extraction.text.CountVectorizer(tokenizer=tokenizer)
        X_train_counts = vect.fit_transform(X_train)
        tf_transformer = sklearn.feature_extraction.text.TfidfTransformer(use_idf=False).fit(X_train_counts)
        X_train_tfidf = tf_transformer.transform(X_train_counts)

        max_k = X_train_tfidf.shape[1]
        if k_best != 'all' and k_best > max_k:
            k_best = 'all'

        ch2 = sklearn.feature_selection.SelectKBest(sklearn.feature_selection.mutual_info_classif, k=k_best)
        X_train_ch2 = ch2.fit_transform(X_train_tfidf, y_train)

        clf = sklearn.linear_model.LogisticRegression().fit(X_train_ch2, y_train)

        predicted = clf.predict(X_train_ch2)
        train_error = 1 - sklearn.metrics.accuracy_score(y_train, predicted)
        total_train_error += train_error

        X_test_counts = vect.transform(X_test)
        X_test_tfidf = tf_transformer.transform(X_test_counts)
        X_test_ch2 = ch2.transform(X_test_tfidf)
        predicted = clf.predict(X_test_ch2)

        test_error = 1 - sklearn.metrics.accuracy_score(y_test, predicted)
        total_test_error += test_error

        total_f1 += sklearn.metrics.f1_score(y_test, predicted, average='macro')
        runs += 1

    return total_train_error / runs, total_test_error / runs, total_f1 / runs

class WhitespaceTokenizer(object):
    def __init__(self):
        pass

    def __call__(self, doc):
        return doc.split(' ')

class TreebankTokenizer(object):
    def __init__(self):
        self.treebank_word_tokenize = nltk.tokenize.treebank.TreebankWordTokenizer().tokenize

    def __call__(self, doc):
        return self.treebank_word_tokenize(doc)

class SentimentTokenizer(object):
    def __init__(self):
        self.sentiment_aware_tokenize = tokenizers.happy_tokenizer.Tokenizer().tokenize

    def __call__(self, doc):
        return self.sentiment_aware_tokenize(doc)

various_tokenizers = {
    # 'Whitespace': WhitespaceTokenizer(),
    'Treebank-style': TreebankTokenizer(),
    'Sentiment-aware': SentimentTokenizer()
}
train_sizes = list(range(60, 100, 10))
k_sizes = list(range(100, 10000, 200))
X, y = get_dataset()

TOKENIZER_F1_FILE = 'analysis/output/tokenizer_f1.csv'
TOKENIZER_ACC_FILE = 'analysis/output/tokenizer_accuracy.csv'

with open(TOKENIZER_F1_FILE, 'w') as f:
    f.writelines('tokenizer, train_size, k, f1\n')

with open(TOKENIZER_ACC_FILE, 'w') as f:
    f.writelines('tokenizer, train_size, k, train_error, test_error\n')

def train_and_output(X, y, tokenizer, train_size, k_best):
    tokenizer_name = tokenizer.__class__.__name__
    print('tokenizer={}, train_size={}, k_best={}'.format(tokenizer_name, train_size, k_best))
    average_train_error, average_test_error, average_f1 = test_tokenizer(X, y, tokenizer, train_size, k_best)
    with open('analysis/output/tokenizer_accuracy.csv', 'a') as acc_file:
        acc_file.write('{}, {}, {}, {:.3f}, {:.3f}\n'.format(tokenizer_name, train_size, k_best, average_train_error, average_test_error))
        acc_file.flush()
    with open('analysis/output/tokenizer_f1.csv', 'a') as f1_file:
        f1_file.write('{}, {}, {}, {:.3f}\n'.format(tokenizer_name, train_size, k_best, average_f1))
        f1_file.flush()

combi = itertools.product(various_tokenizers.values(), train_sizes, k_sizes)
with multiprocessing.Pool() as pool:
    p = pool.starmap(functools.partial(train_and_output, X, y), combi)

exit(0)

class SkipgramSentimentTokenizer(object):
    def __init__(self, n, k, negate=False):
        self.sentiment_aware_tokenize = tokenizers.happy_tokenizer.Tokenizer().tokenize
        self.n = n
        self.k = k
        self.negate = negate

    def __call__(self, doc):
        tokens = list(self.sentiment_aware_tokenize(doc))

        if self.negate:
            tokens = nltk.sentiment.util.mark_negation(tokens)

        if self.n == 1:
            return tokens

        skipgrams = list(nltk.skipgrams(tokens, self.n, self.k))
        return list([' '.join(s) for s in skipgrams])

features_extraction = {
    'Unigram': SkipgramSentimentTokenizer(1, 0),
    'Bigram': SkipgramSentimentTokenizer(2, 0),
    'Trigram': SkipgramSentimentTokenizer(3, 0),
    'Bigram with 1 skip': SkipgramSentimentTokenizer(2, 1),
    'Bigram with 2 skip': SkipgramSentimentTokenizer(2, 2),
    'Bigram with 3 skip': SkipgramSentimentTokenizer(2, 3),
    'Bigram with 4 skip': SkipgramSentimentTokenizer(2, 4),
    'Trigram with 1 skip': SkipgramSentimentTokenizer(3, 1),
    'Trigram with 2 skip': SkipgramSentimentTokenizer(3, 2),
    'Trigram with 3 skip': SkipgramSentimentTokenizer(3, 3),
    'Trigram with 4 skip': SkipgramSentimentTokenizer(3, 4),
    'Unigram (with negation)': SkipgramSentimentTokenizer(1, 0, negate=True),
    'Bigram (with negation)': SkipgramSentimentTokenizer(2, 0, negate=True),
    'Trigram (with negation)': SkipgramSentimentTokenizer(3, 0, negate=True),
    'Bigram with 1 skip (with negation)': SkipgramSentimentTokenizer(2, 1, negate=True),
    'Bigram with 2 skip (with negation)': SkipgramSentimentTokenizer(2, 2, negate=True),
    'Bigram with 3 skip (with negation)': SkipgramSentimentTokenizer(2, 3, negate=True),
    'Bigram with 4 skip (with negation)': SkipgramSentimentTokenizer(2, 4, negate=True),
    'Trigram with 1 skip (with negation)': SkipgramSentimentTokenizer(3, 1, negate=True),
    'Trigram with 2 skip (with negation)': SkipgramSentimentTokenizer(3, 2, negate=True),
    'Trigram with 3 skip (with negation)': SkipgramSentimentTokenizer(3, 3, negate=True),
    'Trigram with 4 skip (with negation)': SkipgramSentimentTokenizer(3, 4, negate=True),
}

features_acc_csv = open('analysis/output/features_accuracy.csv', 'w')
features_f1_csv = open('analysis/output/features_f1.csv', 'w')

features_acc_csv.writelines('features, train_size, train_error, test_error\n')
features_f1_csv.writelines('features, train_size, f1\n')

for keys in features_extraction.keys():
    print(keys)
    tok = features_extraction[keys]

    for size in train_sizes:
        average_train_error, average_test_error, average_f1 = test_tokenizer(X, y, tok, size)
        features_acc_csv.write('{}, {}%, {:.3f}, {:.3f}\n'.format(keys, size, average_train_error, average_test_error))
        features_f1_csv.write('{}, {}%, {:.3f}\n'.format(keys, size, average_f1))

features_acc_csv.close()
features_f1_csv.close()

max_stars_repo_name: chuajiesheng/twitter-sentiment-analysis

max_issues_repo_name: chuajiesheng/twitter-sentiment-analysis

max_forks_repo_name: chuajiesheng/twitter-sentiment-analysis


text:
import os
import numpy as np
import pytest
import tempfile

from mlagents.trainers.demo_loader import (
    load_demonstration,
    demo_to_buffer,
    get_demo_files,
)

def test_load_demo():
    path_prefix = os.path.dirname(os.path.abspath(__file__))
    behavior_spec, pair_infos, total_expected = load_demonstration(
        path_prefix + "/test.demo"
    )
    assert np.sum(behavior_spec.observation_shapes[0]) == 8
    assert len(pair_infos) == total_expected

    _, demo_buffer = demo_to_buffer(path_prefix + "/test.demo", 1)
    assert len(demo_buffer["actions"]) == total_expected - 1

def test_load_demo_dir():
    path_prefix = os.path.dirname(os.path.abspath(__file__))
    behavior_spec, pair_infos, total_expected = load_demonstration(
        path_prefix + "/test_demo_dir"
    )
    assert np.sum(behavior_spec.observation_shapes[0]) == 8
    assert len(pair_infos) == total_expected

    _, demo_buffer = demo_to_buffer(path_prefix + "/test_demo_dir", 1)
    assert len(demo_buffer["actions"]) == total_expected - 1

def test_edge_cases():
    path_prefix = os.path.dirname(os.path.abspath(__file__))
    # nonexistent file and directory
    with pytest.raises(FileNotFoundError):
        get_demo_files(os.path.join(path_prefix, "nonexistent_file.demo"))
    with pytest.raises(FileNotFoundError):
        get_demo_files(os.path.join(path_prefix, "nonexistent_directory"))
    with tempfile.TemporaryDirectory() as tmpdirname:
        # empty directory
        with pytest.raises(ValueError):
            get_demo_files(tmpdirname)
        # invalid file
        invalid_fname = os.path.join(tmpdirname, "mydemo.notademo")
        with open(invalid_fname, "w") as f:
            f.write("I'm not a demo")
        with pytest.raises(ValueError):
            get_demo_files(invalid_fname)
        # invalid directory
        with pytest.raises(ValueError):
            get_demo_files(tmpdirname)
        # valid file
        valid_fname = os.path.join(tmpdirname, "mydemo.demo")
        with open(valid_fname, "w") as f:
            f.write("I'm a demo file")
        assert get_demo_files(valid_fname) == [valid_fname]
        # valid directory
        assert get_demo_files(tmpdirname) == [valid_fname]


text:
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K

def weighted_cross_entropy(y_true, y_pred, pos_weight=7):
    losses = y_true * -K.log(y_pred) * pos_weight + (1-y_true) * -K.log(1-y_pred)
    losses = K.clip(losses, 0.0, 11)
    return K.mean(losses)

def balanced_weighted_loss(weights):
    def weighted_loss(y_true, y_pred):
        background_weights = weights[:,0] ** (1-y_true)
        signal_weights = weights[:,1] ** (y_true)
        CE = K.binary_crossentropy(y_true, y_pred)
        losses = background_weights * signal_weights * CE
        losses = K.clip(losses, 0.0, 11.0)
        return K.mean(losses, axis=-1)
    return weighted_loss

def crossentropy(preds, labels, weights=None, name='loss'):

    with tf.name_scope(name) as scope:
        loss = tf.nn.softmax_cross_entropy_with_logits(logits=preds, labels=labels, name='unweighted_loss')

        if weights is None:
            loss = tf.reduce_mean(loss, name='final_loss')
            return loss

        class_weights = tf.constant(weights, name='class_weights')
        sample_weights = tf.reduce_sum(class_weights*labels, axis=1, name='sample_weights')
        loss = tf.reduce_mean(sample_weights*loss, name='weighted_loss')
        return loss

def RACL_losses(y_true, y_pred, masks, opt):
    ae_label, oe_label, sc_label = y_true
    ae_pred, oe_pred, sc_pred = y_pred
    word_mask, sentiment_mask = masks

    # Format predictions
    ae_pred = tf.cast(ae_pred, tf.float32, name='ae_pred')
    oe_pred = tf.cast(oe_pred, tf.float32, name='oe_pred')
    sc_pred = tf.cast(sc_pred, tf.float32, name='sc_pred')
    word_mask = tf.cast(word_mask, tf.float32, name='word_mask')
    sentiment_mask = tf.cast(sentiment_mask, tf.float32, name='sentiment_mask')

    # Convert values to probabilities
    ae_prob = tf.nn.softmax(ae_pred, axis=-1, name='ae_prob')
    oe_prob = tf.nn.softmax(oe_pred, axis=-1, name='oe_prob')
    # sc_prob = tf.nn.softmax(sc_pred, axis=-1, name='sc_prob')

    # Define shapes
    batch_size = ae_pred.shape[0]
    output_shape = [-1, opt.n_classes]
    mask_shape = [1, 1, opt.n_classes]

    # Mask AE, OE, SC Predictions
    word_mask = tf.tile(tf.expand_dims(word_mask, axis=-1), mask_shape)
    ae_pred = tf.reshape(word_mask*ae_pred, output_shape, name='ae_pred_masked')
    oe_pred = tf.reshape(word_mask*oe_pred, output_shape, name='oe_pred_masked')

    sentiment_mask = tf.tile(tf.expand_dims(sentiment_mask, axis=-1), mask_shape)
    sc_pred = tf.reshape(sentiment_mask*sc_pred, output_shape, name='sc_pred_masked')

    # Relation R4 (only in Training)
    # In training / validation, sentiment masks are set to 1.0 only for aspect terms.
    # In testing, sentiment masks are set to 1.0 for all words (except padded ones).

    # Format Labels
    ae_label = tf.cast(ae_label, tf.float32, name='ae_label')
    oe_label = tf.cast(oe_label, tf.float32, name='oe_label')
    sc_label = tf.cast(sc_label, tf.float32, name='sc_label')
    ae_label = tf.reshape(ae_label, output_shape, name='ae_label_flat')
    oe_label = tf.reshape(oe_label, output_shape, name='oe_label_flat')
    sc_label = tf.reshape(sc_label, output_shape, name='sc_label_flat')

    # AE & OE Regularization cost - only get Beginning [1] and Inside [2] values
    ae_cost = tf.reduce_sum(ae_prob[:,:,1:], axis=-1, name='ae_cost')
    oe_cost = tf.reduce_sum(oe_prob[:,:,1:], axis=-1, name='oe_cost')
    total_cost = ae_cost + oe_cost - 1.
    total_cost = tf.maximum(0., total_cost, name='total_cost')
    reg_cost = tf.reduce_sum(total_cost) / tf.reduce_sum(word_mask)
    reg_cost = tf.identity(reg_cost, name='regularization_cost')

    # Weighted SoftMax Categorical Cross-Entropy for AE, OE, SC
    ae_loss = crossentropy(ae_pred, ae_label, opt.term_weights, name='aspect')
    oe_loss = crossentropy(oe_pred, oe_label, opt.term_weights, name='opinion')
    sc_loss = crossentropy(sc_pred, sc_label, opt.polarity_weights, name='sentiment')

    loss = opt.aspect_weight * ae_loss + \
            opt.opinion_weight * oe_loss + \
            opt.sentiment_weight * sc_loss + \
            opt.regularization_weight * reg_cost
    loss = tf.identity(loss, name='overall_loss')
    return loss, ae_loss, oe_loss, sc_loss, reg_cost

def RACL_losses_wrapper(opt):
    def call_RACL_losses(y_true, y_pred):
        # Split predictions into word_mask, sentiment_mask, aspect_pred, opinion_pred, sentiment_pred
        word_mask, sentiment_mask, *predictions = tf.split(y_pred, [1,1,3,3,3], axis=-1)
        word_mask = tf.squeeze(word_mask, axis=-1)
        sentiment_mask = tf.squeeze(sentiment_mask, axis=-1)

        # Split labels into aspect_label, opinion_label, sentiment_label
        labels = tf.split(y_true, [1,1,1], axis=-1)

        # Call RACL_losses
        losses = RACL_losses(predictions, labels, [word_mask, sentiment_mask], opt)
        return losses[0]
    return call_RACL_losses


text:
import numpy as np
import torch
from torch import nn
from torchvision import models
import sys
import torch.nn.functional as F

sys.path.append("/media/user/a9755522-b17e-4bde-96f6-088bbbc3a1401/OCDA/ADVENT")

class Classifier_Module(nn.Module):

    def __init__(self, dims_in, dilation_series, padding_series, num_classes):
        super(Classifier_Module, self).__init__()
        self.conv2d_list = nn.ModuleList()
        for dilation, padding in zip(dilation_series, padding_series):
            self.conv2d_list.append(
                nn.Conv2d(dims_in, num_classes, kernel_size=3, stride=1, padding=padding, dilation=dilation, bias=True))

        for m in self.conv2d_list:
            m.weight.data.normal_(0, 0.01)

    def forward(self, x):
        out = self.conv2d_list[0](x)
        for i in range(len(self.conv2d_list) - 1):
            out += self.conv2d_list[i + 1](x)
        return out

## 1. Deeplab + VGG baseline
class DeeplabVGG(nn.Module):
    def __init__(self, num_classes, vgg16_caffe_path=None, pretrained=False):
        super(DeeplabVGG, self).__init__()
        vgg = models.vgg16()
        if pretrained:
            vgg.load_state_dict(torch.load(vgg16_caffe_path))

        features, classifier = list(vgg.features.children()), list(vgg.classifier.children())
        # remove pool4/pool5
        features = nn.Sequential(*(features[i] for i in list(range(23)) + list(range(24, 30))))
        for i in [23, 25, 27]:
            features[i].dilation = (2, 2)
            features[i].padding = (2, 2)

        fc6 = nn.Conv2d(512, 1024, kernel_size=3, padding=4, dilation=4)
        fc7 = nn.Conv2d(1024, 1024, kernel_size=3, padding=4, dilation=4)

        self.features = nn.Sequential(
            *([features[i] for i in range(len(features))] + [fc6, nn.ReLU(inplace=True), fc7, nn.ReLU(inplace=True)]))

        self.classifier = Classifier_Module(1024, [6, 12, 18, 24], [6, 12, 18, 24], num_classes)

    def forward(self, x):
        x = self.features(x)
        seg_out = self.classifier(x)
        return x, seg_out

    def classifier_(self, x):
        seg_out = self.classifier(x)
        return seg_out

    def optim_parameters(self, args):
        return self.parameters()

## 2. Deeplab + Parallel Boundary Classifier
class DeeplabVGG_Boundary(DeeplabVGG):
    def __init__(self, num_classes, vgg16_caffe_path=None, pretrained=False):
        super().__init__(num_classes, vgg16_caffe_path, pretrained)
        self.classifier_boundary = Classifier_Module(1024, [6, 12, 18, 24], [6, 12, 18, 24], 1)

    def forward(self, x):
        x = self.features(x)
        seg_out = self.classifier(x)
        boundary = self.classifier_boundary(x)
        return x, seg_out, boundary

    def Boundary_classifier_(self, x):
        seg_out = self.classifier(x)
        boundary = self.classifier_boundary(x)
        return seg_out, boundary

## 3. Deeplab + Attention Boundary
class DeeplabVGG_Boundary_Attention(nn.Module):
    def __init__(self, num_classes, vgg16_caffe_path=None, pretrained=False):
        super(DeeplabVGG_Boundary_Attention, self).__init__()

        vgg = models.vgg16()
        if pretrained:
            vgg.load_state_dict(torch.load(vgg16_caffe_path))

        features, classifier = list(vgg.features.children()), list(vgg.classifier.children())
        # remove pool4/pool5
        features = nn.Sequential(*(features[i] for i in list(range(23)) + list(range(24, 30))))
        for i in [23, 25, 27]:
            features[i].dilation = (2, 2)
            features[i].padding = (2, 2)

        fc6 = nn.Conv2d(512, 1024, kernel_size=3, padding=4, dilation=4)
        fc7 = nn.Conv2d(1024, 1024, kernel_size=3, padding=4, dilation=4)

        self.features = nn.Sequential(
            *([features[i] for i in range(len(features))] + [fc6, nn.ReLU(inplace=True), fc7, nn.ReLU(inplace=True)]))

        self.classifier = Classifier_Module(1025, [6, 12, 18, 24], [6, 12, 18, 24], num_classes)
        self.classifier_boundary = Classifier_Module(1024, [6, 12, 18, 24], [6, 12, 18, 24], 1)

    def forward(self, x):
        x = self.features(x)

        # encoder
        # x_en = self.enc4_1(x)
        # x_en = self.relu(x_en)
        # x_en = self.enc4_2(x_en)
        # x_en = self.relu(x_en)
        # x_en = self.enc4_3(x_en)
        # x_en = self.relu(x_en)
        # boundary = self.enc4_4(x_en)
        # boundary = torch.mean(x_en, dim=1, keepdim=True)

        # decoder
        # x_de = self.dec4(x_en)
        # x_de = self.relu(x_de)
        # feature fusion
        # x = x + torch.sigmoid(x_de)
        boundary = self.classifier_boundary(x)
        x = torch.cat((x, boundary), dim=1)
        seg_out = self.classifier(x)
        return x, seg_out, boundary

    def optim_parameters(self, args):
        return self.parameters()

class DeeplabVGG_Boundary_Attention_v2(nn.Module):
    def __init__(self, num_classes, vgg16_caffe_path=None, pretrained=False):
        super(DeeplabVGG_Boundary_Attention_v2, self).__init__()

        vgg = models.vgg16()
        if pretrained:
            vgg.load_state_dict(torch.load(vgg16_caffe_path))

        features, classifier = list(vgg.features.children()), list(vgg.classifier.children())
        # remove pool4/pool5
        features = nn.Sequential(*(features[i] for i in list(range(23)) + list(range(24, 30))))
        for i in [23, 25, 27]:
            features[i].dilation = (2, 2)
            features[i].padding = (2, 2)

        fc6 = nn.Conv2d(512, 1024, kernel_size=3, padding=4, dilation=4)
        fc7 = nn.Conv2d(1024, 1024, kernel_size=3, padding=4, dilation=4)

        self.features = nn.Sequential(
            *([features[i] for i in range(len(features))] + [fc6, nn.ReLU(inplace=True), fc7, nn.ReLU(inplace=True)]))

        self.classifier = Classifier_Module(1024, [6, 12, 18, 24], [6, 12, 18, 24], num_classes)
        self.classifier_boundary = Classifier_Module(1024, [6, 12, 18, 24], [6, 12, 18, 24], 1)

        self.fusion_block = FusionBlock(1024)

    def forward(self, x):
        x = self.features(x)

        boundary = self.classifier_boundary(x)
        x = self.fusion_block(x, boundary)

        seg_out = self.classifier(x)
        return x, seg_out, boundary

    def optim_parameters(self, args):
        return self.parameters()

class FusionBlock(nn.Module):
    def __init__(self, in_c=1024):
        super(FusionBlock, self).__init__()

        self.mixer = nn.Sequential(
            nn.Conv2d(in_c + 1, in_c, kernel_size=3, padding=1, stride=1, bias=True),
            nn.ReLU()
        )

        self.mlp = nn.Sequential(
            nn.Linear(in_c, in_c//16),
            nn.ReLU(),
            nn.Linear(in_c//16, in_c)
        )

    def forward(self, x, boundary):
        x = torch.cat((x, boundary), dim=1)
        x = self.mixer(x).squeeze()

        x_avg = F.avg_pool2d(x, (x.size(1), x.size(2)), stride=(x.size(1), x.size(2)))
        x_avg = x_avg.view(x_avg.size(0))
        x_avg_att = self.mlp(x_avg)

        x_max = F.max_pool2d(x, (x.size(1), x.size(2)), stride=(x.size(1), x.size(2)))
        x_max = x_max.view(x_max.size(0))
        x_max_att = self.mlp(x_max)

        x_att = x_avg_att + x_max_att

        scale = F.sigmoid(x_att).view(x_att.size(0), 1, 1)
        x = x * scale

        return x.unsqueeze(0)

def get_deeplab_v2_vgg(cfg, num_classes=19, pretrained_model=None, pretrained=True):
    if cfg.TRAIN.OCDA_METHOD == 'baseline' or cfg.TRAIN.OCDA_METHOD == "selfTrain":
        model = DeeplabVGG(num_classes, pretrained_model, pretrained)
    elif cfg.TRAIN.OCDA_METHOD == 'boundary' or cfg.TRAIN.OCDA_METHOD == 'selfTrain_boundary':
        model = DeeplabVGG_Boundary(num_classes, pretrained_model, pretrained)
    elif cfg.TRAIN.OCDA_METHOD == 'ad_boundary':
        model = DeeplabVGG_Boundary_Attention(num_classes, pretrained_model, pretrained)
    elif cfg.TRAIN.OCDA_METHOD == 'attn_boundary':
        model = DeeplabVGG_Boundary_Attention_v2(num_classes, pretrained_model, pretrained)
    return model

max_stars_repo_name: gritYCDA/boundaryOCDA

max_issues_repo_name: gritYCDA/boundaryOCDA

max_forks_repo_name: gritYCDA/boundaryOCDA


text:
from synbioweaver.core import *
from synbioweaver.aspects.designRulesAspect import DesignRules
from transferFunctionAspect import MoleculeExpressionTrace

import numpy as np
import matplotlib.pyplot as plt
import matplotlib

declareNewMolecule('Cl')
declareNewMolecule('Ara')
declareNewMolecule('aTc')
declareNewMolecule('LasI')
declareNewMolecule('RhlI')
declareNewMolecule('YFP')

class SystemCircuit1(Circuit):
    def mainCircuit(self):
        self.createMolecule(aTc)
        self.addPart(PositivePromoter(aTc))
        self.addPart(CodingRegion(YFP))

class SystemCircuit2(Circuit):
    def mainCircuit(self):
        self.createMolecule(aTc)
        self.addPart(PositivePromoter(aTc))
        self.addPart(CodingRegion(Cl))
        self.addPart(NegativePromoter(Cl))
        self.addPart(CodingRegion(YFP))

##
print "Running SystemCircuit1"
compiledSystem1 = Weaver(SystemCircuit1,DesignRules,MoleculeExpressionTrace).output()
inputMol1 = compiledSystem1.moleculeList[0]
outputMol = compiledSystem1.moleculeList[1]
print "input/outputs:", inputMol1, outputMol

molInputRange = np.logspace(-4, 3,100, endpoint=True)
yfpOutput= []

for i in range(0,len(molInputRange)):
    inputMol1.signal = molInputRange[i]
    yfpOutput.append( outputMol.calculateTransfer() )

fig = plt.figure()
plt.plot( np.log10(molInputRange), np.log10(yfpOutput) )
plt.xlabel('Input (aTc)')
plt.ylabel('Ouput (YFP)')
plt.savefig("plot-transfer-system1.pdf", bbox_inches='tight')
plt.close()

##
print "Running SystemCircuit2"
compiledSystem2 = Weaver(SystemCircuit2,DesignRules,MoleculeExpressionTrace).output()
inputMol1 = compiledSystem2.moleculeList[0]
outputMol = compiledSystem2.moleculeList[2]
print "input/outputs:", inputMol1, outputMol

molInputRange = np.logspace(-4, 3,100, endpoint=True)
yfpOutput= []

for i in range(0,len(molInputRange)):
    inputMol1.signal = molInputRange[i]
    yfpOutput.append( outputMol.calculateTransfer() )

fig = plt.figure()
plt.plot( np.log10(molInputRange), np.log10(yfpOutput) )
plt.xlabel('Input (aTc)')
plt.ylabel('Ouput (YFP)')
plt.savefig("plot-transfer-system2.pdf", bbox_inches='tight')
plt.close()

max_stars_repo_name: PhilippBoeing/synbioweaver

max_issues_repo_name: PhilippBoeing/synbioweaver

max_forks_repo_name: PhilippBoeing/synbioweaver


text:
#  NLP written by GAMS Convert at 04/21/18 13:51:47
#
#  Equation counts
#      Total        E        G        L        N        X        C        B
#         88        7        6       75        0        0        0        0
#
#  Variable counts
#                   x        b        i      s1s      s2s       sc       si
#      Total     cont   binary  integer     sos1     sos2    scont     sint
#         62       62        0        0        0        0        0        0
#  FX      0        0        0        0        0        0        0        0
#
#  Nonzero counts
#      Total    const       NL      DLL
#        378       69      309        0
#
#  Reformulation has removed 1 variable and 1 equation

from pyomo.environ import *

model = m = ConcreteModel()

m.x2 = Var(within=Reals,bounds=(6.21460809842219,8.41183267575841),initialize=6.21460809842219)
m.x3 = Var(within=Reals,bounds=(6.21460809842219,8.41183267575841),initialize=6.21460809842219)
m.x4 = Var(within=Reals,bounds=(6.21460809842219,8.41183267575841),initialize=6.21460809842219)
m.x5 = Var(within=Reals,bounds=(160,163.752806164),initialize=160)
m.x6 = Var(within=Reals,bounds=(160,163.752806164),initialize=160)
m.x7 = Var(within=Reals,bounds=(160,163.752806164),initialize=160)
m.x8 = Var(within=Reals,bounds=(160,163.752806164),initialize=160)
m.x9 = Var(within=Reals,bounds=(160,163.752806164),initialize=160)
m.x10 = Var(within=Reals,bounds=(160,178.461227596),initialize=160)
m.x11 = Var(within=Reals,bounds=(160,178.461227596),initialize=160)
m.x12 = Var(within=Reals,bounds=(160,178.461227596),initialize=160)
m.x13 = Var(within=Reals,bounds=(160,178.461227596),initialize=160)
m.x14 = Var(within=Reals,bounds=(160,178.461227596),initialize=160)
m.x15 = Var(within=Reals,bounds=(160,200),initialize=160)
m.x16 = Var(within=Reals,bounds=(160,200),initialize=160)
m.x17 = Var(within=Reals,bounds=(160,200),initialize=160)
m.x18 = Var(within=Reals,bounds=(160,200),initialize=160)
m.x19 = Var(within=Reals,bounds=(160,200),initialize=160)
m.x20 = Var(within=Reals,bounds=(160,221.538772404),initialize=160)
m.x21 = Var(within=Reals,bounds=(160,221.538772404),initialize=160)
m.x22 = Var(within=Reals,bounds=(160,221.538772404),initialize=160)
m.x23 = Var(within=Reals,bounds=(160,221.538772404),initialize=160)
m.x24 = Var(within=Reals,bounds=(160,221.538772404),initialize=160)
m.x25 = Var(within=Reals,bounds=(160,236.247193836),initialize=160)
m.x26 = Var(within=Reals,bounds=(160,236.247193836),initialize=160)
m.x27 = Var(within=Reals,bounds=(160,236.247193836),initialize=160)
m.x28 = Var(within=Reals,bounds=(160,236.247193836),initialize=160)
m.x29 = Var(within=Reals,bounds=(160,236.247193836),initialize=160)
m.x30 = Var(within=Reals,bounds=(60,63.752806164),initialize=60)
m.x31 = Var(within=Reals,bounds=(60,78.461227596),initialize=60)
m.x32 = Var(within=Reals,bounds=(60,100),initialize=60)
m.x33 = Var(within=Reals,bounds=(60,121.538772404),initialize=60)
m.x34 = Var(within=Reals,bounds=(60,136.247193836),initialize=60)
m.x35 = Var(within=Reals,bounds=(60,63.752806164),initialize=60)
m.x36 = Var(within=Reals,bounds=(60,78.461227596),initialize=60)
m.x37 = Var(within=Reals,bounds=(60,100),initialize=60)
m.x38 = Var(within=Reals,bounds=(60,121.538772404),initialize=60)
m.x39 = Var(within=Reals,bounds=(60,136.247193836),initialize=60)
m.x40 = Var(within=Reals,bounds=(60,63.752806164),initialize=60)
m.x41 = Var(within=Reals,bounds=(60,78.461227596),initialize=60)
m.x42 = Var(within=Reals,bounds=(60,100),initialize=60)
m.x43 = Var(within=Reals,bounds=(60,121.538772404),initialize=60)
m.x44 = Var(within=Reals,bounds=(60,136.247193836),initialize=60)
m.x45 = Var(within=Reals,bounds=(60,63.752806164),initialize=60)
m.x46 = Var(within=Reals,bounds=(60,78.461227596),initialize=60)
m.x47 = Var(within=Reals,bounds=(60,100),initialize=60)
m.x48 = Var(within=Reals,bounds=(60,121.538772404),initialize=60)
m.x49 = Var(within=Reals,bounds=(60,136.247193836),initialize=60)
m.x50 = Var(within=Reals,bounds=(60,63.752806164),initialize=60)
m.x51 = Var(within=Reals,bounds=(60,78.461227596),initialize=60)
m.x52 = Var(within=Reals,bounds=(60,100),initialize=60)
m.x53 = Var(within=Reals,bounds=(60,121.538772404),initialize=60)
m.x54 = Var(within=Reals,bounds=(60,136.247193836),initialize=60)
m.x55 = Var(within=Reals,bounds=(4.82831,7.1591),initialize=4.82831)
m.x56 = Var(within=Reals,bounds=(4.42285,6.7071),initialize=4.42285)
m.x57 = Var(within=Reals,bounds=(0.00622203373693604,0.0640002391877942),initialize=0.00622203373693604)
m.x58 = Var(within=Reals,bounds=(0.0155550843423401,0.160000597969486),initialize=0.0155550843423401)
m.x59 = Var(within=Reals,bounds=(0.00622203373693604,0.0640002391877942),initialize=0.00622203373693604)
m.x60 = Var(within=Reals,bounds=(0.019555254080048,0.191999736805455),initialize=0.019555254080048)
m.x61 = Var(within=Reals,bounds=(0.00488881352001201,0.0479999342013636),initialize=0.00488881352001201)
m.x62 = Var(within=Reals,bounds=(0.00488881352001201,0.0479999342013636),initialize=0.00488881352001201)

m.obj = Objective(expr=0.3*(10*exp(0.6*m.x2) + 10*exp(0.6*m.x3) + 10*exp(0.6*m.x4)) - 1.54711033913716E-6*m.x5
                        - 0.000219040316990534*m.x6 - 0.00264813118267794*m.x7 - 0.000219040316990534*m.x8
                        - 1.54711033913716E-6*m.x9 - 0.000219040316990533*m.x10 - 0.0310117896917886*m.x11
                        - 0.374923157717238*m.x12 - 0.0310117896917886*m.x13 - 0.000219040316990532*m.x14
                        - 0.00264813118267793*m.x15 - 0.374923157717237*m.x16 - 4.5327075795914*m.x17
                        - 0.374923157717237*m.x18 - 0.00264813118267791*m.x19 - 0.000219040316990532*m.x20
                        - 0.0310117896917884*m.x21 - 0.374923157717236*m.x22 - 0.0310117896917884*m.x23
                        - 0.000219040316990531*m.x24 - 1.54711033913713E-6*m.x25 - 0.000219040316990529*m.x26
                        - 0.00264813118267789*m.x27 - 0.000219040316990529*m.x28 - 1.54711033913712E-6*m.x29
                        - 1.9690495225382E-6*m.x30 - 0.000278778585260679*m.x31 - 0.00337034877795374*m.x32
                        - 0.000278778585260679*m.x33 - 1.9690495225382E-6*m.x34 - 0.000278778585260679*m.x35
                        - 0.0394695505168218*m.x36 - 0.477174928003758*m.x37 - 0.0394695505168218*m.x38
                        - 0.000278778585260677*m.x39 - 0.00337034877795372*m.x40 - 0.477174928003756*m.x41
                        - 5.7689005558436*m.x42 - 0.477174928003756*m.x43 - 0.00337034877795371*m.x44
                        - 0.000278778585260677*m.x45 - 0.0394695505168216*m.x46 - 0.477174928003755*m.x47
                        - 0.0394695505168216*m.x48 - 0.000278778585260676*m.x49 - 1.96904952253816E-6*m.x50
                        - 0.000278778585260674*m.x51 - 0.00337034877795367*m.x52 - 0.000278778585260674*m.x53
                        - 1.96904952253816E-6*m.x54, sense=minimize)

m.c2 = Constraint(expr=   m.x2 - m.x55 >= 0.693147180559945)

m.c3 = Constraint(expr=   m.x3 - m.x55 >= 1.09861228866811)

m.c4 = Constraint(expr=   m.x4 - m.x55 >= 1.38629436111989)

m.c5 = Constraint(expr=   m.x2 - m.x56 >= 1.38629436111989)

m.c6 = Constraint(expr=   m.x3 - m.x56 >= 1.79175946922805)

m.c7 = Constraint(expr=   m.x4 - m.x56 >= 1.09861228866811)

m.c8 = Constraint(expr=m.x5*m.x57 + m.x30*m.x60 <= 8)

m.c9 = Constraint(expr=m.x6*m.x57 + m.x31*m.x60 <= 8)

m.c10 = Constraint(expr=m.x7*m.x57 + m.x32*m.x60 <= 8)

m.c11 = Constraint(expr=m.x8*m.x57 + m.x33*m.x60 <= 8)

m.c12 = Constraint(expr=m.x9*m.x57 + m.x34*m.x60 <= 8)

m.c13 = Constraint(expr=m.x10*m.x57 + m.x35*m.x60 <= 8)

m.c14 = Constraint(expr=m.x11*m.x57 + m.x36*m.x60 <= 8)

m.c15 = Constraint(expr=m.x12*m.x57 + m.x37*m.x60 <= 8)

m.c16 = Constraint(expr=m.x13*m.x57 + m.x38*m.x60 <= 8)

m.c17 = Constraint(expr=m.x14*m.x57 + m.x39*m.x60 <= 8)

m.c18 = Constraint(expr=m.x15*m.x57 + m.x40*m.x60 <= 8)

m.c19 = Constraint(expr=m.x16*m.x57 + m.x41*m.x60 <= 8)

m.c20 = Constraint(expr=m.x17*m.x57 + m.x42*m.x60 <= 8)

m.c21 = Constraint(expr=m.x18*m.x57 + m.x43*m.x60 <= 8)

m.c22 = Constraint(expr=m.x19*m.x57 + m.x44*m.x60 <= 8)

m.c23 = Constraint(expr=m.x20*m.x57 + m.x45*m.x60 <= 8)

m.c24 = Constraint(expr=m.x21*m.x57 + m.x46*m.x60 <= 8)

m.c25 = Constraint(expr=m.x22*m.x57 + m.x47*m.x60 <= 8)

m.c26 = Constraint(expr=m.x23*m.x57 + m.x48*m.x60 <= 8)

m.c27 = Constraint(expr=m.x24*m.x57 + m.x49*m.x60 <= 8)

m.c28 = Constraint(expr=m.x25*m.x57 + m.x50*m.x60 <= 8)

m.c29 = Constraint(expr=m.x26*m.x57 + m.x51*m.x60 <= 8)

m.c30 = Constraint(expr=m.x27*m.x57 + m.x52*m.x60 <= 8)

m.c31 = Constraint(expr=m.x28*m.x57 + m.x53*m.x60 <= 8)

m.c32 = Constraint(expr=m.x29*m.x57 + m.x54*m.x60 <= 8)

m.c33 = Constraint(expr=m.x5*m.x58 + m.x30*m.x61 <= 8)

m.c34 = Constraint(expr=m.x6*m.x58 + m.x31*m.x61 <= 8)

m.c35 = Constraint(expr=m.x7*m.x58 + m.x32*m.x61 <= 8)

m.c36 = Constraint(expr=m.x8*m.x58 + m.x33*m.x61 <= 8)

m.c37 = Constraint(expr=m.x9*m.x58 + m.x34*m.x61 <= 8)

m.c38 = Constraint(expr=m.x10*m.x58 + m.x35*m.x61 <= 8)

m.c39 = Constraint(expr=m.x11*m.x58 + m.x36*m.x61 <= 8)

m.c40 = Constraint(expr=m.x12*m.x58 + m.x37*m.x61 <= 8)

m.c41 = Constraint(expr=m.x13*m.x58 + m.x38*m.x61 <= 8)

m.c42 = Constraint(expr=m.x14*m.x58 + m.x39*m.x61 <= 8)

m.c43 = Constraint(expr=m.x15*m.x58 + m.x40*m.x61 <= 8)

m.c44 = Constraint(expr=m.x16*m.x58 + m.x41*m.x61 <= 8)

m.c45 = Constraint(expr=m.x17*m.x58 + m.x42*m.x61 <= 8)

m.c46 = Constraint(expr=m.x18*m.x58 + m.x43*m.x61 <= 8)

m.c47 = Constraint(expr=m.x19*m.x58 + m.x44*m.x61 <= 8)

m.c48 = Constraint(expr=m.x20*m.x58 + m.x45*m.x61 <= 8)

m.c49 = Constraint(expr=m.x21*m.x58 + m.x46*m.x61 <= 8)

m.c50 = Constraint(expr=m.x22*m.x58 + m.x47*m.x61 <= 8)

m.c51 = Constraint(expr=m.x23*m.x58 + m.x48*m.x61 <= 8)

m.c52 = Constraint(expr=m.x24*m.x58 + m.x49*m.x61 <= 8)

m.c53 = Constraint(expr=m.x25*m.x58 + m.x50*m.x61 <= 8)

m.c54 = Constraint(expr=m.x26*m.x58 + m.x51*m.x61 <= 8)

m.c55 = Constraint(expr=m.x27*m.x58 + m.x52*m.x61 <= 8)

m.c56 = Constraint(expr=m.x28*m.x58 + m.x53*m.x61 <= 8)

m.c57 = Constraint(expr=m.x29*m.x58 + m.x54*m.x61 <= 8)

m.c58 = Constraint(expr=m.x5*m.x59 + m.x30*m.x62 <= 8)

m.c59 = Constraint(expr=m.x6*m.x59 + m.x31*m.x62 <= 8)

m.c60 = Constraint(expr=m.x7*m.x59 + m.x32*m.x62 <= 8)

m.c61 = Constraint(expr=m.x8*m.x59 + m.x33*m.x62 <= 8)

m.c62 = Constraint(expr=m.x9*m.x59 + m.x34*m.x62 <= 8)

m.c63 = Constraint(expr=m.x10*m.x59 + m.x35*m.x62 <= 8)

m.c64 = Constraint(expr=m.x11*m.x59 + m.x36*m.x62 <= 8)

m.c65 = Constraint(expr=m.x12*m.x59 + m.x37*m.x62 <= 8)

m.c66 = Constraint(expr=m.x13*m.x59 + m.x38*m.x62 <= 8)

m.c67 = Constraint(expr=m.x14*m.x59 + m.x39*m.x62 <= 8)

m.c68 = Constraint(expr=m.x15*m.x59 + m.x40*m.x62 <= 8)

m.c69 = Constraint(expr=m.x16*m.x59 + m.x41*m.x62 <= 8)

m.c70 = Constraint(expr=m.x17*m.x59 + m.x42*m.x62 <= 8)

m.c71 = Constraint(expr=m.x18*m.x59 + m.x43*m.x62 <= 8)

m.c72 = Constraint(expr=m.x19*m.x59 + m.x44*m.x62 <= 8)

m.c73 = Constraint(expr=m.x20*m.x59 + m.x45*m.x62 <= 8)

m.c74 = Constraint(expr=m.x21*m.x59 + m.x46*m.x62 <= 8)

m.c75 = Constraint(expr=m.x22*m.x59 + m.x47*m.x62 <= 8)

m.c76 = Constraint(expr=m.x23*m.x59 + m.x48*m.x62 <= 8)

m.c77 = Constraint(expr=m.x24*m.x59 + m.x49*m.x62 <= 8)

m.c78 = Constraint(expr=m.x25*m.x59 + m.x50*m.x62 <= 8)

m.c79 = Constraint(expr=m.x26*m.x59 + m.x51*m.x62 <= 8)

m.c80 = Constraint(expr=m.x27*m.x59 + m.x52*m.x62 <= 8)

m.c81 = Constraint(expr=m.x28*m.x59 + m.x53*m.x62 <= 8)

m.c82 = Constraint(expr=m.x29*m.x59 + m.x54*m.x62 <= 8)

m.c83 = Constraint(expr=-exp(2.07944154167984 - m.x55) + m.x57 == 0)

m.c84 = Constraint(expr=-exp(2.99573227355399 - m.x55) + m.x58 == 0)

m.c85 = Constraint(expr=-exp(2.07944154167984 - m.x55) + m.x59 == 0)

m.c86 = Constraint(expr=-exp(2.77258872223978 - m.x56) + m.x60 == 0)

m.c87 = Constraint(expr=-exp(1.38629436111989 - m.x56) + m.x61 == 0)

m.c88 = Constraint(expr=-exp(1.38629436111989 - m.x56) + m.x62 == 0)

reason: from pyomo


text:
import numpy as np

### Binary operations
def hweight(n):
    """ Return the Hamming weight of 'n' """
    c = 0
    while n>0:
        c += (n & 1)
        n >>= 1
    return c

def hdistance(x,y):
    """ Return the Hamming distance between 'x' and 'y' """
    return hweight(x^y)

def binary_writing(n, nb_bits=32, with_hamming=False):
    """ Return the binary writing 'w' of 'n' with 'nb_bits'
    If with_hamming is True, return a couple (w, h) with 'h' the Hamming weight of 'n'
    """
    n = np.array(n)
    w, h = np.zeros((nb_bits, len(n))), np.zeros((len(n)))

    for ind in range(nb_bits):
        w[ind] = (n & 1)
        h += w[ind]

        n >>= 1
        ind += 1

    return (w, h) if with_hamming else w

### Conversion
def to_hex(v, nb_bits=16):
    """ Convert the value 'v' into a hexadecimal string (without the prefix 0x)"""
    try:
        v_hex = v.hex()
    except AttributeError:
        v_hex = hex(v)[2:]
    return '0'*(nb_bits//4-len(v_hex)) + v_hex

def split_octet(hexstr):
    """ Split a hexadecimal string (without the prefix 0x)
    into a list of bytes described with a 2-length hexadecimal string
    """
    return [hexstr[i:i+2] for i in range(0, len(hexstr), 2)]

def to_signed_hex(v, nb_bits=16):
    """ Convert the signed value 'v'
    into a list of bytes described with a 2-length hexadecimal string
    """
    try:
        return split_octet(to_hex(v & ((1<<nb_bits)-1), nb_bits=nb_bits))
    except TypeError as err:
        raise TypeError('Error to transform a <{}> into signed hex.'.format(type(v))) from err

### Write function
def write(_input, uint, nb_bits=16):
    """ Write in '_input' the value 'uint' by writing each byte
    with hexadecimal format in a new line
    """
    uint = to_signed_hex(uint, nb_bits=nb_bits)
    for i in range(nb_bits//8):
        _input.write(uint[i]+'\n')

def write_list(_input, uint_list, nb_bits=16):
    """ Write in '_input' the list of values 'uint_list' by writing each byte
    with hexadecimal format in a new line
    """
    for uint in uint_list:
        write(_input, uint, nb_bits=nb_bits)

### Print Utils
class Color:
    """ Color codes to print colored text in stdout """
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKCYAN = '\033[96m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

max_stars_repo_name: ThFeneuil/python-elmo

max_issues_repo_name: ThFeneuil/python-elmo

max_forks_repo_name: ThFeneuil/python-elmo


text:
import imageio
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
matplotlib.use('Agg')
import numpy as np
import seaborn as sns
import os
import re
sns.set()

__author__ = "Andy Wang"
__license__ = "MIT"

# Plotting info
cutoff_age = 38
task = 'scan_age'
imgs_per_sec = 4
num_outliers = 5

# Load metadata
meta = pd.read_csv("meta_data.tsv", delimiter='\t')
# Put participant_id and session_id together to get a unique key and use as index
meta['unique_key'] = meta['participant_id'] + "_" + meta['session_id'].astype(str)
meta.set_index('unique_key', inplace=True)

# Find the min and max scan_age to fix plot sizes
results = pd.read_csv(f"test_logs/{task}/testacc_full_log_1.csv", header=0,
                      names=['unique_key', 'pred', 'label', 'error'])
min_actual_age = min(results.loc[:, 'label'])
max_actual_age = max(results.loc[:, 'label'])

# Find the number of epochs
files = os.listdir(f'test_logs/{task}')
re_pattern = r'[a-zA-Z_]+(\d+)\.csv$'
pattern = re.compile(re_pattern)
max_epoch = max(int(re.match(pattern, file).group(1)) for file in files)

def retrieve_plot_data(epoch):
    # Load test results
    results = pd.read_csv(f"test_logs/{task}/testacc_full_log_{epoch}.csv", header=0,
                          names=['unique_key', 'pred', 'label', 'error'])
    # Retrieve data for visualisation
    gender = [0 if indiv == 'Female' else 1 for indiv in meta.loc[results.loc[:, "unique_key"], 'gender']]
    pred = results.loc[:, "pred"]
    scan_age = results.loc[:, "label"]
    errors = results.loc[:, "error"].to_numpy()
    acc = np.round(np.mean(errors), 2)

    # Get the outliers marked
    sorted_errors = np.argsort(np.abs(errors))
    outlier_indices = sorted_errors[-num_outliers:]
    outlier_y_pred = pred[outlier_indices]
    outlier_x = scan_age[outlier_indices]
    outlier_lines = [(out_x, out_y_pred) for out_x, out_y_pred in zip(outlier_x, outlier_y_pred)]

    male = {"x": [], "y": []}
    female = {"x": [], "y": []}

    for label, actual_age, pred in zip(gender, scan_age, pred):
        if label == 0:
            female["x"].append(actual_age)
            female["y"].append(pred)
        else:
            male["x"].append(actual_age)
            male["y"].append(pred)
    return female, male, outlier_lines, acc

def generate_img(epoch):
    # Data for plotting
    print(epoch)
    female, male, outlier_lines, acc = retrieve_plot_data(epoch)
    fig, ax = plt.subplots(figsize=(10, 5))
    ax.scatter(**female, label="Female", color="red")
    ax.scatter(**male, label="Male", color="blue")
    ax.plot([min_actual_age, max_actual_age], [min_actual_age, max_actual_age], linestyle='dashed')
    ax.set(xlabel='Actual Scan Age', ylabel='Predicted', title=f'Scan Age Prediction - Epoch {epoch}')
    ax.set_ylim(min_actual_age - 1, max_actual_age + 1)

    for start, end in outlier_lines:
        ax.vlines(start, start, end, label=f"Top {num_outliers} Errors", linestyles="dashed")

    # To avoid printing multiple labels for vlines
    handles, labels = plt.gca().get_legend_handles_labels()
    by_label = dict(zip(labels, handles))
    plt.legend(by_label.values(), by_label.keys())

    textbox = f"L1: {acc}"
    props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
    # place a text box in upper left in axes coords
    ax.text(0.025, 0.75, textbox, transform=ax.transAxes, fontsize=14,
            verticalalignment='top', bbox=props)

    fig.canvas.draw()
    image = np.frombuffer(fig.canvas.tostring_rgb(), dtype='uint8')
    image = image.reshape(fig.canvas.get_width_height()[::-1] + (3,))
    return image

imageio.mimsave(f'./{task}.gif', [generate_img(i) for i in range(1, max_epoch)], fps=imgs_per_sec)