text:
"""Streamlit web app"""

import numpy as np
import streamlit as st
import matplotlib.pyplot as plt
from PIL import Image
from typing import List
import pickle

st.set_option("deprecation.showfileUploaderEncoding", False)

@st.cache
def cached_model(x_valid: List):
    """
    Load logreg model from disk
    """
    model = pickle.load(open("model.txt", "rb"))
    y_pred = model.predict(x_valid)
    # result = model.score(x_valid, y_valid)
    # print(result)
    return y_pred
# result = cached_model(x_valid)

logo_file = "ai4life_logo.png"
image = np.array(Image.open(logo_file))
st.image(image, use_column_width=False)

st.title("AI4LIFE")

uploaded_file = "1053.png"
if uploaded_file is None:
    uploaded_file = st.file_uploader("Load patient's cardiotocography (CTG)...", type="png")
if uploaded_file is not None:
    image = np.array(Image.open(uploaded_file))
    st.image(image, caption="Cardiotocography signals", use_column_width=True)

st.header("EHR")
st.write("EGA (weeks): 39")
st.write("Parity: 1")
st.write("Age: 30")
st.checkbox("Pre-eclampsia")
st.checkbox("Diabetes")


text:
import pandas as pd
import numpy as np
import sklearn
import time
from sklearn.feature_extraction import text
from sklearn.feature_extraction.text import CountVectorizer
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn import cluster
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.metrics import silhouette_samples, silhouette_score
from wordcloud import WordCloud
import itertools
import warnings
import re
import matplotlib
from sklearn.feature_extraction.text  import TfidfVectorizer
warnings.filterwarnings('ignore')

def preprocess_text(text):
    text = text.lower()
    text = re.sub(r'\d+', '', text)
    return text

stop_words_protests = text.ENGLISH_STOP_WORDS.union(['protest', 'protests', 'protesters', 'protested','demonstrate','demonstrated','demonstrators','demonstrations','demonstrator','demonstration','january', 'february', 'march','april', 'may', 'june', 'july','august','september','october','november','december'])

stop_words_battles = text.ENGLISH_STOP_WORDS.union(['january', 'february', 'march','april', 'may', 'june', 'july','august','september','october','november','december', 'killed','kill', 'attack', 'attacked', 'kills', 'killing', 'soldiers','soldier', 'gun', 'gunmen','dead', 'death', 'died'])

def max_sum_sim(doc_embedding, word_embeddings, words, top_n, nr_candidates):
    # Calculate distances and extract keywords
    distances = cosine_similarity(doc_embedding, word_embeddings)
    distances_candidates = cosine_similarity(word_embeddings,
                                            word_embeddings)

    # Get top_n words as candidates based on cosine similarity
    words_idx = list(distances.argsort()[0][-nr_candidates:])
    words_vals = [words[index] for index in words_idx]
    distances_candidates = distances_candidates[np.ix_(words_idx, words_idx)]

    # Calculate the combination of words that are the least similar to each other
    min_sim = np.inf
    candidate = None
    for combination in itertools.combinations(range(len(words_idx)), top_n):
        sim = sum([distances_candidates[i][j] for i in combination for j in combination if i != j])
        if sim < min_sim:
            candidate = combination
            min_sim = sim

    return [words_vals[idx] for idx in candidate]

def mmr(doc_embedding, word_embeddings, words, top_n, diversity):

    # Extract similarity within words, and between words and the document
    word_doc_similarity = cosine_similarity(word_embeddings, doc_embedding)
    word_similarity = cosine_similarity(word_embeddings)

    # Initialize candidates and already choose best keyword/keyphras
    keywords_idx = [np.argmax(word_doc_similarity)]
    candidates_idx = [i for i in range(len(words)) if i != keywords_idx[0]]

    for _ in range(top_n - 1):
        # Extract similarities within candidates and
        # between candidates and selected keywords/phrases
        candidate_similarities = word_doc_similarity[candidates_idx, :]
        target_similarities = np.max(word_similarity[candidates_idx][:, keywords_idx], axis=1)

        # Calculate MMR
        mmr = (1-diversity) * candidate_similarities - diversity * target_similarities.reshape(-1, 1)
        mmr_idx = candidates_idx[np.argmax(mmr)]

        # Update keywords & candidates
        keywords_idx.append(mmr_idx)
        candidates_idx.remove(mmr_idx)

    return [words[idx] for idx in keywords_idx]

def keyword_extraction(data,event_type,n_gram_range=(1,3),keyword_n=5,mss_n=5,mss_nr=10,mmr_n=5,mmr_diversity=0.7):
    """
    Performs keyword extraction for the given country and event type

    Parameters
    ----------
    data : pandas dataframe that contains the following columns ['event_type', 'notes']

    event_type : the event type for which keywords are being extracted (important as list of stop words used changes by event type)

    n_gram_range : the length of keywords/phrases to be extracted
         (Default value = (1,3))

    keyword_n : top number of keywords to extract
         (Default value = 5)
    mss_n : top number of keywords to extract from mss algorithm
         (Default value = 5)
    mss_nr : top number of keywords/keyphrases to select from
         (Default value = 10)
    mmr_n : top number of keywords to extract from mmr algorithm
         (Default value = 5)
    mmr_diversity : level of diversity between keywords (higher value = keywords are more different from each other)
         (Default value = 0.7)

    References
    -------
    Towards Data Science Article by Maarten Grootendorst: Keyword Extraction with BERT
    https://towardsdatascience.com/keyword-extraction-with-bert-724efca412ea

    Returns
    -------
    Dataframe with added column for keywords for each algorithm

    """
    if event_type == 'Protests':
            stop_word_list = stop_words_protests
    elif event_type == 'Battles':
            stop_word_list = stop_words_battles
    initial_start = time.time()
    initial_start_readable = time.ctime(int(initial_start))
    print("Start time =", initial_start_readable)
    for row in range(len(data)):
        doc = data.iloc[row]['notes']
        #Extract candidate words/phrases
        count = CountVectorizer(preprocessor=preprocess_text, ngram_range=n_gram_range, stop_words=stop_word_list).fit([doc])
        candidates = count.get_feature_names()
        model = SentenceTransformer('distilbert-base-nli-mean-tokens')
        doc_embedding = model.encode([doc])
        candidate_embeddings = model.encode(candidates)
        distances = cosine_similarity(doc_embedding, candidate_embeddings)
        keywords = [candidates[index] for index in distances.argsort()[0][-keyword_n:]]
        max_sum_sim_out = max_sum_sim(doc_embedding = doc_embedding, word_embeddings=candidate_embeddings,
                                      words=candidates, top_n=mss_n, nr_candidates=mss_nr)
        mmr_out = mmr(doc_embedding=doc_embedding, word_embeddings=candidate_embeddings,
                      words=candidates, top_n=mmr_n, diversity=mmr_diversity)
        keywords = ', '.join(keywords)
        data.loc[data.index[row], 'keywords'] = keywords
        max_sum_sim_out = ', '.join(max_sum_sim_out)
        data.loc[data.index[row], 'max_sum_sim_out'] = max_sum_sim_out
        mmr_out = ', '.join(mmr_out)
        data.loc[data.index[row], 'mmr_out'] = mmr_out
    finish = time.time()
    finish_readable = time.ctime(int(finish))
    print("Total time elapsed:", (finish-initial_start), "seconds")
    print("End time =", finish_readable)

def run_KMeans(max_k, data):
    max_k += 1
    kmeans_results = dict()
    for k in range(2 , max_k):
        kmeans = cluster.KMeans(n_clusters = k
                               , init = 'k-means++'
                               , n_init = 10
                               , tol = 0.0001
                               , n_jobs = -1
                               , random_state = 1
                               , algorithm = 'full')

        kmeans_results.update( {k : kmeans.fit(data)} )

    return kmeans_results

def printAvg(avg_dict):
    for avg in sorted(avg_dict.keys(), reverse=True):
        print("Avg: {}\tK:{}".format(avg.round(4), avg_dict[avg]))

def plotSilhouette(df, n_clusters, kmeans_labels, silhouette_avg):
    fig, ax1 = plt.subplots(1)
    fig.set_size_inches(8, 6)
    ax1.set_xlim([-0.2, 1])
    ax1.set_ylim([0, len(df) + (n_clusters + 1) * 10])

    ax1.axvline(x=silhouette_avg, color="red", linestyle="--") # The vertical line for average silhouette score of all the values
    ax1.set_yticks([])  # Clear the yaxis labels / ticks
    ax1.set_xticks([-0.2, 0, 0.2, 0.4, 0.6, 0.8, 1])
    plt.title(("Silhouette analysis for K = %d" % n_clusters), fontsize=10, fontweight='bold')

    y_lower = 10
    sample_silhouette_values = silhouette_samples(df, kmeans_labels) # Compute the silhouette scores for each sample
    for i in range(n_clusters):
        ith_cluster_silhouette_values = sample_silhouette_values[kmeans_labels == i]
        ith_cluster_silhouette_values.sort()

        size_cluster_i = ith_cluster_silhouette_values.shape[0]
        y_upper = y_lower + size_cluster_i

        color = matplotlib.cm.nipy_spectral(float(i) / n_clusters)
        ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values, facecolor=color, edgecolor=color, alpha=0.7)

        ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i)) # Label the silhouette plots with their cluster numbers at the middle
        y_lower = y_upper + 10  # Compute the new y_lower for next plot. 10 for the 0 samples
    plt.show()

def silhouette(kmeans_dict, df, plot=False):
    df = df.to_numpy()
    avg_dict = dict()
    for n_clusters, kmeans in kmeans_dict.items():
        kmeans_labels = kmeans.predict(df)
        silhouette_avg = silhouette_score(df, kmeans_labels) # Average Score for all Samples
        avg_dict.update( {silhouette_avg : n_clusters} )
        printAvg(avg_dict)
        if(plot): plotSilhouette(df, n_clusters, kmeans_labels, silhouette_avg)

def centroidsDict(centroids, index):
    a = centroids.T[index].sort_values(ascending = False).reset_index().values
    centroid_dict = dict()

    for i in range(0, len(a)):
        centroid_dict.update( {a[i,0] : a[i,1]} )

    return centroid_dict

def generateWordClouds(centroids):
    wordcloud = WordCloud(max_font_size=100, background_color = 'white')
    for i in range(0, len(centroids)):
        centroid_dict = centroidsDict(centroids, i)
        wordcloud.generate_from_frequencies(centroid_dict)

        plt.figure(figsize=(10,20))
        plt.title('Cluster {}'.format(i))
        plt.imshow(wordcloud)
        plt.axis("off")
        plt.show()

def cluster_analysis(data, country, word_type, k, plot_silhouette=True, word_cloud=True, save=True):
    """
    This function performs cluster_analysis on the extracted keywords

    Parameters
    ----------
    data : pandas data frame containing acled data with the following columns:
            ['country', 'event_type'] and a column containing keywords

    country : country for which analysis is being performed in the data

    event_type : event type for which cluster analysis is being performed

    word_type : column in pandas dataframe that contains keywords to perform cluster analysis on

    k : max number of clusters 'k' to perform kMeans

    plot_silhouette :
         (Default value = True)
    word_cloud :
         (Default value = True)
    save :
         (Default value = True)

    References
    -------
    Medium article by Lucas de Sá: Text Clustering with K-Means
    https://medium.com/@lucasdesa/text-clustering-with-k-means-a039d84a941b
    https://github.com/lucas-de-sa/national-anthems-clustering/blob/master/Cluster_Anthems.ipynb

    Returns
    -------
    Silhouette Score plots and Word Clouds generated using kMeans cluster analysis
    """
    data = data[data['country']==country]
    data = data[word_type]
    #tf-idf analysis
    corpus = data
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(corpus)
    tf_idf = pd.DataFrame(data = X.toarray(), columns=vectorizer.get_feature_names())
    final_df = tf_idf

    # Running Kmeans
    kmeans_results = run_KMeans(k, final_df)

    # Plotting Silhouette Analysis
    if plot_silhouette==True:
        silhouette(kmeans_results, final_df, plot=True)

    selection=input("Best k: ")

    while not selection.isdigit:
        selection=input("Must be an integer. Try again: ")

    selection = int(selection)
    kmeans = kmeans_results.get(selection)

    if word_cloud==True:
        centroids = pd.DataFrame(kmeans.cluster_centers_)
        centroids.columns = final_df.columns
        generateWordClouds(centroids)

    if save==True:
        # Assigning the cluster labels to each row of data
        labels = kmeans.labels_
        data['keyword_label'] = labels

ext: py

reason: import numpy


text:
# coding: utf-8

# # Import Libraries

# In[85]:

# Wrangling
import pandas as pd
import numpy as np
# Plotting
import matplotlib.pyplot as plt
import seaborn as sns
import folium
import os

# In[2]:

# Set Plotting Styles
sns.set_style('whitegrid')
plt.rcParams['figure.figsize'] = 15,15

# In[3]:

from jupyterthemes import jtplot
jtplot.style(figsize=(15,15))

# # Import Datasets

# ### Original Datasets

# In[4]:

airports = pd.read_csv('original_datasets/airports.csv')

# In[5]:

carriers = pd.read_csv('original_datasets/carriers.csv')

# In[6]:

planes = pd.read_csv('original_datasets/plane-data.csv')

# In[7]:

oldData = pd.read_csv('original_datasets/1998.csv')

# In[8]:

newData = pd.read_csv('original_datasets/2006.csv')

# ### Modified Datasets (From Hive Code)

# In[9]:

byCarrier = pd.read_csv('generated_datasets/byCarr.csv')
byCord = pd.read_csv('generated_datasets/byCord.csv')
byModel = pd.read_csv('generated_datasets/bymodel.csv')
byManu = pd.read_csv('generated_datasets/bymyear.csv')
byState = pd.read_csv('generated_datasets/byState.csv')

# ### Subset Datasets

# In[10]:

oldCancel = oldData[oldData['Cancelled'] == 1]
newCancel = newData[newData['Cancelled'] == 1]

# # Plots

# ## Plots from 1998

# In[11]:

# Generate the Plot
sns.countplot(data=oldCancel, x='Month').set_title("Number of Cancellations by Month in 1998")
# Save the Plot
plt.savefig('Number of Cancellations by Month in 1998', dpi=100)

# ## Plots from 2006

# In[15]:

# Set Subplot Grids
f,ax=plt.subplots(1,2,figsize=(20,8))

# Generate the Plots
newCancel['CancellationCode'].value_counts().plot.pie(explode=[0.05,0.05,0.05,0.05],autopct='%1.1f%%',ax=ax[0],shadow=True)
ax[0].set_ylabel('')
sns.countplot('CancellationCode', order = newCancel['CancellationCode'].value_counts().index, data=newCancel, ax=ax[1])

# Save the Plots
plt.savefig('Distribution of Cancellation Codes')
print('A = carrier, B = weather, C = NAS, D = security')

# In[19]:

# Generate the Plot
sns.countplot(data=newCancel, x='Month', hue="CancellationCode").set_title("Number of Cancellations by Code in 2006")

# Save the Plot
plt.savefig("Number of Cancellations by Code in 2006")

# # Rate of Cancellation by Carrier

# In[21]:

# Generate the Plot
cancelRateCarrier = sns.barplot(data = byCarrier, x = 'year', y= byCarrier['num_cancl']/byCarrier['num_flight'] ,
                                        ci =None , hue = 'carrier')

# Set the Plot's title
cancelRateCarrier.set_title('Rate of Cancellation by Carrier')
# Rotate the Plot's X-axis Ticks
cancelRateCarrier.set_xticklabels(cancelRateCarrier.get_xticklabels(),
                                          rotation=40, ha="right")
plt.tight_layout()

#Save the Plot
plt.savefig("Rate of Cancellation by Carrier")

# In[84]:

# Subset the Dataset
cleanedManu = byManu[~((byManu.year == 1998) & (byManu.manu_year > 1998))]

# Generate Color Palette
palette = sns.color_palette("mako_r", 2)

# Generate the Plot
delayByManuYear = sns.lineplot(data = cleanedManu, x = 'manu_year',y= 'avg_dely',
                               hue='year', ci =None, estimator='mean', palette=palette)

# Set the Plot's Style
delayByManuYear.set_title('Mean Delay by Manufactured Year')
delayByManuYear.set_aspect(2.4)
plt.xlim(1956,2007)
delayByManuYear.legend()

# Save the Plot
plt.savefig("Mean Delay by Manufactured Year")
plt.show()

# # Map Plot for Cancellation Rate

# In[23]:

# Generate the Heatmap
plt.matshow(byManu.corr())

# Adjust Y and X labels and ticks
plt.xticks(range(len(byManu.columns)), byManu.columns)
plt.yticks(range(len(byManu.columns)), byManu.columns)

# Generate the Legend (Color Bar)
plt.colorbar()

# Save the Plot
plt.savefig("Correlation Heatmap")

# In[28]:

# Set Relative Path to the States Json File
state_geo = os.path.join('generated_datasets/', 'us-states.json')

# Initialize the map:
rateCancellationMap = folium.Map(location=[37, -102], zoom_start=5)

# Add the color for the chloropleth:
rateCancellationMap.choropleth(
 geo_data=state_geo,
 name='choropleth',
 data=byState,
 columns=['state', 'r_cancl'],
 key_on='feature.id',
 fill_color='YlGn',
 fill_opacity=0.7,
 line_opacity=0.2,
 legend_name='Cancellation Rate (%)'
)
folium.LayerControl().add_to(rateCancellationMap)

# Save to html
rateCancellationMap.save('rateCancellationMap.html')

# In[29]:

# Initialize the map:
rateDelayMap = folium.Map(location=[37, -102], zoom_start=5)

# Add the color for the chloropleth:
rateDelayMap.choropleth(
 geo_data=state_geo,
 name='choropleth',
 data=byState,
 columns=['state', 'r_dely'],
 key_on='feature.id',
 fill_color='YlGn',
 fill_opacity=0.7,
 line_opacity=0.2,
 legend_name='Delay Rate (%)'
)
folium.LayerControl().add_to(rateDelayMap)

# Save to html
rateDelayMap.save('rateDelayMap.html')

# In[30]:

# Initialize the map:
numFlightMap = folium.Map(location=[37, -102], zoom_start=5)

# Add the color for the chloropleth:
numFlightMap.choropleth(
 geo_data=state_geo,
 name='choropleth',
 data=byState,
 columns=['state', 'num_flight'],
 key_on='feature.id',
 fill_color='YlGn',
 fill_opacity=0.7,
 line_opacity=0.2,
 legend_name='Number of Flights'
)
folium.LayerControl().add_to(numFlightMap)

# Save to html
numFlightMap.save('numFlightMap.html')


text:
from models.common.basePortfolio import PortfolioModel
import scipy.optimize as opt
import numpy as np

class SurrogateMixPortfolio(PortfolioModel):

    def __init__(self, parameter, hiddenInfo):
        PortfolioModel.__init__(self, parameter=parameter, hiddenInfo=hiddenInfo)

    def __implicitFunction(self, x, weights, level):
        if self.hiddenInfo.FIRSTORDER:
            return np.inner(weights ** self.para.IND_REG, self.hiddenInfo.SCALES['first']) \
                   / (x ** self.para.IND_REG) - level
        else:
            if self.hiddenInfo.SINGLEFACTOR:
                return np.inner([w ** self.para.IND_REG for w in weights], self.hiddenInfo.SCALES['first']) / (x ** self.para.IND_REG * self.para.NUM_OF_ASSETS) + np.inner(weights, self.hiddenInfo.SCALES['second']) ** self.para.IND_HIDDEN / (x ** self.para.IND_HIDDEN) - 1 + level/100
            else:
                print('Not implemented!')
                return 0

    def calculateVaR(self, weights, level, start):
        result = opt.root(self.__implicitFunction, x0=start, args=(weights, level))
        return result.x[0]

    def calculateES(self, weights, level, start):
        VaR = self.calculateVaR(weights, level, start)
        return 100/(100 - level) * ( (sum([w ** self.para.IND_REG for w in weights]) * self.para.IND_REG)/(self.para.NUM_OF_ASSETS * (self.para.IND_REG - 1) * VaR ** (self.para.IND_REG - 1)) + self.para.IND_HIDDEN * np.inner(weights, self.hiddenInfo.SCALES['second']) ** self.para.IND_HIDDEN / ((self.para.IND_HIDDEN - 1) * VaR ** (self.para.IND_HIDDEN - 1)))

reason: import numpy,import scipy


text:
from contourpy import contour_generator
from contourpy.util.mpl_util import filled_to_mpl_paths, lines_to_mpl_paths
import matplotlib.pyplot as plt
import matplotlib.collections as mcollections
import numpy as np
import os
from pathlib import Path

# Configuration.
input_filename = 'coefficients.txt'
border = 0.05
nmodes = 8
z_mean = -0.03429101451261287

ngrid = 40
xborder = [0.03, 0.05]
yborder = [0.06, 0.02]
font_path = Path('../fonts/Catamaran-Bold.ttf')
svg_linewidth = 0.5
png_linewidth = 1.0

# Colors are colorbrewer RdYlGn palette.
colors = ["#1a9641", "#a6d96a", "#ffffbf", "#fdae61", "#d7191c"]

def full_output_filename(filename):
    return os.path.join('output', filename)

def plot_logo(ax, linewidth):
    cont_gen = contour_generator(x2, y2, z)
    for i in range(len(colors)):
        filled = cont_gen.filled(levels[i], levels[i+1])
        paths = filled_to_mpl_paths(filled, cont_gen.fill_type)
        collection = mcollections.PathCollection(
            paths, facecolors=colors[i], edgecolors='none', lw=0)
        ax.add_collection(collection)

    for level in levels:
        lines = cont_gen.lines(level)
        paths = lines_to_mpl_paths(lines, cont_gen.line_type)
        collection = mcollections.PathCollection(
            paths, facecolors='none', edgecolors='k', lw=linewidth)
        ax.add_collection(collection)

    ax.set_xlim(xborder[0], 1.0-xborder[1])
    ax.set_ylim(yborder[0], 1.0-yborder[1])
    ax.axis('off')

# Load Fourier coefficients from file.
coefficients = np.loadtxt(input_filename)

# Create z-field on quad grid from Fourier components.
x = np.linspace(xborder[0], 1.0-xborder[1], ngrid)
y = np.linspace(yborder[0], 1.0-yborder[1], ngrid)
x2, y2 = np.meshgrid(x, y)
xflat = x2.ravel()
yflat = y2.ravel()
A = np.empty((ngrid*ngrid, nmodes*nmodes))
for i in range(nmodes):
    for j in range(nmodes):
        A[:, i+j*nmodes] = np.sin((i+1)*np.pi*xflat) * np.sin((j+1)*np.pi*yflat)

z = (A*coefficients).sum(axis=1) + z_mean
z.shape = (ngrid, ngrid)
levels = np.linspace(0.0, z.max(), 6)

# Logo only, SVG.
fig = plt.figure(figsize=(1, 1), dpi=100)
ax = fig.add_axes([0, 0, 1, 1], aspect='equal')
plot_logo(ax, svg_linewidth)
filename = full_output_filename('contourpy_logo.svg')
print(f'Writing file {filename}')
fig.savefig(filename, transparent=True)

# Logo only, PNG files.
fig = plt.figure(figsize=(1, 1), dpi=100)
ax = fig.add_axes([0, 0, 1, 1], aspect='equal')
plot_logo(ax, png_linewidth)
for pixels in (200, 300):
    size = pixels / 100.0
    fig.set_size_inches(size, size)
    filename = full_output_filename(f'contourpy_logo_{pixels}.png')
    print(f'Writing file {filename}')
    fig.savefig(filename, transparent=True)

# PNG file with white border to fit in github circle.
fig = plt.figure(figsize=(3, 3), dpi=100)  # So 300 pixels.
ax = fig.add_axes([0, 0, 1, 1], aspect='equal')
plot_logo(ax, png_linewidth)
border = 0.1
ax.set_xlim(xborder[0]-border, 1.0-xborder[1]+border)
ax.set_ylim(yborder[0]-border, 1.0-yborder[1]+border)
filename = full_output_filename(f'contourpy_logo_{pixels}_border.png')
print(f'Writing file {filename}')
fig.savefig(filename, transparent=True)

# Logo and name, horizontal.
fig = plt.figure(figsize=(2.55, 0.75))
ax = fig.add_axes([0, 0, 1, 1], aspect='equal')
plot_logo(ax, svg_linewidth)
ax.text(0.95, 0.5, 'ContourPy', font=font_path, fontsize=28, ha='left', va='center_baseline', c="#000000")
ax.set_xlim(0.0, 3.4)
filename = full_output_filename('contourpy_logo_horiz.svg')
print(f'Writing file {filename}')
fig.savefig(filename, transparent=True)

# Logo and name, vertical.
fig = plt.figure(figsize=(2.0, 1.1))
ax = fig.add_axes([0, 0, 1, 1], aspect='equal')
plot_logo(ax, svg_linewidth)
ax.text(0.5, 0.05, 'ContourPy', font=font_path, fontsize=28, ha='center', va='top', c="#000000")
ax.set_ylim(-0.5, 1.0)
filename = full_output_filename('contourpy_logo_vert.svg')
print(f'Writing file {filename}')
fig.savefig(filename, transparent=True)


text:
import time
import sys
import os

#ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
#sys.path.append(ROOT)

from franka.ArmController import ArmController

import yaml
import numpy as np
from scipy.spatial.transform import Rotation as RR

from franka import RobotDriver

class FrankaController(ArmController):
    def __init__(self, robot_configuration_file_path):
        super(FrankaController, self).__init__()
        self._cfg = yaml.load(open(robot_configuration_file_path,'r'),Loader=yaml.FullLoader)
        robot_ip = self._cfg['ip']
        self.fk = RobotDriver.FrankaDriver(robot_ip)
        #F_T_EE: 4*4 Matrix from Flange to EE
        # this is setted in website
        F_T_EE = np.array(self.fk.getF_T_EE())
        self.F_Matrix = np.reshape(F_T_EE,(4,4),order='F')
        self.v = self._cfg['velocity']
        self.fk.setPositioningVelocity(self.v)

    def getMatrixO_T_EE(self):
        O_T_EE = np.array(self.fk.getO_T_EE())
        O_Matrix = np.reshape(O_T_EE,(4,4),order='F')
        O_Matrix_R = O_Matrix[:3, :3]
        O_Matrix_t = O_Matrix[:3, 3:].squeeze(1)

        return O_Matrix_R, O_Matrix_t

    def getCartesianPose(self):
        #O_T_EE: 4*4 Matrix from Base to EE
        O_T_EE = np.array(self.fk.getO_T_EE())
        O_Matrix = np.reshape(O_T_EE,(4,4),order='F')
        O_Matrix_R = O_Matrix[:3, :3]
        O_Matrix_t = O_Matrix[:3, 3:].squeeze(1)

        # For debugging on jetson
        # Add by JingYL 20200718
        #print("Origin O_Matrix: \n")
        #print(O_Matrix)
        #print("Rotation part: \n")
        #print(O_Matrix_R)

        #O_rotation = RR.from_matrix(O_Matrix)
        O_rotation = RR.from_matrix(O_Matrix_R)
        O_euler = O_rotation.as_euler('xyz', degrees=False)

        # #F_T_EE: 4*4 Matrix from Flange to EE
        # F_T_EE = np.array(self.fk.getF_T_EE())
        # F_Matrix = np.reshape(F_T_EE,(4,4),order='F')
        return O_Matrix_t, O_euler

    def getJoint(self):
        return self.fk.getJointPos()

    def getState(self):
        current_state = self.fk.getAllState()
        current_state = eval(current_state)
        return current_state

    def gripperOpen(self):
        self.fk.gripperOpen()

    def gripperGrasp(self,width=0.05,force=1):
        self.fk.gripperGrasp(width,force)

        # rad/s
    def speed_j(self,joint_speed):
        self.fk.speedJ(joint_speed)

    def stopSpeed(self):
        self.fk.stopMotion()

    def move_j(self, joints_angle, velocity=0.5, accelerate=0.6, solution_space='Joint'):
        self.fk.setJointPos(joints_angle)

    def move_p(self,position, velocity=0.5, accelerate=0.6, solution_space='Joint'):
        # frome euler to Matrix, TCP
        target_pose = np.array(position)
        temp_euler = RR.from_euler('xyz', target_pose[3:6], degrees=False)
        temp_matrix = temp_euler.as_matrix()
        target_matrix = np.eye(4)
        target_matrix[0:3,0:3] = temp_matrix
        target_matrix[0:3,3] = target_pose[0:3]
        # TCP to Flange
        O_T_F = np.dot(target_matrix,np.linalg.pinv(self.F_Matrix))

        #print("O_T_F:")
        #print(O_T_F)

        # inverse kinematics
        sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/Inverse_Kinematics')
        from franka import ikpy

        link_used = [False, True, True, True,True,True,True,False]
        robot_urdf_path = os.path.dirname(os.path.abspath(__file__)) + "/panda.urdf"
        my_chain = ikpy.chain.Chain.from_urdf_file(robot_urdf_path, active_links_mask = None)
        # base_joint and gripper_joint, so there are 9 joint angles
        current_joint = self.getJoint()
        init_joint = np.zeros(9)
        init_joint[1:8] = current_joint
        ik_joint = my_chain.inverse_kinematics(O_T_F,init_joint)
        target_joint = ik_joint[1:8]

        #
        self.fk.setJointPos(target_joint)

# if __name__ == '__main__':
#     FC = FrankaController(ROOT + '/config/robcell-panda1-default-d435/'+'franka.yaml')
#     print(FC.getJoint())
#     print(FC.getCartesianPose())
#     allState = FC.getState()
#     print(allState['F_T_EE'])
#     print(allState['O_T_EE'])

#     joint_target = np.array([-0.0137566,0.0150639,0.06416,-2.50988,-0.00736516,2.80153,-1.8411])
#     FC.move_j(joint_target)
#     pose = [0.5,0,0.4,3.14,0.0,0.0]
#     FC.move_p(pose)
#     # speed_j
#     joint_speed = [0,0,0,0,0,0,0.1]
#     FC.speed_j(joint_speed)
#     time.sleep(2)
#     FC.stopSpeed()

#     FC.move_p([0.5,0,0.3,3.14,0.0,0.0])
#     FC.move_p([0.6,0,0.4,3.14,0.0,1.0])
#     FC.move_p([0.6,0.3,0.2,3.14,0.0,1.0])

max_stars_repo_name: bionicdl-sustech/AmphibiousManipulation

max_issues_repo_name: bionicdl-sustech/AmphibiousManipulation

max_forks_repo_name: bionicdl-sustech/AmphibiousManipulation

reason: import numpy,from scipy


text:
import logging
import numpy as np
import tensorflow as tf

from rl.core import Policy

logger = logging.getLogger(__name__)

class LSTMVanillaPolicy(Policy):

    def __init__(self, environment, weights, action_sampler, log_directory, random_seed, learning_rate):
        self.environment = environment
        self.weights = weights
        self.action_sampler = action_sampler

        self.log_directory = log_directory
        self.random_seed = random_seed
        self.learning_rate = learning_rate

        self.observation_dimensions = np.prod(self.environment.observation_space.shape)
        self.num_actions = self.environment.action_space.n

        self.graph = tf.Graph()
        update_summaries = []

        with self.graph.as_default():

            tf.set_random_seed(self.random_seed)

            with tf.variable_scope('Observations'):
                self.observations_placeholder = tf.placeholder(shape=(None, None, self.observation_dimensions), dtype=tf.float32)

            with tf.variable_scope('Actions'):
                self.actions_placeholder = tf.placeholder(shape=(None, None,), dtype=tf.int32)

            with tf.variable_scope('Weights'):
                self.weights_placeholder = tf.placeholder(shape=(None, None,), dtype=tf.float32)

            with tf.variable_scope('Policy'):
                self.lstm = tf.keras.layers.LSTM(units=32, activation=tf.tanh, return_sequences=True)(self.observations_placeholder)
                self.hidden = tf.layers.Dense(units=32, activation=tf.tanh)(self.lstm)
                self.logits = tf.layers.Dense(units=self.num_actions, activation=None)(self.hidden)

            with tf.variable_scope('Sample'):
                self.probabilities = tf.nn.softmax(self.logits)

            with tf.variable_scope('Loss'):
                self.action_masks = tf.one_hot(indices=self.actions_placeholder, depth=self.num_actions)
                self.log_probabilities = tf.reduce_sum(self.action_masks * tf.broadcast_to(tf.nn.log_softmax(self.logits), shape=(200, 25, 3)), axis=2)
                self.psuedo_loss = -tf.reduce_mean(self.weights_placeholder * self.log_probabilities)
                update_summaries.append(tf.summary.scalar('PolicyLoss', self.psuedo_loss))

            with tf.variable_scope('Optimizer'):
                self.training_operation = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.psuedo_loss)

            self.update_summary = tf.summary.merge(inputs=update_summaries)

        self.session = tf.Session(graph=self.graph)
        self.writer = tf.summary.FileWriter(logdir=self.log_directory + "/policy", graph=self.graph)

    def __enter__(self):
        self.session.__enter__()
        self.writer.__enter__()
        self.session.run(tf.global_variables_initializer())
        self.session.run(tf.local_variables_initializer())
        self.weights.__enter__()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.weights.__exit__(exc_type, exc_val, exc_tb)
        self.writer.__exit__(exc_type, exc_val, exc_tb)
        self.session.__exit__(exc_type, exc_val, exc_tb)

    def get_action(self, observation):
        probabilities = self.session.run(
            fetches=self.probabilities,
            feed_dict={self.observations_placeholder: np.array(observation.reshape(1, 1, -1))})[0][0]
        return self.action_sampler.sample(probabilities)

    def update(self, epoch, episodes):
        _, summary = self.session.run(
            fetches=[self.training_operation, self.update_summary],
            feed_dict={
                self.observations_placeholder: self.get_batch_observations(episodes),
                self.actions_placeholder: self.get_batch_actions(episodes),
                self.weights_placeholder: self.weights.get_lstm_batch_weights(episodes)})
        self.writer.add_summary(summary, epoch)
        self.weights.update(epoch, episodes)

    def get_batch_observations(self, episodes):
        observations = np.array([[observation for observation in episode.observations] for episode in episodes])
        observations = np.swapaxes(observations, 0, 1)
        logger.debug("Obsevations shape: {}".format(observations.shape))
        return observations

    def get_batch_actions(self, episodes):
        actions = np.array([[action for action in episode.actions] for episode in episodes])
        actions = np.swapaxes(actions, 0, 1)
        logger.debug("Actions shape: {}".format(actions.shape))
        return actions

max_stars_repo_name: djjh/reinforcement-learning-labs

max_issues_repo_name: djjh/reinforcement-learning-labs

max_forks_repo_name: djjh/reinforcement-learning-labs


text:
# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
#
# NVIDIA CORPORATION and its licensors retain all intellectual property
# and proprietary rights in and to this software, related documentation
# and any modifications thereto.  Any use, reproduction, disclosure or
# distribution of this software and related documentation without an express
# license agreement from NVIDIA CORPORATION is strictly prohibited.

"""Compare average power spectra between real and generated images,
or between multiple generators."""

import os
import numpy as np
import torch
import torch.fft
import scipy.ndimage
import matplotlib.pyplot as plt
import click
import tqdm
import dnnlib

import legacy
from training import dataset

#----------------------------------------------------------------------------
# Setup an iterator for streaming images, in uint8 NCHW format, based on the
# respective command line options.

def stream_source_images(source, num, seed, device, data_loader_kwargs=None): # => num_images, image_size, image_iter
    ext = source.split('.')[-1].lower()
    if data_loader_kwargs is None:
        data_loader_kwargs = dict(pin_memory=True, num_workers=3, prefetch_factor=2)

    if ext == 'pkl':
        if num is None:
            raise click.ClickException('--num is required when --source points to network pickle')
        with dnnlib.util.open_url(source) as f:
            G = legacy.load_network_pkl(f)['G_ema'].to(device)
        def generate_image(seed):
            rnd = np.random.RandomState(seed)
            z = torch.from_numpy(rnd.randn(1, G.z_dim)).to(device)
            c = torch.zeros([1, G.c_dim], device=device)
            if G.c_dim > 0:
                c[:, rnd.randint(G.c_dim)] = 1
            return (G(z=z, c=c) * 127.5 + 128).clamp(0, 255).to(torch.uint8)
        _ = generate_image(seed) # warm up
        image_iter = (generate_image(seed + idx) for idx in range(num))
        return num, G.img_resolution, image_iter

    elif ext == 'zip' or os.path.isdir(source):
        dataset_obj = dataset.ImageFolderDataset(path=source, max_size=num, random_seed=seed)
        if num is not None and num != len(dataset_obj):
            raise click.ClickException(f'--source contains fewer than {num} images')
        data_loader = torch.utils.data.DataLoader(dataset_obj, batch_size=1, **data_loader_kwargs)
        image_iter = (image.to(device) for image, _label in data_loader)
        return len(dataset_obj), dataset_obj.resolution, image_iter

    else:
        raise click.ClickException('--source must point to network pickle, dataset zip, or directory')

#----------------------------------------------------------------------------
# Load average power spectrum from the specified .npz file and construct
# the corresponding heatmap for visualization.

def construct_heatmap(npz_file, smooth):
    npz_data = np.load(npz_file)
    spectrum = npz_data['spectrum']
    image_size = npz_data['image_size']
    hmap = np.log10(spectrum) * 10 # dB
    hmap = np.fft.fftshift(hmap)
    hmap = np.concatenate([hmap, hmap[:1, :]], axis=0)
    hmap = np.concatenate([hmap, hmap[:, :1]], axis=1)
    if smooth > 0:
        sigma = spectrum.shape[0] / image_size * smooth
        hmap = scipy.ndimage.gaussian_filter(hmap, sigma=sigma, mode='nearest')
    return hmap, image_size

#----------------------------------------------------------------------------

@click.group()
def main():
    """Compare average power spectra between real and generated images,
    or between multiple generators.

    Example:

    \b
    # Calculate dataset mean and std, needed in subsequent steps.
    python avg_spectra.py stats --source=~/datasets/ffhq-1024x1024.zip

    \b
    # Calculate average spectrum for the training data.
    python avg_spectra.py calc --source=~/datasets/ffhq-1024x1024.zip \\
        --dest=tmp/training-data.npz --mean=112.684 --std=69.509

    \b
    # Calculate average spectrum for a pre-trained generator.
    python avg_spectra.py calc \\
        --source=https://api.ngc.nvidia.com/v2/models/nvidia/research/stylegan3/versions/1/files/stylegan3-r-ffhq-1024x1024.pkl \\
        --dest=tmp/stylegan3-r.npz --mean=112.684 --std=69.509 --num=70000

    \b
    # Display results.
    python avg_spectra.py heatmap tmp/training-data.npz
    python avg_spectra.py heatmap tmp/stylegan3-r.npz
    python avg_spectra.py slices tmp/training-data.npz tmp/stylegan3-r.npz

    \b
    # Save as PNG.
    python avg_spectra.py heatmap tmp/training-data.npz --save=tmp/training-data.png --dpi=300
    python avg_spectra.py heatmap tmp/stylegan3-r.npz --save=tmp/stylegan3-r.png --dpi=300
    python avg_spectra.py slices tmp/training-data.npz tmp/stylegan3-r.npz --save=tmp/slices.png --dpi=300
    """

#----------------------------------------------------------------------------

@main.command()
@click.option('--source', help='Network pkl, dataset zip, or directory', metavar='[PKL|ZIP|DIR]', required=True)
@click.option('--num', help='Number of images to process  [default: all]', metavar='INT', type=click.IntRange(min=1))
@click.option('--seed', help='Random seed for selecting the images', metavar='INT', type=click.IntRange(min=0), default=0, show_default=True)
def stats(source, num, seed, device=torch.device('cuda')):
    """Calculate dataset mean and standard deviation needed by 'calc'."""
    torch.multiprocessing.set_start_method('spawn')
    num_images, _image_size, image_iter = stream_source_images(source=source, num=num, seed=seed, device=device)

    # Accumulate moments.
    moments = torch.zeros([3], dtype=torch.float64, device=device)
    for image in tqdm.tqdm(image_iter, total=num_images):
        image = image.to(torch.float64)
        moments += torch.stack([torch.ones_like(image).sum(), image.sum(), image.square().sum()])
    moments = moments / moments[0]

    # Compute mean and standard deviation.
    mean = moments[1]
    std = (moments[2] - moments[1].square()).sqrt()
    print(f'--mean={mean:g} --std={std:g}')

#----------------------------------------------------------------------------

@main.command()
@click.option('--source', help='Network pkl, dataset zip, or directory', metavar='[PKL|ZIP|DIR]', required=True)
@click.option('--dest', help='Where to store the result', metavar='NPZ', required=True)
@click.option('--mean', help='Dataset mean for whitening', metavar='FLOAT', type=float, required=True)
@click.option('--std', help='Dataset standard deviation for whitening', metavar='FLOAT', type=click.FloatRange(min=0), required=True)
@click.option('--num', help='Number of images to process  [default: all]', metavar='INT', type=click.IntRange(min=1))
@click.option('--seed', help='Random seed for selecting the images', metavar='INT', type=click.IntRange(min=0), default=0, show_default=True)
@click.option('--beta', help='Shape parameter for the Kaiser window', metavar='FLOAT', type=click.FloatRange(min=0), default=8, show_default=True)
@click.option('--interp', help='Frequency-domain interpolation factor', metavar='INT', type=click.IntRange(min=1), default=4, show_default=True)
def calc(source, dest, mean, std, num, seed, beta, interp, device=torch.device('cuda')):
    """Calculate average power spectrum and store it in .npz file."""
    torch.multiprocessing.set_start_method('spawn')
    num_images, image_size, image_iter = stream_source_images(source=source, num=num, seed=seed, device=device)
    spectrum_size = image_size * interp
    padding = spectrum_size - image_size

    # Setup window function.
    window = torch.kaiser_window(image_size, periodic=False, beta=beta, device=device)
    window *= window.square().sum().rsqrt()
    window = window.ger(window).unsqueeze(0).unsqueeze(1)

    # Accumulate power spectrum.
    spectrum = torch.zeros([spectrum_size, spectrum_size], dtype=torch.float64, device=device)
    for image in tqdm.tqdm(image_iter, total=num_images):
        image = (image.to(torch.float64) - mean) / std
        image = torch.nn.functional.pad(image * window, [0, padding, 0, padding])
        spectrum += torch.fft.fftn(image, dim=[2,3]).abs().square().mean(dim=[0,1])
    spectrum /= num_images

    # Save result.
    if os.path.dirname(dest):
        os.makedirs(os.path.dirname(dest), exist_ok=True)
    np.savez(dest, spectrum=spectrum.cpu().numpy(), image_size=image_size)

#----------------------------------------------------------------------------

@main.command()
@click.argument('npz-file', nargs=1)
@click.option('--save', help='Save the plot and exit', metavar='[PNG|PDF|...]')
@click.option('--dpi', help='Figure resolution', metavar='FLOAT', type=click.FloatRange(min=1), default=100, show_default=True)
@click.option('--smooth', help='Amount of smoothing', metavar='FLOAT', type=click.FloatRange(min=0), default=1.25, show_default=True)
def heatmap(npz_file, save, smooth, dpi):
    """Visualize 2D heatmap based on the given .npz file."""
    hmap, image_size = construct_heatmap(npz_file=npz_file, smooth=smooth)

    # Setup plot.
    plt.figure(figsize=[6, 4.8], dpi=dpi, tight_layout=True)
    freqs = np.linspace(-0.5, 0.5, num=hmap.shape[0], endpoint=True) * image_size
    ticks = np.linspace(freqs[0], freqs[-1], num=5, endpoint=True)
    levels = np.linspace(-40, 20, num=13, endpoint=True)

    # Draw heatmap.
    plt.xlim(ticks[0], ticks[-1])
    plt.ylim(ticks[0], ticks[-1])
    plt.xticks(ticks)
    plt.yticks(ticks)
    plt.contourf(freqs, freqs, hmap, levels=levels, extend='both', cmap='Blues')
    plt.gca().set_aspect('equal')
    plt.colorbar(ticks=levels)
    plt.contour(freqs, freqs, hmap, levels=levels, extend='both', linestyles='solid', linewidths=1, colors='midnightblue', alpha=0.2)

    # Display or save.
    if save is None:
        plt.show()
    else:
        if os.path.dirname(save):
            os.makedirs(os.path.dirname(save), exist_ok=True)
        plt.savefig(save)

#----------------------------------------------------------------------------

@main.command()
@click.argument('npz-files', nargs=-1, required=True)
@click.option('--save', help='Save the plot and exit', metavar='[PNG|PDF|...]')
@click.option('--dpi', help='Figure resolution', metavar='FLOAT', type=click.FloatRange(min=1), default=100, show_default=True)
@click.option('--smooth', help='Amount of smoothing', metavar='FLOAT', type=click.FloatRange(min=0), default=0, show_default=True)
def slices(npz_files, save, dpi, smooth):
    """Visualize 1D slices based on the given .npz files."""
    cases = [dnnlib.EasyDict(npz_file=npz_file) for npz_file in npz_files]
    for c in cases:
        c.hmap, c.image_size = construct_heatmap(npz_file=c.npz_file, smooth=smooth)
        c.label = os.path.splitext(os.path.basename(c.npz_file))[0]

    # Check consistency.
    image_size = cases[0].image_size
    hmap_size = cases[0].hmap.shape[0]
    if any(c.image_size != image_size or c.hmap.shape[0] != hmap_size for c in cases):
        raise click.ClickException('All .npz must have the same resolution')

    # Setup plot.
    plt.figure(figsize=[12, 4.6], dpi=dpi, tight_layout=True)
    hmap_center = hmap_size // 2
    hmap_range = np.arange(hmap_center, hmap_size)
    freqs0 = np.linspace(0, image_size / 2, num=(hmap_size // 2 + 1), endpoint=True)
    freqs45 = np.linspace(0, image_size / np.sqrt(2), num=(hmap_size // 2 + 1), endpoint=True)
    xticks0 = np.linspace(freqs0[0], freqs0[-1], num=9, endpoint=True)
    xticks45 = np.round(np.linspace(freqs45[0], freqs45[-1], num=9, endpoint=True))
    yticks = np.linspace(-50, 30, num=9, endpoint=True)

    # Draw 0 degree slice.
    plt.subplot(1, 2, 1)
    plt.title('0\u00b0 slice')
    plt.xlim(xticks0[0], xticks0[-1])
    plt.ylim(yticks[0], yticks[-1])
    plt.xticks(xticks0)
    plt.yticks(yticks)
    for c in cases:
        plt.plot(freqs0, c.hmap[hmap_center, hmap_range], label=c.label)
    plt.grid()
    plt.legend(loc='upper right')

    # Draw 45 degree slice.
    plt.subplot(1, 2, 2)
    plt.title('45\u00b0 slice')
    plt.xlim(xticks45[0], xticks45[-1])
    plt.ylim(yticks[0], yticks[-1])
    plt.xticks(xticks45)
    plt.yticks(yticks)
    for c in cases:
        plt.plot(freqs45, c.hmap[hmap_range, hmap_range], label=c.label)
    plt.grid()
    plt.legend(loc='upper right')

    # Display or save.
    if save is None:
        plt.show()
    else:
        if os.path.dirname(save):
            os.makedirs(os.path.dirname(save), exist_ok=True)
        plt.savefig(save)

#----------------------------------------------------------------------------

if __name__ == "__main__":
    main() # pylint: disable=no-value-for-parameter

#----------------------------------------------------------------------------

max_stars_repo_name: KeepNoob/PokemonGAN

max_issues_repo_name: KeepNoob/PokemonGAN

max_forks_repo_name: KeepNoob/PokemonGAN


text:
from adjust_data import save, load
from tqdm import tqdm
from glob import glob
import os
import numpy as np

def remove_batch_ids(data: dict, batch_ids: set, inplace: bool = True, modify_batch_ids: bool = True) -> dict:
    if len(batch_ids) == 0:
        return data
    if not modify_batch_ids:
        batch_ids = batch_ids.copy()
    # create list of indices which have to be removed
    indices_to_remove = []
    for i, (batch_id,) in enumerate(data['batch_ids']):
        if batch_id in batch_ids:
            indices_to_remove.append(i)
            batch_ids.remove(batch_id)
    # reverse list of indices to enable easy removing of items of a list by consecutive pops
    indices_to_remove.reverse()
    if inplace:
        new_data = data
    else:
        new_data = {}
    new_data['input'] = np.delete(data['input'], indices_to_remove, axis=0)
    ids: list = data['batch_ids'].copy()
    for i in indices_to_remove:
        ids.pop(i)
    new_data['batch_ids'] = ids
    new_data['batch_comp'] = np.delete(data['batch_comp'], indices_to_remove, axis=0)
    if not inplace:
        new_data['target'] = {}
    for target in data['target']:
        new_data['target'][target] = np.delete(data['target'][target], indices_to_remove, axis=0)
    new_data['comps'] = np.delete(data['comps'], indices_to_remove, axis=0)

    return new_data

def get_ids(file):
    data = load(file)
    return set([batch_id for batch_id, in data['batch_ids']])

def get_test_and_val_ids(path_to_dir):
    files = glob(os.path.join(path_to_dir, 'val', '*.pickle.gz')) + \
            glob(os.path.join(path_to_dir, 'test', '*.pickle.gz'))
    ids = set()
    for file in tqdm(files):
        ids |= get_ids(file)
    return ids

def main():
    path = 'graph_embeddings'
    target_dir = os.path.join(path, 'train')
    print('Gathering ids...')
    test_and_val_ids = get_test_and_val_ids(path)

    print('Deleting test and validation entries form training data...')
    files = glob(os.path.join(path, '*.pickle.gz'))
    if not os.path.isdir(target_dir):
        os.makedirs(target_dir)
    for file in tqdm(files):
        data = load(file)
        data = remove_batch_ids(data, test_and_val_ids)
        save(data, os.path.join(target_dir, os.path.basename(file)))

if __name__ == '__main__':
    main()

max_stars_repo_name: noah-hoffmann/CGAT

max_issues_repo_name: noah-hoffmann/CGAT

max_forks_repo_name: noah-hoffmann/CGAT


text:
'''
#Example script to generate text from Nietzsche's writings.

At least 20 epochs are required before the generated text
starts sounding coherent.

It is recommended to run this script on GPU, as recurrent
networks are quite computationally intensive.

If you try this script on new data, make sure your corpus
has at least ~100k characters. ~1M is better.

You can try some other texts too:

What about Tolstoys Anna Karenina:
https://raw.githubusercontent.com/udacity/deep-learning/master/tensorboard/anna.txt

Or some Nietzsche:
https://s3.amazonaws.com/text-datasets/nietzsche.txt

Germany Wikipedia Articles:
https://www2.htw-dresden.de/~guhr/dist/wiki.txt

Shakesspears Sonnets:
https://raw.githubusercontent.com/vivshaw/shakespeare-LSTM/master/sonnets.txt
'''

from __future__ import print_function
from keras.callbacks import LambdaCallback, TensorBoard
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM, CuDNNLSTM, CuDNNGRU, Dropout
from keras.optimizers import RMSprop, SGD, Nadam
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys
import io
from datetime import datetime
import re

path = get_file(
    'shakespear.txt',
    origin='https://cs.stanford.edu/people/karpathy/char-rnn/shakespear.txt')

with io.open(path, encoding='utf-8') as f:
    text = f.read().lower()
print('corpus length:', len(text))

# build lookup table
chars = sorted(list(set(text)))
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
# How does the network react when you change the sequence length or stepsize
maxlen = 40
step = 3
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        x[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1

# build the model: a single LSTM layer
# experiment:
# - add some more neurons
# - add some more layers
# - add dropout
# - try out GRU's

print('Build model...')
model = Sequential()
model.add(CuDNNLSTM(128,input_shape=(maxlen, len(chars))))
model.add(Dense(len(chars), activation='softmax'))

rms = RMSprop(lr=0.01)
# try some other optimizers
#sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
#nadam = Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004)
model.compile(loss='categorical_crossentropy', optimizer=rms)

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    # Read more about this softmax with temperature here:
    # Distilling the Knowledge in a Neural Network (Geoffrey Hinton, Oriol Vinyals, Jeff Dean)
    # https://arxiv.org/abs/1503.02531
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

def on_epoch_end(epoch, _):
    # Function invoked at end of each epoch. Prints generated text.
    print()
    print('----- Generating text after Epoch: %d' % epoch)

    start_index = random.randint(0, len(text) - maxlen - 1)
    for diversity in [0.8, 1.0, 1.2]:
        print('----- diversity:', diversity)

        generated = ''
        sentence = text[start_index: start_index + maxlen]
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')
        sys.stdout.write(generated)
        sys.stdout.write("\n----- result ------\n")
        for i in range(300):
            x_pred = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(sentence):
                x_pred[0, t, char_indices[char]] = 1.

            preds = model.predict(x_pred, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = indices_char[next_index]

            sentence = sentence[1:] + next_char

            sys.stdout.write(next_char)
            sys.stdout.flush()
        print()

# print some text with the current model
print_callback = LambdaCallback(on_epoch_end=on_epoch_end)

# train the model
model.fit(x, y,
          batch_size=128,
          epochs=90,
          callbacks=[print_callback])

# save the model
model.save("shakespear-rnn")

max_stars_repo_name: oliverguhr/htw-nlp-lecture

max_issues_repo_name: oliverguhr/htw-nlp-lecture

max_forks_repo_name: oliverguhr/htw-nlp-lecture


text:
import os

import numpy

import theano
from theano import config

# This is a big hack to avoid creating a second context on the card.
from theano.sandbox.cuda.nvcc_compiler import (NVCC_compiler as NVCC_base,
                                               hash_from_file)

class NVCC_compiler(NVCC_base):
    @staticmethod
    def compile_args():
        """
        Re-implementation of compile_args that does not create an
        additionnal context on the GPU.
        """
        flags = [flag for flag in config.nvcc.flags.split(' ') if flag]
        if config.nvcc.fastmath:
            flags.append('-use_fast_math')
        cuda_ndarray_cuh_hash = hash_from_file(
            os.path.join(os.path.split(theano.sandbox.cuda.__file__)[0],
                         'cuda_ndarray.cuh'))
        flags.append('-DCUDA_NDARRAY_CUH=' + cuda_ndarray_cuh_hash)

        # numpy 1.7 deprecated the following macros but they didn't
        # exist in the past
        numpy_ver = [int(n) for n in numpy.__version__.split('.')[:2]]
        if bool(numpy_ver < [1, 7]):
            flags.append("-DNPY_ARRAY_ENSURECOPY=NPY_ENSURECOPY")
            flags.append("-DNPY_ARRAY_ALIGNED=NPY_ALIGNED")
            flags.append("-DNPY_ARRAY_WRITEABLE=NPY_WRITEABLE")
            flags.append("-DNPY_ARRAY_UPDATE_ALL=NPY_UPDATE_ALL")
            flags.append("-DNPY_ARRAY_C_CONTIGUOUS=NPY_C_CONTIGUOUS")
            flags.append("-DNPY_ARRAY_F_CONTIGUOUS=NPY_F_CONTIGUOUS")

        # If the user didn't specify architecture flags add them
        if not any(['-arch=sm_' in f for f in flags]):
            dev = theano.sandbox.gpuarray.init_dev.device
            if dev is None:
                raise Exception("Trying to compile GPU code without a context")
            if dev.startswith("opencl"):
                raise Exception("Trying to call nvcc with an OpenCL context")
            assert dev.startswith('cuda')
            if dev == 'cuda':
                n = theano.sandbox.cuda.use.device_number
            else:
                n = int(dev[4:])
            p = theano.sandbox.cuda.device_properties(n)
            flags.append('-arch=sm_' + str(p['major']) + str(p['minor']))

        return flags

max_stars_repo_name: jych/Theano

max_issues_repo_name: mayunpeng/Theano

max_forks_repo_name: mayunpeng/Theano

reason: import numpy,import theano,from theano


text:
from marble_coaster import calc_cost, calc_length, solve_track
from MultiObjectiveOptimizer import MultiObjectiveOptimizer
from numpy import random
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import sys
sys.setrecursionlimit(200)

plt.ion()
fig = plt.figure(1)
# fig = plt.figure(2)
ax = fig.add_subplot(111)
# ax2 = fig.add_subplot(122)
ax.autoscale_view(True, True, True)
# ax2.autoscale_view(True, True, True)
plt.show()

def plot_front(generation):
    plt.ion()
    if len(ax.lines) > 40:
        del ax.lines[0]
    ax.plot(-generation[:, 1], generation[:, 2], 'o')
    plt.xlabel('Length')
    plt.ylabel('Cost')
    plt.title('Marble Coaster Pareto Front')
    ax.relim()
    plt.pause(0.01)
    plt.show()

def check_design(design):
    gene_per_section = 2

    num_div_x = 3
    num_div_y = 3
    num_div_z = 3

    # try:

    design = design.astype(int)
    t_length, t_cost, p_list, p_loc, r_list, e_list = solve_track(design)

    return [-t_length, t_cost]

if __name__ == "__main__":
    # thread = threading.Thread(target=opt_coaster)
    # thread.start()
    design_space = []
    for i in range(10**3):
        design_space.append({'type': 'integer', 'bounds': (0, 5)})
        design_space.append({'type': 'integer', 'bounds': (0, 4)})
    optimizer = MultiObjectiveOptimizer(design_space, check_design, n_generations=1000, population_size=200,
                                        n_objectives=2, generation_func=plot_front)
    opts = optimizer.find_min()

    np.savetxt('optmimum_coasters.csv', opts, delimiter=',')
    plt.savefig("pareto_front.pdf")

reason: import numpy,from numpy


text:
from pathlib import Path
import time
from typing import List, Any, Tuple
import logging

import hydra
from omegaconf import DictConfig, OmegaConf
import numpy as np
import cv2

from img_proc.np import preprocess
from utils import log
from utils.util import fix_seed
from utils.trt import (
    load_engine,
    allocate_buffers,
    do_inference_v2,
    load_data,
)
from consts.imagenet_labels import IMAGENET_LABELS

OUTPUT_DIR = "../images_out"

def normPRED(d: np.ndarray) -> np.ndarray:
    ma = np.max(d)
    mi = np.min(d)
    dn = (d - mi) / (ma - mi)
    return dn

def postprocess(
    res: List[np.ndarray],
    input_shape: Tuple[int, int],
    resize_shape: Tuple[int, int],
    is_portrait: bool = False,
) -> np.ndarray:
    pred = res[0].reshape((1, *input_shape))
    if is_portrait:
        pred = 1.0 - pred
    pred = normPRED(pred) * 255
    pred = pred.transpose((1, 2, 0)).astype(np.uint8)
    out_img = cv2.resize(pred, resize_shape)
    return out_img

@hydra.main(config_path="../configs", config_name="u2net")
def main(cfg: DictConfig) -> None:
    fix_seed(cfg.general.seed)
    is_portrait = True if cfg.u2net.mode == "portrait" else False
    engine_path = Path(cfg.u2net.engine_path)
    img_path = Path(cfg.general.image_path)
    out_dir_path = Path(OUTPUT_DIR)
    out_dir_path.mkdir(parents=True, exist_ok=True)
    if not engine_path.is_file():
        raise Exception(f"File Not Found. {str(engine_path)}")

    engine = load_engine(engine_path)
    context = engine.create_execution_context()
    inputs, outputs, bindings, stream = allocate_buffers(engine)

    img = cv2.imread(str(img_path))
    input_data = preprocess(
        img,
        cfg.u2net.input_shape,
        devide_max=True,
    )
    # 1st inference
    start = time.time()
    load_data(input_data, inputs[0].host)
    _ = do_inference_v2(
        context=context,
        bindings=bindings,
        inputs=inputs,
        outputs=outputs,
        stream=stream,
    )
    logging.info(f"1st inference time: {time.time() - start} [sec]")

    # 2nd inference
    start = time.time()
    load_data(input_data, inputs[0].host)
    res = do_inference_v2(
        context=context,
        bindings=bindings,
        inputs=inputs,
        outputs=outputs,
        stream=stream,
    )
    out_img = postprocess(
        res,
        input_shape=cfg.u2net.input_shape,
        resize_shape=(img.shape[1], img.shape[0]),
        is_portrait=is_portrait,
    )
    logging.info(f"2nd inference time: {time.time() - start} [sec]")

    # save image
    cv2.imwrite(str(out_dir_path / img_path.name), out_img)

if __name__ == "__main__":
    main()


text:
import unittest

import numpy as np

from autosklearn.pipeline.components.feature_preprocessing.select_percentile_regression \
    import SelectPercentileRegression
from autosklearn.pipeline.util import _test_preprocessing, get_dataset

class SelectPercentileRegressionTest(unittest.TestCase):
    def test_default_configuration(self):
        transformation, original = _test_preprocessing(
            dataset="boston",
            Preprocessor=SelectPercentileRegression,
            )
        self.assertEqual(transformation.shape[0], original.shape[0])
        self.assertEqual(transformation.shape[1], int(original.shape[1]/2))
        self.assertFalse((transformation == 0).all())

    def test_preprocessing_dtype(self):
        # Dense
        # np.float32
        X_train, Y_train, X_test, Y_test = get_dataset("iris")
        self.assertEqual(X_train.dtype, np.float32)

        configuration_space = SelectPercentileRegression.get_hyperparameter_search_space()
        default = configuration_space.get_default_configuration()
        preprocessor = SelectPercentileRegression(random_state=1,
                                                  **{hp_name: default[hp_name]
                                                     for hp_name in default})
        preprocessor.fit(X_train, Y_train)
        Xt = preprocessor.transform(X_train)
        self.assertEqual(Xt.dtype, np.float32)

        # np.float64
        X_train, Y_train, X_test, Y_test = get_dataset("iris")
        X_train = X_train.astype(np.float64)
        configuration_space = SelectPercentileRegression.get_hyperparameter_search_space()
        default = configuration_space.get_default_configuration()
        preprocessor = SelectPercentileRegression(random_state=1,
                                                  **{hp_name: default[hp_name]
                                                     for hp_name in default})
        preprocessor.fit(X_train, Y_train)
        Xt = preprocessor.transform(X_train)
        self.assertEqual(Xt.dtype, np.float64)

max_stars_repo_name: Louquinze/auto-sklearn

max_issues_repo_name: Louquinze/auto-sklearn

max_forks_repo_name: Louquinze/auto-sklearn


text:
import argparse
import torch
import sys
import numpy as np
import gin.tf
gin.enter_interactive_mode()
import time
import os
import json
import traceback
from scripts.model import reparametrize
from scripts.model import BetaVAE_H as BetaVAE
from disentanglement_lib.utils import results

# needed later:
from disentanglement_lib.evaluation.metrics import beta_vae  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import dci  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import downstream_task  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import factor_vae  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import fairness  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import irs  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import mig  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import modularity_explicitness  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import reduced_downstream_task  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import sap_score  # pylint: disable=unused-import
from disentanglement_lib.evaluation.metrics import unsupervised_metrics  # pylint: disable=unused-import

import mcc_metric.metric as mcc  # pylint: disable=unused-import

def main(args, dataset):
	device = torch.device('cuda' if torch.cuda.is_available() and args.cuda else 'cpu')
	net = BetaVAE
	net = net(args.z_dim, args.num_channel, args.pcl).to(device)
	file_path = os.path.join(args.ckpt_dir, 'last')
	checkpoint = torch.load(file_path)
	net.load_state_dict(checkpoint['model_states']['net'])

	def mean_rep(x):
		distributions = net._encode(
			torch.from_numpy(x).float().to(device))
		mu = distributions[:, :net.z_dim]
		logvar = distributions[:, net.z_dim:]
		return np.array(mu.detach().cpu())

	def sample_rep(x):
		distributions = net._encode(
			torch.from_numpy(x).float().to(device))
		mu = distributions[:, :net.z_dim]
		logvar = distributions[:, net.z_dim:]
		return np.array(reparametrize(mu, logvar).detach().cpu())

	@gin.configurable("evaluation")
	def evaluate(post,
				 output_dir,
				 evaluation_fn=gin.REQUIRED,
				 random_seed=gin.REQUIRED,
				 name=""):
		experiment_timer = time.time()
		assert post == 'mean' or post == 'sampled'
		results_dict = evaluation_fn(
			dataset,
			mean_rep if post == 'mean' else sample_rep,
			random_state=np.random.RandomState(random_seed))
		results_dict["elapsed_time"] = time.time() - experiment_timer
		results.update_result_directory(output_dir, "evaluation", results_dict)

	random_state = np.random.RandomState(0)
	config_dir = 'metric_configs'
	eval_config_files = [f for f in os.listdir(config_dir) if not (f.startswith('.') or 'others' in f)]
	t0 = time.time()
	posts = ['mean']
	for post in posts:
		for eval_config in eval_config_files:
			metric_name = os.path.basename(eval_config).replace(".gin", "")
			continuous = False
			if args.dataset == 'kittimasks' or (
					args.dataset == 'natural' and not args.natural_discrete):
				continuous = True
			if continuous:
				if metric_name != 'mcc':
					continue
			contains = True
			if args.specify:
				contains = False
				for specific in args.specify.split('_'):
					if specific in metric_name:
						contains = True
						break
			if contains:
				if args.verbose:
					print("Computing metric '{}' on '{}'...".format(metric_name, post))
				eval_bindings = [
					"evaluation.random_seed = {}".format(random_state.randint(2 ** 32)),
					"evaluation.name = '{}'".format(metric_name)
				]
				gin.parse_config_files_and_bindings(
					[os.path.join(config_dir, eval_config)], eval_bindings)
				output_dir = os.path.join(
					args.output_dir, 'evaluation', args.ckpt_name, post, metric_name)
				evaluate(post, output_dir)
				gin.clear_config()
				if args.verbose:
					print('took', time.time() - t0, 's')
			t0 = time.time()