text:
##############################################################################################################
# ███████╗███████╗███████╗██╗░█████╗░██╗███████╗███╗░░██╗████████╗
# ██╔════╝██╔════╝██╔════╝██║██╔══██╗██║██╔════╝████╗░██║╚══██╔══╝
# █████╗░░█████╗░░█████╗░░██║██║░░╚═╝██║█████╗░░██╔██╗██║░░░██║░░░
# ██╔══╝░░██╔══╝░░██╔══╝░░██║██║░░██╗██║██╔══╝░░██║╚████║░░░██║░░░
# ███████╗██║░░░░░██║░░░░░██║╚█████╔╝██║███████╗██║░╚███║░░░██║░░░
# ╚══════╝╚═╝░░░░░╚═╝░░░░░╚═╝░╚════╝░╚═╝╚══════╝╚═╝░░╚══╝░░░╚═╝░░░

# ███████╗██████╗░░█████╗░███╗░░██╗████████╗██╗███████╗██████╗░
# ██╔════╝██╔══██╗██╔══██╗████╗░██║╚══██╔══╝██║██╔════╝██╔══██╗
# █████╗░░██████╔╝██║░░██║██╔██╗██║░░░██║░░░██║█████╗░░██████╔╝
# ██╔══╝░░██╔══██╗██║░░██║██║╚████║░░░██║░░░██║██╔══╝░░██╔══██╗
# ██║░░░░░██║░░██║╚█████╔╝██║░╚███║░░░██║░░░██║███████╗██║░░██║
# ╚═╝░░░░░╚═╝░░╚═╝░╚════╝░╚═╝░░╚══╝░░░╚═╝░░░╚═╝╚══════╝╚═╝░░╚═╝
##############################################################################################################

##############################################################################################################
# Portfolio Optimization with Python using Efficient Frontier with Practical Examples
##############################################################################################################

# Portfolio optimization is the process of creating a portfolio of assets, for which your investment has the maximum return and minimum risk.
# Modern Portfolio Theory (MPT), or also known as mean-variance analysis is a mathematical process which allows the user to maximize returns for a given risk level.
# It was formulated by H. Markowitz and while it is not the only optimization technique known, it is the most widely used.

# Efficient frontier is a graph with ‘returns’ on the Y-axis and ‘volatility’ on the X-axis.
# It shows the set of optimal portfolios that offer the highest expected return for a given risk level or the lowest risk for a given level of expected return.

# In this example, we are considering a portfolio made up of stocks from just 2 companies, Tesla and Facebook.

##############################################################################################################
# Step 1: Pull the stock price data
##############################################################################################################
# Load Packages
import numpy as np
import pandas as pd
from pandas_datareader import data
import matplotlib.pyplot as plt

# The following is for JupiterNotebook
# %matplotlib inline

# Read Data
test = data.DataReader(['TSLA', 'FB'], 'yahoo', start='2018/01/01', end='2019/12/31')
print(test.head())

# The ‘Adj Close’ column. This colum gives us the closing price of company’s stock on the given day.
# Closing price
test = test['Adj Close']
print(test.head())

##############################################################################################################
# Step 2: Calculate percentage change in stock prices
##############################################################################################################
# Next, we calculate the percentage change in stock prices of tesla everyday. You will notice that that we take the log of percentage change.
# Why take log? The reason for this is that log of the returns is time additive. Then, log(r13) = log(r12) + log(r23)

# It is common practice in portfolio optimization to take log of returns for calculations of covariance and correlation.

# Log of percentage change for Tesla
tesla = test['TSLA'].pct_change().apply(lambda x: np.log(1+x))
print(tesla.head())

##############################################################################################################
# Variance
##############################################################################################################

# Variance
var_tesla = tesla.var()
print(var_tesla)
# 0.0011483734269334596

# Log of Percentage change for Facebook
fb = test['FB'].pct_change().apply(lambda x: np.log(1+x))
print(fb.head())

# Variance
var_fb = fb.var()
print(var_fb)
#> .00045697258417022536

# But volatility for the annual standard deviation.
# What we get from square root of variance is the daily standard deviation.
# To convert it to annual standard deviation we multiply the variance by 250.
# 250 is used because there are 250 trading days in a year.

# In US the trading days in a year are 250 (this may be different in UK)
trading_days_year = 250

##############################################################################################################
# Volatility
##############################################################################################################

# Volatility
tesla_vol = np.sqrt(var_tesla * trading_days_year)
fb_vol = np.sqrt(var_fb * trading_days_year)
print(tesla_vol, fb_vol)
#> .5358109337568289  .33799873674698305

# Volatility of both stocks
test.pct_change().apply(lambda x: np.log(1+x)).std().apply(lambda x: x*np.sqrt(trading_days_year)).plot(kind='bar')
plt.show()

##############################################################################################################
# Covariance
##############################################################################################################

# Covariance measures the directional relationship between the returns on two assets.
# A positive covariance means that returns of the two assets move together while a negative covariance means they move inversely.
# Risk and volatility can be reduced in a portfolio by pairing assets that have a negative covariance.

# Log of Percentage change
test1 = test.pct_change().apply(lambda x: np.log(1+x))
print(test1.head())

# Covariance
test1['TSLA'].cov(test1['FB'])
#> .00018261623156030972
print(test1['TSLA'].cov(test1['FB']))

##############################################################################################################
# Correlation
##############################################################################################################
# Correlation, in the finance and investment industries, is a statistic that measures the degree to which two securities move in relation to each other.
# Correlations are used in advanced portfolio management, computed as the correlation coefficient, which has a value that must fall between -1.0 and +1.0.

# A correlation of -1 means negative relation, i.e, if correlation between Asset A and Asset B is -1, if Asset A increases, Asset B decreases.
# A correlation of +1 means positive relation, i.e, if correlation between Asset A and Asset B is 1, if Asset A increases, Asset B increases.

# Correlation
test1['TSLA'].corr(test1['FB'])
#> .2520883272466132
print(test1['TSLA'].corr(test1['FB']))

##############################################################################################################
# Expected returns
##############################################################################################################
# Expected returns of an asset are simply the mean of percentage change in its stock prices.
# So, the value of expected return we obtain here are daily expected returns.
# For an yearly expected return value, you will need to resample the data year-wise, as you will see further.

# For expected returns, you need to define weights for the assets choosen.
# In simpler terms, this means you need to decide what percentage of your total money to you want to hold in each company’s stock.
# Usually this decision is done by using the optimization techniques we will discuss later but for now we will consider random weights for Tesla and Facebook.

# First, let’s compute the log of percentage change.
test2 = test.pct_change().apply(lambda x: np.log(1+x))
print(test2.head())

# Let’s define an array of random weights for the purpose of calculation.
# These weights will represent the percentage allocation of investments between these two stocks.
# They must add up to 1.

# Define weights for allocation
w = [0.2, 0.8]
# Expected returns of the single assets
e_r_ind = test2.mean()
print(e_r_ind)

# Total expected return
e_r = (e_r_ind*w).sum()
print(e_r)
#> .0003027691524101118

##############################################################################################################
# Building an optimal risky portfolio
##############################################################################################################
# We will be using stocks from 4 companies, namely, Apple, Nike, Google and Amazon for a period of 5 years.
# You will learn to calculate the weights of assets for each one.
# Then, we will calculate the expected returns, minimum variance portfolio, optimal risky portfolio and efficient frontier.
# You will also learn a new term called Sharpe Ratio.

# Import data
df = data.DataReader(['AAPL', 'NKE', 'GOOGL', 'AMZN'], 'yahoo', start='2015/01/01', end='2019/12/31')
print(df.head())

# Closing price
df = df['Adj Close']
print(df.head())

##############################################################################################################
# Covariance and Correlation matrix
##############################################################################################################
# The first step is to obtain a covariance and correlation matrix to understand how different assets behave with respect to each other.
# When we had a 2 asset portfolio, we directly plugged in the names of the assets into .cov() and .corr() functions.
# In this case, we will need a matrix for better visualisation.
# This is also achieved by using the same 2 functions on our dataframe df.

# Log of percentage change and covariance matrix
cov_matrix = df.pct_change().apply(lambda x: np.log(1+x)).cov()
print(cov_matrix)

# Correlation matrix
corr_matrix = df.pct_change().apply(lambda x: np.log(1+x)).corr()
print(corr_matrix)
# As you can see, an asset always has a perfectly positive correlation of 1 with itself.

##############################################################################################################
# Portfolio variance
##############################################################################################################
# The formula for calculating portfolio variance differs from the usual formula of variance. It looks like this:
# $$\sigma^2(Rp) = \sum{i=1}^{n} \sum_{j=1}^{n} w_i w_j COV(R_i, R_j) $$

# The simplest way to do this complex calculation is defining a list of weights and multiplying this list horizontally and vertically with our covariance matrix.
# For this purpose, let’s define a random list of weights for all 4 assets.
# Remember that sum of weights should always be 1.

# Randomly weighted portfolio's variance
w = {'AAPL': 0.1, 'NKE': 0.2, 'GOOGL': 0.5, 'AMZN': 0.2}
port_var = cov_matrix.mul(w, axis=0).mul(w, axis=1).sum().sum()
print(port_var)
# 0.00016069523003596587

# Thus we have found the portfolio variance. But for truly optimizing the portfolio, we cant plug in random weights.
# We will need to calculate it according to what gives us maximum expected returns.

##############################################################################################################
# Portfolio expected returns
##############################################################################################################
# The mean of returns (given by change in prices of asset stock prices) give us the expected returns of that asset.
# The sum of all individual expected returns further multiplied by the weight of assets give us expected return for the portfolio.

# Note that we use the resample() function to get yearly returns. The argument to function, ‘Y’, denotes yearly.
# If we dont perform resampling, we will get daily returns, like you saw earlier in the ‘Fundamental Terms’ section.

# Yearly returns for individual companies
ind_er = df.resample('Y').last().pct_change().mean()
print(ind_er)

# Portfolio returns
w = [0.1, 0.2, 0.5, 0.2]
port_er = (w*ind_er).sum()
print(port_er)
# 0.27071990038443955

##############################################################################################################
# Plotting the efficient frontier
##############################################################################################################
# Efficient frontier is a graph with ‘returns’ on the Y-axis and ‘volatility’ on the X-axis.
# It shows us the maximum return we can get for a set level of volatility, or conversely, the volatility that we need to accept for certain level of returns.

# But first, lets take a look at the volatiltilty and returns of individual assets for a better understanding.

# In US the trading days in a year are 250 (this may be different in UK)
# trading_days_year = 250

# Volatility is given by the annual standard deviation. We multiply by 250 because there are 250 trading days/year.
ann_sd = df.pct_change().apply(lambda x: np.log(1+x)).std().apply(lambda x: x*np.sqrt(trading_days_year))
print(ann_sd)

# Creating a table for visualising returns and volatility of assets
assets = pd.concat([ind_er, ann_sd], axis=1)
assets.columns = ['Returns', 'Volatility']
print(assets)

# Next, to plot the graph of efficient frontier, we need run a loop.
# In each iteration, the loop considers different weights for assets and calculates the return and volatility of that particular portfolio combination.
# We run this loop a 1000 times. To get random numbers for weights, we use the np.random.random() function.
# But remember that the sum of weights must be 1, so we divide those weights by their cumulative sum.

p_ret = [] # Define an empty array for portfolio returns
p_vol = [] # Define an empty array for portfolio volatility
p_weights = [] # Define an empty array for asset weights

num_assets = len(df.columns)
num_portfolios = 10000

for portfolio in range(num_portfolios):
    weights = np.random.random(num_assets)
    weights = weights/np.sum(weights)
    p_weights.append(weights)
    returns = np.dot(weights, ind_er) # Returns are the product of individual expected returns of asset and its
                                      # weights
    p_ret.append(returns)
    var = cov_matrix.mul(weights, axis=0).mul(weights, axis=1).sum().sum() # Portfolio Variance
    sd = np.sqrt(var) # Daily standard deviation
    ann_sd = sd*np.sqrt(trading_days_year) # Multiply by 250. Annual standard deviation = volatility
    p_vol.append(ann_sd)

data = {'Returns':p_ret, 'Volatility':p_vol}

for counter, symbol in enumerate(df.columns.tolist()):
    #print(counter, symbol)
    data[symbol+' weight'] = [w[counter] for w in p_weights]

portfolios  = pd.DataFrame(data)
print(portfolios.head()) # Dataframe of the 10000 portfolios created

# You can see that there are a number of portfolios with different weights, returns and volatility.
# Plotting the returns and volatility from this dataframe will give us the efficient frontier for our portfolio.

# Plot efficient frontier
portfolios.plot.scatter(x='Volatility', y='Returns', marker='o', s=10, alpha=0.3, grid=True, figsize=[10,10])
plt.show()

# Each point on the line (left edge) represents an optimal portfolio of stocks that maximises the returns for any given level of risk.
# On this graph, you can also see the combination of weights that will give you all possible combinations:
#   Minimum volatility (left most point)
#   Maximum returns (top most point)

min_vol_port = portfolios.iloc[portfolios['Volatility'].idxmin()]
# idxmin() gives us the minimum value in the column specified.
print(min_vol_port)
# The values at the minimum index found are as follows
print(portfolios.iloc[min_vol_port])

##############################################################################################################
# plotting the minimum volatility portfolio
##############################################################################################################
plt.subplots(figsize=[10,10])
plt.scatter(portfolios['Volatility'], portfolios['Returns'],marker='o', s=10, alpha=0.3)
plt.scatter(min_vol_port[1], min_vol_port[0], color='r', marker='*', s=500)
plt.show()

# The red star denotes the most efficient portfolio with minimum volatility.

##############################################################################################################
# Sharpe ratio
##############################################################################################################

# The question arises that how do we find this optimal risky portfolio and finally optimize our portfolio to the maximum?
# This is done by using a parameter called the Sharpe Ratio.
# The ratio is the average return earned in excess of the risk-free rate per unit of volatility or total risk.
# The risk-free rate of return is the return on an investment with zero risk, meaning it’s the return investors could expect for taking no risk.

# We define as an assumption the risk-free rate to be 1% or 0.01.

# Finding the optimal portfolio
rf = 0.01 # risk factor
optimal_risky_port = portfolios.iloc[((portfolios['Returns']-rf)/portfolios['Volatility']).idxmax()]
print(optimal_risky_port)

# Plotting optimal portfolio
plt.subplots(figsize=(10, 10))
plt.scatter(portfolios['Volatility'], portfolios['Returns'],marker='o', s=10, alpha=0.3)
plt.scatter(min_vol_port[1], min_vol_port[0], color='r', marker='*', s=500)
plt.scatter(optimal_risky_port[1], optimal_risky_port[0], color='g', marker='*', s=500)
plt.show()

# The green star represents the optimal risky portfolio.

max_stars_repo_name: enginance/ModernPortfolioTheory

max_issues_repo_name: enginance/ModernPortfolioTheory

max_forks_repo_name: enginance/ModernPortfolioTheory


text:
import torch
import numpy as np
from torch.autograd import Function
from .._ext import roi1d_pooling

class RoI1DPoolFunction(Function):
    def __init__(self, pooled_depth, temporal_scale=1.):
        self.pooled_depth = int(pooled_depth)
        self.temporal_scale = float(temporal_scale)
        self.rois = None
        self.feature_size = None

    def forward(self, features, rois):
        batch_size, num_channels, data_depth = features.size()
        num_rois = rois.size()[1]
        output = torch.zeros(batch_size, num_rois, num_channels,
                             self.pooled_depth)

        output = output.cuda()
        roi1d_pooling.roi1d_pooling_forward_cuda(self.pooled_depth, self.temporal_scale, features, rois, output)
        import pdb; pdb.set_trace()
        # output.data.masked_fill_(torch.isnan(output), 0)
        self.rois = rois
        self.feature_size = features.size()

        return output

    def backward(self, grad_output):
        assert(self.feature_size is not None and grad_output.is_cuda)

        batch_size, num_channels, data_depth = self.feature_size

        grad_input = torch.zeros(batch_size, num_channels, data_depth).cuda()
        roi1d_pooling.roi1d_pooling_backward_cuda(self.pooled_depth, self.temporal_scale, grad_output,
                                                  self.rois, grad_input)
        if np.isnan(grad_input.data.cpu().numpy()).any():
            import pdb; pdb.set_trace()
        self.rois = None

        return grad_input, None

ext: py

reason: import numpy


text:
import librosa
import numpy as np

def extract_features(signal, freq=16000, n_mfcc=5, size=512, step=16):
    # Mel Frequency Cepstral Coefficents
    mfcc = librosa.feature.mfcc(
        y=signal, sr=freq, n_mfcc=n_mfcc, n_fft=size, hop_length=step
    )
    mfcc_delta = librosa.feature.delta(mfcc)
    mfcc_delta2 = librosa.feature.delta(mfcc, order=2)

    # Root Mean Square Energy
    mel_spectogram = librosa.feature.melspectrogram(
        y=signal, sr=freq, n_fft=size, hop_length=step
    )
    rmse = librosa.feature.rms(S=mel_spectogram, frame_length=size, hop_length=step)

    mfcc = np.asarray(mfcc)
    mfcc_delta = np.asarray(mfcc_delta)
    mfcc_delta2 = np.asarray(mfcc_delta2)
    rmse = np.asarray(rmse)

    features = np.concatenate((mfcc, mfcc_delta, mfcc_delta2, rmse), axis=0)
    return features

max_stars_repo_name: dennisgbrown/vocal-activity-detector

max_issues_repo_name: dennisgbrown/vocal-activity-detector

max_forks_repo_name: dennisgbrown/vocal-activity-detector


text:
import numpy as np
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
from mpl_toolkits.mplot3d import Axes3D  # noqa

MAP_WIDTH = 10.0  # map width in degrees

# Temporary function for plotting maps.
# The idea is to separate plotting methods from modules like rupture.

def plot_rupture_wire3d(rupture, ax=None):
    """
    Method for making a simple representation of a Rupture instance.
    This method draws the outline of each quadrilateral in 3D.

    Args:
        rupture: A Rupture instance.
        ax: A matplotlib axis (optional).

    Returns:
        Matplotlib axis.
    """

    if ax is None:
        fig = plt.figure()
        ax = fig.add_subplot(111, projection="3d")
    else:
        if "xlim3d" not in list(ax.properties().keys()):
            raise TypeError("Non-3d axes object passed to plot() method.")
    for quad in rupture.getQuadrilaterals():
        x = [p.longitude for p in quad]
        x.append(x[0])
        y = [p.latitude for p in quad]
        y.append(y[0])
        z = [-p.depth for p in quad]
        z.append(z[0])
        ax.plot(x, y, z, "k")
        ax.set_xlabel("Longitude")
        ax.set_ylabel("Latitude")
        ax.set_zlabel("Depth")

    return ax

def map_rupture(rupture):
    """
    Method for making a simple representation of a Rupture instance.
    This method draws the surface projection of the rupture on a map.

    Args:
        rupture: A Rupture instance.

    """
    rlats = rupture.lats
    rlons = rupture.lons

    # remove nans in these arrays
    rlats = rlats[~np.isnan(rlats)]
    rlons = rlons[~np.isnan(rlons)]

    minbufx = 0.2
    minbufy = 0.2
    lat1 = np.nanmin(rupture.lats)
    lat2 = np.nanmax(rupture.lats)
    dlat = lat2 - lat1
    bufy = np.max([dlat / 2, minbufy])
    lon1 = np.nanmin(rupture.lons)
    lon2 = np.nanmax(rupture.lons)
    dlon = lon2 - lon1
    bufx = np.max([dlon / 2, minbufx])

    ax = plt.axes(projection=ccrs.Robinson())
    bufx = MAP_WIDTH / 2.0
    bufy = MAP_WIDTH / 2.0
    ax.set_extent([lon1 - bufx, lon2 + bufx, lat1 - bufy, lat2 + bufy])
    ax.stock_img()
    ax.coastlines()
    ax.plot(rlons, rlats, "r", transform=ccrs.PlateCarree())
    return ax

max_stars_repo_name: cbworden/shakemap

max_issues_repo_name: cbworden/shakemap

max_forks_repo_name: cbworden/shakemap


text:
import tkinter as tk
from tkinter import E, W, N, S
from PIL import ImageTk
from PIL import Image
import numpy as np

def showpoints(oldpoints,newpoints,pradii,File):
    """Show the image again along with the uncertainty circles,
    plotting the optimized points on top.

    Inputs =>
      oldpoints: old points guessed by user
      newpoints: optimized joint image points
      pradii: uncertainty radii
      File: filename of image
    """
    root = tk.Tk()

    #setting up a tkinter canvas with scrollbars
    frame = tk.Frame(root, bd=2, relief=tk.SUNKEN)
    frame.grid_rowconfigure(0, weight=1)
    frame.grid_columnconfigure(0, weight=1)
    xscroll = tk.Scrollbar(frame, orient=tk.HORIZONTAL)
    xscroll.grid(row=1, column=0, sticky=E+W)
    yscroll = tk.Scrollbar(frame)
    yscroll.grid(row=0, column=1, sticky=N+S)
    canvas = tk.Canvas(frame, bd=0, xscrollcommand=xscroll.set, yscrollcommand=yscroll.set)
    canvas.grid(row=0, column=0, sticky=N+S+E+W)
    xscroll.config(command=canvas.xview)
    yscroll.config(command=canvas.yview)
    frame.pack(fill=tk.BOTH,expand=1)

    #adding the image
    img = ImageTk.PhotoImage(Image.open(File))
    canvas.create_image(0,0,image=img,anchor="nw")
    canvas.config(width=img.width(), height=img.height(), scrollregion=canvas.bbox(tk.ALL))

    oldpoints = np.copy(oldpoints)
    newpoints = np.copy(newpoints)
    oldpoints[:,1] = img.height()-oldpoints[:,1]
    newpoints[:,1] = img.height()-newpoints[:,1]
    linewidthouter=3
    linewidthinner=1
    for i in range(len(pradii)):
        circle = canvas.create_oval(oldpoints[i,0]-pradii[i],oldpoints[i,1]-pradii[i],oldpoints[i,0]+pradii[i],oldpoints[i,1]+pradii[i],stipple="gray12",fill="green")
        canvas.create_line(oldpoints[i,0]-5,oldpoints[i,1]-5,oldpoints[i,0]+5,oldpoints[i,1]+5,width=linewidthouter)
        canvas.create_line(oldpoints[i,0]-5,oldpoints[i,1]+5,oldpoints[i,0]+5,oldpoints[i,1]-5,width=linewidthouter)
        canvas.create_line(oldpoints[i,0]-5,oldpoints[i,1]-5,oldpoints[i,0]+5,oldpoints[i,1]+5,width=linewidthinner,fill="white")
        canvas.create_line(oldpoints[i,0]-5,oldpoints[i,1]+5,oldpoints[i,0]+5,oldpoints[i,1]-5,width=linewidthinner,fill="white")
    for i in range(len(pradii)):
        canvas.create_line(newpoints[i,0]-5,newpoints[i,1]-5,newpoints[i,0]+5,newpoints[i,1]+5,width=linewidthouter, fill="red")
        canvas.create_line(newpoints[i,0]-5,newpoints[i,1]+5,newpoints[i,0]+5,newpoints[i,1]-5,width=linewidthouter, fill="red")
        canvas.create_line(newpoints[i,0]-5,newpoints[i,1]-5,newpoints[i,0]+5,newpoints[i,1]+5,width=linewidthinner,fill="yellow")
        canvas.create_line(newpoints[i,0]-5,newpoints[i,1]+5,newpoints[i,0]+5,newpoints[i,1]-5,width=linewidthinner,fill="yellow")

    root.mainloop()


text:
import numpy as np

# This is where you can build a decision tree for determining throttle, brake and steer
# commands based on the output of the perception_step() function
def decision_step(Rover):

    # Implement conditionals to decide what to do given perception data
    # Here you're all set up with some basic functionality but you'll need to
    # improve on this decision tree to do a good job of navigating autonomously!
    # determin indices where angles are inside -+ 10 degree corridor infront of rover
    cut_lim = 10
    idx = (Rover.nav_angles < cut_lim*np.pi/180) & (Rover.nav_angles >-cut_lim*np.pi/180)
    # determin indices where distances are above 5m
    idx2 = (Rover.nav_dists >= 50)
    # define range when to stop infront of wall
    stop_range = 44 # consider offset of rover to image
    # Example:
    # Check if we have vision data to make decisions with
    if Rover.nav_angles is not None:
        # Check for Rover.mode stati forward, stop, stuck
        if Rover.mode == 'forward':

            # Check if rover position is changing. -> Lots of Nonetype errors or inaccurate
            #R_pos_x = np.round(np.float(Rover.pos[0]),decimals=1)
            #R_pos_y = np.round(np.float(Rover.pos[1]),decimals=1)
            #R_pos_x_last = np.round(np.float(Rover.last_pos[0]),decimals=1)
            #R_pos_y_last = np.round(np.float(Rover.last_pos[1]),decimals=1)
            #if (np.absolute(R_pos_x - R_pos_x_last) <= 0.1) and (np.absolute(R_pos_y - R_pos_y_last) <= 0.1) :

            # Check if rover is not moving even though rover tries to accelerate
            if Rover.vel == 0 and Rover.throttle == 0.3:
               Rover.stuck_count +=1

            # Check if rover is is moving
            if Rover.stuck_count <=20:
                # Check the extent of navigable terrain and if there is terrain that goes further than the stop_stop range.
                if len(Rover.nav_angles) >= Rover.stop_forward and np.any(Rover.nav_dists>=stop_range):
                    # If mode is forward, navigable terrain looks good
                    # and velocity is below max, then throttle
                    if Rover.vel < Rover.max_vel:
                        # Set throttle value to throttle setting
                        Rover.throttle = Rover.throttle_set
                    else: # Else coast
                        Rover.throttle = 0
                    Rover.brake = 0
                    # Set steering to average angle clipped to the range +/- 15
                    # catch incase idx2 results in an empty array
                    try:
                        # Check if maximum angle for distances that are further than 5m is closer than 5 degree. Assumption: if statement is true, the wall on the left is close.
                        if np.max(Rover.nav_angles[idx2]) < 5* np.pi/180: # or Rover.nav_dists[np.max(Rover.nav_angles == np.max(Rover.nav_angles))]<40:
                         # Normal steering if wall is close
                            Rover.steer = np.clip(np.mean(Rover.nav_angles * 180/np.pi), -15, 15)
                        else:
                         # If wall is not close then steer with 20% of the angles standard deviation to the left.
                            Rover.steer = np.clip(np.mean(Rover.nav_angles * 180/np.pi)+ 0.22*np.std(Rover.nav_angles*180/np.pi), 0, 15) #clip to allow only turning to the left
                    except ValueError:
                        # When empty array assume rover is stuck.
                        Rover.mode = 'stuck'

                    # Check rover position every 15th iteration
                    #Rover.last_count += 1
                    #if Rover.last_count >=15:
                    #   Rover.last_pos = Rover.pos
                    #    Rover.last_count = 0

                # If there's a lack of navigable terrain pixels then go to 'stop' mode
                elif len(Rover.nav_angles) < Rover.stop_forward or np.any(Rover.nav_dists<stop_range):
                        # Set mode to "stop" and hit the brakes!
                        Rover.throttle = 0
                        # Set brake to stored brake value
                        Rover.brake = Rover.brake_set
                        Rover.steer = 0
                        Rover.mode = 'stop'
            else:
                # if rover is not moving after 20 iterations, then stop and go to stuck mode.
                Rover.throttle = 0
                # Set brake to stored brake value
                Rover.brake = Rover.brake_set
                Rover.steer = 0
                Rover.mode = 'stuck'
                # reset count for stuck mode
                Rover.stuck_angle = 0

        # If we're already in "stop" mode then make different decisions
        elif Rover.mode == 'stop':
            # If we're in stop mode but still moving keep braking
            if Rover.vel > 0.2:
                Rover.throttle = 0
                Rover.brake = Rover.brake_set
                Rover.steer = 0
            # If we're not moving (vel < 0.2) then do something else
            elif Rover.vel <= 0.2:
                # Now we're stopped and we have vision data to see if there's a path forward that is further than the stop range.
                if len(Rover.nav_angles) < Rover.go_forward or np.all(Rover.nav_dists<stop_range):
                    Rover.throttle = 0
                    # Release the brake to allow turning
                    Rover.brake = 0
                    # Turn range is +/- 15 degrees, when stopped the next line will induce 4-wheel turning
                    Rover.steer = -15 # Could be more clever here about which way to turn
                # If we're stopped but see sufficient navigable terrain in front then go!
                # Additionally check if inside the cut_lim range (+-10 degrees) in front of the rover the number of distances that are longer than 5m are above the threshold for going forward.
                if len(Rover.nav_angles) >= Rover.go_forward and len(Rover.nav_dists[idx]>=stop_range)>=Rover.go_forward:
                    # Set throttle back to stored value
                    Rover.throttle = Rover.throttle_set
                    # Release the brake
                    Rover.brake = 0
                    # Set steer to mean angle
                    Rover.steer = np.clip(np.mean(Rover.nav_angles * 180/np.pi), -15, 15)
                    Rover.mode = 'forward'

        # If we are in stuck mode then do
        elif Rover.mode == 'stuck':
            # start counting calls of suck mode
            Rover.stuck_angle+=1
            # turn to the right as long as stuck calls are below 12
            if Rover.stuck_angle<=12:
                Rover.steer =-15
                Rover.brake = 0
                Rover.throttle = 0
            # if calls are above 12 then go to forward mode. (if this is not possible, rover will jump back to stuck and continue turning.)
            else:
                Rover.mode = 'forward'
                Rover.stuck_count = 0
    # Just to make the rover do something
    # even if no modifications have been made to the code
    else:
        Rover.throttle = Rover.throttle_set
        Rover.steer = 0
        Rover.brake = 0

    # If in a state where want to pickup a rock send pickup command
    if Rover.near_sample and Rover.vel == 0 and not Rover.picking_up:
        Rover.send_pickup = True

    return Rover

max_stars_repo_name: Dracoadit/Udacity

max_issues_repo_name: Dracoadit/Udacity

max_forks_repo_name: Dracoadit/Udacity


text:
"""Class for computing mutual information for an alphabet and discrete input distribution
"""
import numpy as np
import random as rand
import os
import math
from MakeInputDict import MakeInputDict

class MIcalculator(MakeInputDict):
    def __init__(self,response_file,_input_size):
        #self.read_input_pdf(input_file)
        self.input_size = _input_size
        self.read_responses(response_file)
        #self.compute_mean_response()

    def read_input_pdf(self,input_file):
        ifile = open(input_file,'r')
        all_lines = ifile.readlines()
        ifile.close()

        self.input_loc = []
        self.input_wts = []

        for line in all_lines:
            this_set = line.rstrip('\r\n').split(',')
            self.input_loc.append(float(this_set[0]))
            self.input_wts.append(float(this_set[1]))

        self.input_size = len(self.input_loc)

        print(sum(self.input_wts))

    def get_input_pdf(self,_input_loc,_input_wts):
        self.input_loc = _input_loc
        self.input_wts = _input_wts

    def read_responses(self,response_file):
        ifile = open(response_file,'r')
        all_lines = ifile.readlines()
        ifile.close()

        self.conditional_entropy = 0.0

        #print all_lines

        self.response_pdfs = {}
        self.response_wts = {}
        self.partial_entropy = []

        #response_size = len(self.input_loc)
        response_size = self.input_size

        for i in range(0,response_size):
            this_entropy = 0.0
            loc_set = all_lines[2*i].rstrip('\r\n').split(',')
            wt_set = all_lines[2*i+1].rstrip('\r\n').split(',')

            self.response_pdfs[i] = {}

            set_size = len(loc_set)

            for k in range(0,set_size):
                if loc_set[k]!='':
                    #self.response_pdfs[i][int(loc_set[k])] = float(wt_set[k])
                    self.response_pdfs[i][loc_set[k]] = float(wt_set[k])
                    try:
                        this_entropy += float(wt_set[k])*math.log(float(wt_set[k]))
                    except ValueError:
                        pass

            self.partial_entropy.append(this_entropy)

            #self.conditional_entropy -= self.input_wts[i]*this_entropy
            #print sum(self.response_pdfs[i].values())

    def compute_conditional_entropy(self):
        self.conditional_entropy = 0.0

        for i in range(0,self.input_size):
            self.conditional_entropy -= self.input_wts[i]*self.partial_entropy[i]

        #print self.conditional_entropy

    def compute_mutual_information(self):
        key_set = []
        self.mean_response_pdf = {}
        self.mean_entropy = 0.0

        for k in range(0,self.input_size):
            for key in self.response_pdfs[k].keys():
                if key not in key_set:
                    key_set.append(key)
                    self.mean_response_pdf[key] = 0.0

        for key in key_set:
            for response_no in range(0,self.input_size):
                try:
                    self.mean_response_pdf[key] += self.input_wts[response_no]*self.response_pdfs[response_no][key]
                except KeyError:
                    pass

        for value in self.mean_response_pdf.values():
            try:
                self.mean_entropy -= value*math.log(value)
            except ValueError:
                pass

        #print self.mean_entropy
        #print 'Mutual information: ', self.mean_entropy - self.conditional_entropy

        self.mutual_information = (self.mean_entropy - self.conditional_entropy)/math.log(2.0)

        return max(0.0,self.mutual_information)

max_stars_repo_name: SwarnavoSarkar/InGene

max_issues_repo_name: SwarnavoSarkar/InGene

max_forks_repo_name: SwarnavoSarkar/InGene


text:
def func1():
    import datetime
    import matplotlib as mpl
    import matplotlib.pyplot as plt
    from PIL import Image
    import numpy as np
    import os
    import dtlpy as dl
    from dtlpy.ml import train_utils

def func2():
    model = dl.models.get(model_name='ResNet')
    snapshot = model.snapshots.get('pretrained-resnet18')
    model.snapshots.list().to_df()

def func3():
    project = dl.projects.get('Sheeps Face Proj')
    dataset = project.datasets.create('Sheep Face')
    dataset.to_df()
    _ = dataset.items.upload(local_path='../../assets/sample_datasets/SheepFace/items/*',
                             local_annotations_path='../../assets/sample_datasets/SheepFace/json')

def func4():
    adapter = model.build()

def func5():
    adapter.load_from_snapshot(snapshot=snapshot)

def func6():
    item = dl.items.get(item_id='611e174e4c09acc3c5bb81d3')
    annotations = adapter.predict_items([item], with_upload=True)

    image = Image.open(item.download())
    plt.imshow(item.annotations.show(np.asarray(image),
                                     thickness=5))
    print('Classification: {}'.format(annotations[0][0].label))

def func7():
    dataset = project.datasets.get(dataset_name='Sheep Face')

    partitions = {dl.SnapshotPartitionType.TRAIN: 0.8,
                  dl.SnapshotPartitionType.VALIDATION: 0.2}

    cloned_dataset = train_utils.prepare_dataset(dataset,
                                                 filters=None,
                                                 partitions=partitions)

    snapshot_name = 'sheep-soft-augmentations'
    # create an Item Bucket to save snapshot in your project
    bucket = project.buckets.create(bucket_type=dl.BucketType.ITEM,
                                    model_name=model.name,
                                    snapshot_name=snapshot_name)
    new_snapshot = snapshot.clone(snapshot_name=snapshot_name,
                                  dataset_id=cloned_dataset.id,
                                  bucket=bucket,
                                  configuration={'batch_size': 16,
                                                 'start_epoch': 0,
                                                 'num_epochs': 2,
                                                 'input_size': 256})

    new_snapshot = model.snapshots.get(snapshot_name=snapshot_name)

def func8():
    adapter.load_from_snapshot(snapshot=new_snapshot)

    root_path, data_path, output_path = adapter.prepare_training()

def func9():
    print("Training {!r} with snapshot {!r} on data {!r}".format(model.name, new_snapshot.id, data_path))

    adapter.train(data_path=data_path,
                  output_path=output_path)

def func10():
    adapter.save_to_snapshot(local_path=output_path,
                             replace=True)

def func11():
    adapter.snapshot.bucket.list_content()

def func12():
    from imgaug import augmenters as iaa
    from torchvision import transforms
    from dtlpy.ml.ml_dataset import get_torch_dataset
    augmentation = iaa.Sequential([
        iaa.Resize({"height": 256, "width": 256}),
        # iaa.Superpixels(p_replace=(0, 0.5), n_segments=(10, 50)),
        iaa.flip.Fliplr(p=0.5),
        iaa.flip.Flipud(p=0.5),
        #         iaa.GaussianBlur(sigma=(0.0, 0.8)),
    ])
    tfs = transforms.Compose([
        augmentation,
        np.copy,
        transforms.ToTensor()
    ])
    dataloader = get_torch_dataset()(data_path=os.path.join(data_path, 'train'),
                                     dataset_entity=cloned_dataset,
                                     annotation_type=dl.AnnotationType.CLASSIFICATION,
                                     transforms=tfs,
                                     with_orig=True)

def func13():
    adapter = model.build()
    trained_snapshot = model.snapshots.get(snapshot_name='sheep-soft-augmentations')
    adapter.load_from_snapshot(snapshot=trained_snapshot,
                               overwrite=True)

def func14():
    fig, ax = plt.subplots(1, 2)
    for i in range(2):
        image, target, orig_image, orig_targets = dataloader[np.random.randint(len(dataloader))]
        anno = adapter.predict([orig_image])
        ax[i].imshow(orig_image)
        ax[i].set_title('GT: {!r}\n Pred: {!r}:{:.2f}'.format(adapter.label_map[str(int(target))],
                                                              anno[0][0].label,
                                                              anno[0][0].metadata['user']['model']['confidence']))

max_stars_repo_name: shiragit/dtlpy-documentation

max_issues_repo_name: shiragit/dtlpy-documentation

max_forks_repo_name: shiragit/dtlpy-documentation


text:
"""
expression matrix parser/validator
"""
import csv
import numpy as np
import pandas as pd
from lrgasp import LrgaspException
from lrgasp import gopen, iter_to_str
from lrgasp.defs import validate_feature_ident
from lrgasp.data_sets import get_lrgasp_rna_seq_metadata

##
# In order to detect empty columns and short rows, disallow space as a NA value, as well
# as remove other weirder values
##
na_values = frozenset(['NaN', 'NA', 'nan', 'null', '-nan', 'n/a', 'N/A', 'NULL', '-NaN'])

class ExpressionMatrix:
    """Expression matrix and operations to access columns by file or biosample accession"""
    def __init__(self, df):
        self.df = df

def sample_column_names(expression_mat):
    "return all but ID column"
    return [col for col in expression_mat.df.columns if col != "ID"]

def validate_header(expression_mat):
    if "ID" not in expression_mat.df.columns:
        raise LrgaspException("TSV must have column 'ID'")
    if len(expression_mat.df.columns) < 2:
        raise LrgaspException("TSV must have at least one sample column")

def check_column_type(expression_mat, col):
    "validate column type is correct"
    if expression_mat.df.dtypes[col] not in (np.float64, np.int64):
        raise LrgaspException(f"Invalid value(s) in column '{col}', must be a number, NA, or NaN, appears to contain other types or a short row")

def validate_data(expression_mat):
    if expression_mat.df.size == 0:
        raise LrgaspException("TSV contains no data")
    expression_mat.df.ID.apply(validate_feature_ident)
    for col in sample_column_names(expression_mat):
        check_column_type(expression_mat, col)

def validate_replicates(experiment_md, expression_mat):
    """validate that samples match those for the experiment and that all samples
    are in the matrix."""
    rna_seq_md = get_lrgasp_rna_seq_metadata()
    sample_cols = sample_column_names(expression_mat)

    def _check_in_expr():
        "check that all columns are in the experiment"
        for sc in sample_cols:
            if sc not in experiment_md.libraries:
                raise LrgaspException(f"matrix column '{sc}' is not listed in experiment_md.libraries for '{experiment_md.experiment_id}'")

    def _build_run_replicates(file_accs):
        "create a set with tuples of (run_acc, replicate_number)"
        run_reps = set()
        for file_acc in file_accs:
            file_md = rna_seq_md.get_file_by_acc(file_acc)
            run_reps.add((file_md.run_acc, file_md.biological_replicate_number))
        return run_reps

    def _check_covers_expr():
        """check that every run and replicate for an experiment is in matrix"""
        expr_rr = _build_run_replicates(experiment_md.libraries)
        mat_rr = _build_run_replicates(sample_cols)
        if len(mat_rr) > len(expr_rr):
            raise LrgaspException(f"BUG: matrix has more run replicates than experiment: matrix='{mat_rr}' experiment='{expr_rr}'")
        missing_rr = expr_rr - mat_rr
        if len(missing_rr) > 0:
            missing_desc = iter_to_str(["{}/{}".format(*rr) for rr in missing_rr])
            raise LrgaspException(f"matrix does not have an entries for all run and replicate in experiment, missing: {missing_desc}")

    _check_in_expr()
    _check_covers_expr()

def load(expression_tsv, experiment_md=None):
    "if experiment_md is provide, validate samples"
    try:
        with gopen(expression_tsv) as fh:
            # na_value handling is needed to detect empty cells and short
            # rows, preventing implicit conversion to NaN.
            df = pd.read_csv(fh, dialect=csv.excel_tab, header=0,
                             converters={"ID": str},
                             na_values=na_values,
                             keep_default_na=False)
            expression_mat = ExpressionMatrix(df)
            validate_header(expression_mat)
            expression_mat.df.set_index("ID", drop=False, inplace=True, verify_integrity=True)
            validate_data(expression_mat)
            if experiment_md is not None:
                validate_replicates(experiment_md, expression_mat)
        return expression_mat
    except (LrgaspException, pd.errors.ParserError, pd.errors.EmptyDataError, ValueError) as ex:
        raise LrgaspException(f"Parse of expression matrix TSV failed: {expression_tsv}") from ex


text:
import numpy as np
import matplotlib.pyplot as plt
import scipy.optimize  # For check_grad, approx_fprime
np.random.seed(34)
class RNN:
    def __init__ (self, numHidden, numInput, numOutput):
        self.numHidden = numHidden
        self.numInput = numInput
        self.U = np.random.randn(numHidden, numHidden) * 1e-1
        self.V = np.random.randn(numHidden, numInput) * 1e-1
        self.w = np.random.randn(numHidden) * 1e-1

    def backward (self, yhat,y,zt,ht,xs):

        lr=0.00061
        self.w=self.w.reshape((numHidden,numInput))
        dU=np.zeros((self.numHidden,self.numHidden))
        for t in range(len(zt)-1,0,-1):
            #print(t)
            gt_trans=(1-(np.tanh(zt[t]))**2).T
            qt_trans= ((yhat[t] - y[t-1]).dot(self.w.T))*gt_trans
            q_tau_trans=qt_trans
            for tau in range(t+1,1,-1):
                #print(tau)
                prev_tau_g_trans=(1-(np.tanh(zt[tau-1]))**2).T
                prev_tau_q_trans= (q_tau_trans.dot(self.U))*prev_tau_g_trans
                q_tau_trans=prev_tau_q_trans
                r_tau=q_tau_trans.T.dot(ht[tau-1].T)
                dU+=r_tau
            self.U=self.U-lr*dU
        #print(dU)

        dV=np.zeros((self.numHidden,self.numInput))
        for t in range(len(zt)-1,0,-1):
            #print(t)
            gt_trans=(1-(np.tanh(zt[t]))**2).T
            qt_trans= ((yhat[t] - y[t-1]).dot(self.w.T))*gt_trans
            q_tau_trans=qt_trans
            for tau in range(t+1,1,-1):
                #print(tau)
                prev_tau_g_trans=(1-(np.tanh(zt[tau-1]))**2).T
                prev_tau_q_trans= (q_tau_trans.dot(self.U))*prev_tau_g_trans
                q_tau_trans=prev_tau_q_trans
                r_tau=q_tau_trans.T.dot(xs[tau-2].T)
                dV+=r_tau
            self.V=self.V-lr*dV
        #print(dV)

        dw=np.zeros((numHidden,1))
        for i in range(len(yhat)-1,1,-1):
             #print(ht[i].T.shape)
             dw+=(yhat[i]-y[i-1])*(ht[i])
             self.w=self.w-lr*dw
        #print(dw)

    def forward (self, x,y):
        # TODO: IMPLEMENT ME

        #uv = np.hstack((self.U,self.V))
        yhat=[None]
        jt=[0]
        zt=[None]
        h0=np.zeros((numHidden,numInput))
        ht=[h0]
        for i in range(1,len(x)+1):
            #for j in range(1,numHidden+1):
                #if j==1:
                    #hprevx = np.vstack((ht[j-1],x[i-1]))
                #else:
                    #hprevx = np.vstack((hj,x[i-1]))
                #zj=uv.dot(hprevx)
                #hj=np.tanh(zj)
            #print("zj shape",zj.shape)
            zt.append(np.dot(self.U,ht[i-1])+np.dot(self.V,x[i-1]))
            ht.append(np.tanh(zt[i]))
            yhat.append(np.dot(ht[i].T,self.w))
            jt.append(0.05*((yhat[i]-y[i-1])**2))
        #print("zt.shape",zt[2].shape)
        #print(len(jt))
        cost=np.squeeze(sum(jt))
        print('cost',cost)
        return jt,yhat,zt,ht,cost

def generateData ():
    total_series_length = 50
    echo_step = 2  # 2-back task
    batch_size = 1
    x = np.random.choice(2, total_series_length, p=[0.5, 0.5])
    y = np.roll(x, echo_step)
    y[0:echo_step] = 0
    y = list(y)
    return (x, y)

if __name__ == "__main__":
    xs, ys = generateData()
    #print(xs)
    #print (ys)
    numHidden = 6
    numInput = 1
    numTimesteps = len(xs)
    rnn = RNN(numHidden, numInput, 1)
    for i in range(600):
        print('Epoch',i)
        jt,yhat,zt,ht,cost=rnn.forward(xs,ys)
        if(cost<0.05):
            break
        rnn.backward(yhat,ys,zt,ht,xs)

reason: import numpy,import scipy


text:
import os
import time
import numpy
from PIL import Image
from os.path import join
from itertools import groupby

img_path = "/Users/dqy/My/captcha/checkcode/"
pic_path = "/Users/dqy/My/captcha/crop_img/"

def get_bin_table(threshold=190):
    """
    获取灰度转二值的映射table
    :param threshold:
    :return:
    """
    table = []
    for i in range(256):
        if i < threshold:
            table.append(0)
        else:
            table.append(1)

    return table

def depoint(img):
    """
    去噪
    对于像素值>245的邻域像素，判别为属于背景色，如果一个像素上下左右4各像素值有超过2个像素属于背景色，那么该像素就是噪声
    :param img:
    :return:
    """
    pixdata = img.load()
    w,h = img.size
    for y in range(1,h-1):
        for x in range(1,w-1):
            count = 0
            if pixdata[x,y-1] > 250:
                count = count + 1
            if pixdata[x,y+1] > 250:
                count = count + 1
            if pixdata[x-1,y] > 250:
                count = count + 1
            if pixdata[x+1,y] > 250:
                count = count + 1
            if count > 2:
                pixdata[x,y] = 255
    return img

def get_crop_imgs(img):
    """
    按照图片的特点,进行切割,这个要根据具体的验证码来进行工作. # 见本例验证图的结构原理图
    分割图片是传统机器学习来识别验证码的重难点，如果这一步顺利的话，则多位验证码的问题可以转化为1位验证字符的识别问题
    :param img:
    :return:
    """
    child_img_list = []
    for i in range(4):
        x = i * 19  # 见原理图
        y = 0
        child_img = img.crop((x, y, x + 19, img.height))
        child_img_list.append(child_img)
    return child_img_list

def get_clear_bin_image(image):
    """
    获取干净的二值化的图片。
    图像的预处理：
    1. 先转化为灰度
    2. 再二值化
    3. 然后清除噪点
    参考:http://python.jobbole.com/84625/
    :type img:Image
    :return:
    """
    width, height = image.size
    image = image.crop([width/9,height/4,width*8/9,height*4/5])
    imgry = image.convert('L')  # 转化为灰度图

    table = get_bin_table()

    out = depoint(imgry)
    img = out.point(table, '1')  # 变成二值图片:0表示黑色,1表示白色
    return img

def process(image_path):
    image = Image.open(image_path)
    img = get_clear_bin_image(image)
    # img.save(pic_path + "%d"%(int(time.time()*10000))+'.png')
    pics = get_crop_imgs(img)
    for pic in pics:
        pic.save(pic_path + "%d"%(int(time.time()*10000))+'.png')

def process_all(dir):
    # print("1")
    for root, dirs, files in os.walk(dir):
        for f in files:
            if f.rfind(u'.DS_Store')==-1:
            # print(os.path.join(root, f))
                process(os.path.join(root, f))

if __name__ == '__main__':
    # print("1")
    process_all(img_path)

max_stars_repo_name: ohhuola/Data-Mining-for-Cybersecurity

max_issues_repo_name: ohhuola/Data-Mining-for-Cybersecurity

max_forks_repo_name: ohhuola/Data-Mining-for-Cybersecurity


text:
import numpy as np

def convert_to_fixed_point(float_value: float, n_bits: int) -> float:
    f = (1 << n_bits)

    return np.round(float_value * f) * (1.0 / f)

def quantize_data(train_data: np.ndarray, test_data: np.ndarray, number_of_bits: int,
                  flag_save_details_to_file: bool = False, path: str = "./"):
    train_data_quantized = np.array([convert_to_fixed_point(x, number_of_bits) for x in train_data])
    test_data_quantized = np.array([convert_to_fixed_point(x, number_of_bits) for x in test_data])

    if flag_save_details_to_file:
        with open(path + "/quantization_comparision.txt", "w") as file_quantization:
            print("Train data size before quantization: " + str(len(train_data)), file=file_quantization)
            print("First element before quantization:\n" + str(train_data[0]), file=file_quantization)
            print("Size after quantization: " + str(len(train_data_quantized)), file=file_quantization)
            print("First element after quantization:\n" + str(train_data_quantized[0]), file=file_quantization)

    return train_data_quantized, test_data_quantized

def test_convert_to_fixed_point():
    assert convert_to_fixed_point(1 / 3, 2) == 0.25
    assert convert_to_fixed_point(1 / 3, 3) == 0.375

if __name__ == "__main__":
    value = 13 / 16
    print(value)

    for i in range(10, -1, -1):
        value_as_fixed_point = convert_to_fixed_point(value, i)
        print("Number of bits used: " + str(i) + " value: " + str(value_as_fixed_point))
        print("% error: {0:.2f}".format(abs(value - value_as_fixed_point) / value * 100))


text:
import numpy as np
from tqdm import tqdm

from act_src import particleClass
from act_src import interactionClass
from act_src import problemClass
from act_src import relationClass

def ini_particle(name='...'):
    tptc = particleClass.limFiniteDipole2D(length=length, name=name)
    tptc.u = u
    prb1.add_obj(tptc)
    return tptc

update_fun, update_order = '5bs', (1e-12, 1e-15)
overlap_epsilon = 1e-5
u, length = 1, 1
attract, align = 0, 0
tmax = 50
# x, y, phi1, phi2 = 0, 4,    np.pi / 2, np.pi / 2   # Liebergen2015, Fig 4.1, case b
# x, y, phi1, phi2 = 2, 4,    np.pi / 2, np.pi / 2   # Liebergen2015, Fig 4.1, case c
# x, y, phi1, phi2 = 4, 2.25, np.pi / 2, np.pi / 2   # Liebergen2015, Fig 4.1, case d
# x, y, phi1, phi2 = 4, 1,    np.pi / 2, np.pi / 2   # Liebergen2015, Fig 4.1, case e
x, y, phi1, phi2 = 4, 0,    np.pi / 2, np.pi / 2   # Liebergen2015, Fig 4.1, case f

# y, x, phi1, phi2 = 0.5,        -8, np.pi, 0   # Liebergen2015, Fig 4.2, case b
# y, x, phi1, phi2 = 0.874,      -8, np.pi, 0   # Liebergen2015, Fig 4.2, case c
# y, x, phi1, phi2 = 0.875,      -8, np.pi, 0   # Liebergen2015, Fig 4.2, case d
# y, x, phi1, phi2 = 1,          -8, np.pi, 0   # Liebergen2015, Fig 4.2, case e
# y, x, phi1, phi2 = 0.87465, -8, np.pi, 0   # Liebergen2015, Fig 4.2, case f
# y, x, phi1, phi2 = 0.87465151, -8, np.pi, 0   # Liebergen2015, Fig 4.2, case f

# x, y, phi1, phi2 = 0.5,  1,    0, np.pi   # Liebergen2015, Fig 4.5, case a
# x, y, phi1, phi2 = 0,    1.15, 0, np.pi   # Liebergen2015, Fig 4.5, case b
# x, y, phi1, phi2 = 0.649, 0.7,  0, np.pi   # Liebergen2015, Fig 4.5, case c

# x, y, phi1, phi2 = -0.85,  0,    0, np.pi / 3       # Liebergen2015, Fig 4.8, case b
# x, y, phi1, phi2 =  0,    -0.9,  0, np.pi / 3       # Liebergen2015, Fig 4.8, case c
# x, y, phi1, phi2 = -0.3,  -1.15, 0, np.pi * 2 / 3   # Liebergen2015, Fig 4.8, case d
# x, y, phi1, phi2 = -0.75,  0,    0, np.pi * 2 / 3   # Liebergen2015, Fig 4.8, case e
# x, y, phi1, phi2 = -0.77,  0.4,  0, np.pi * 2 / 3   # Liebergen2015, Fig 4.8, case f

# problem = problemClass.finiteDipole2DProblem(name='testFiniteDipole2D')
prb1 = problemClass.finiteDipole2DProblem(name='testFiniteDipole2D')
prb1.attract = attract
prb1.align = align
prb1.update_fun = update_fun
prb1.update_order = update_order
prb1.tqdm_fun = tqdm

tptc1 = ini_particle(name='ptc2D')
tptc1.X = (0, 0)
tptc1.phi = phi1
tptc2 = ini_particle(name='ptc2D')
# print(tptc1.phi, tptc1.P1)
# print(tptc2.phi, tptc2.P1)
tptc2.X = (x, y)
tptc2.phi = phi2
# print(tptc2.phi, tptc2.P1)
# tptc1.length = 0.1

rlt1 = relationClass._baseRelation2D(name='relation1')
rlt1.overlap_epsilon = overlap_epsilon
prb1.relationHandle = rlt1

act1 = interactionClass.selfPropelled2D(name='selfPropelled2D')
prb1.add_act(act1)
act2 = interactionClass.limFiniteDipole2D(name='limFiniteDipole2D')
prb1.add_act(act2)

prb1.update_step()
prb1.update_self(t1=tmax)

max_stars_repo_name: pcmagic/ActiveMatter

max_issues_repo_name: pcmagic/ActiveMatter

max_forks_repo_name: pcmagic/ActiveMatter


text:
r"""Routines to create data from external modellers, for comparison purposes.

- DIPOLE1D: You must have Dipole1D installed and it must be in your system
  path; https://software.seg.org/2012/0003.

- EMmod: You must have Dipole1D installed and it must be in your system
  path; https://software.seg.org/2015/0001.

- Green3D: You must have Green3D installed (for which you need to be a member
  of the CEMI consortium). The following files must be in the folder
  `empymod/tests/green3d`: `green3d.m`, `grint.mexa64`,
  `grint.mexw64`,`normal.mexa64`, and `normal.mexw64`. Furthermore, you need
  Matlab.

Tested only on Linux (Ubuntu 16.04 LTS, x86_64).

Warning: These functions are to generate test-data with the provided scripts.
         They do not check the input, and are therefore very fragile if you do
         not provide the input as expected.

"""
import os
import subprocess
import numpy as np
from scipy.constants import mu_0
from os.path import join, dirname

class ChDir(object):
    r"""Step into a directory temporarily.

    Taken from:
    pythonadventures.wordpress.com/2013/12/15/
                    chdir-a-context-manager-for-switching-working-directories

    """

    def __init__(self, path):
        self.old_dir = os.getcwd()
        self.new_dir = path

    def __enter__(self):
        os.chdir(self.new_dir)

    def __exit__(self, *args):
        os.chdir(self.old_dir)

def green3d(src, rec, depth, res, freq, aniso, par, strength=0):
    r"""Run model with green3d (CEMI).

    You must have Green3D installed (for which you need to be a member of the
    CEMI consortium). The following files must be in the folder
    `empymod/tests/green3d`:
        - `green3d.m`
        - `grint.mexa64`
        - (`grint.mexw64`)
        - (`normal.mexa64`)
        - (`normal.mexw64`).
    Furthermore, you need to have Matlab installed.

    http://www.cemi.utah.edu

    """

    # Execution directory
    # (no need to create it, it HAS to exist with the necessary green3d-files).
    rundir = join(dirname(__file__), 'tmp/green3d/')

    # Source-input depending on par
    if par in [9, 10]:
        srcstr = str(src[0]) + ' ' + str(src[1]) + ' ' + str(src[2]) + ' '
        srcstr += str(src[3]) + ' ' + str(src[4])
    elif par in [2, 3]:
        srcstr = str(strength) + ' ' + str(src[0]) + ' ' + str(src[2]) + ' '
        srcstr += str(src[4]) + ' ' + str(src[1]) + ' ' + str(src[3]) + ' '
        srcstr += str(src[5])
    elif par in [6, 7, 8]:
        srcstr = str(src[0]) + ' ' + str(src[1]) + ' ' + str(src[2])

    # Write input file
    with open(rundir + 'run.sh', 'wb') as runfile:

        runfile.write(bytes(
            '#!/bin/bash\n\nmatlab -nodesktop -nosplash -r "[e, h] = green3d('
            '[' + ','.join(map(str, freq))+'], '
            '[' + ','.join(map(str, depth[1:] - np.r_[0, depth[1:-1]])) + '], '
            '[' + ','.join(map(str, 1/res[1:])) + '], '
            '[' + ','.join(map(str, aniso[1:])) + '], '
            '[' + ','.join(map(str, rec[0].ravel())) + '], '
            '[' + ','.join(map(str, rec[1].ravel())) + '], '
            '[' + ','.join(map(str, np.ones(np.size(rec[0])) * rec[2])) + '], '
            '[' + str(par) + ' ' + srcstr + ']); exit"', 'UTF-8'))

    # Run Green3D
    with ChDir(rundir):
        subprocess.run('bash run.sh', shell=True,
                       stderr=subprocess.STDOUT, stdout=subprocess.PIPE)

    # Read output-file
    with open(rundir + 'out.txt', 'rb') as outfile:
        temp = np.loadtxt(outfile)

        Ex = temp[:, 0] + 1j*temp[:, 1]
        Ey = temp[:, 2] + 1j*temp[:, 3]
        Ez = temp[:, 4] + 1j*temp[:, 5]
        Hx = temp[:, 6] + 1j*temp[:, 7]
        Hy = temp[:, 8] + 1j*temp[:, 9]
        Hz = temp[:, 10] + 1j*temp[:, 11]

        if par in [6, 7, 8, 10]:
            Ex /= 2j*freq*np.pi*mu_0
            Ey /= 2j*freq*np.pi*mu_0
            Ez /= 2j*freq*np.pi*mu_0
            Hx /= 2j*freq*np.pi*mu_0
            Hy /= 2j*freq*np.pi*mu_0
            Hz /= 2j*freq*np.pi*mu_0

        return Ex, Ey, Ez, Hx, Hy, Hz

def dipole1d(src, rec, depth, res, freq, srcpts=5):
    r"""Run model with dipole1d (Scripps).

    You must have Dipole1D installed and it must be in your system path.

    https://software.seg.org/2012/0003

    """

    # Create directory, overwrite existing
    rundir = join(dirname(__file__), 'tmp/dipole1d/')
    os.makedirs(rundir, exist_ok=True)

    # Source: A bipole in dipole1d is defined as: center point, angles, length
    if len(src) == 6:
        dx = src[1] - src[0]
        dy = src[3] - src[2]
        dz = src[5] - src[4]
        r = np.sqrt(dx**2 + dy**2 + dz**2)
        theta = np.rad2deg(np.arctan2(dy, dx))
        phi = np.rad2deg(np.pi/2-np.arccos(dz/r))
        src = [src[0]+dx/2, src[2]+dy/2, src[4]+dz/2, theta, phi]

    else:
        r = 0  # 0 = dipole

    # Angle: In empymod, x is Easting, and the angle is the deviation from x
    #        anticlockwise.  In Dipole1D, x is Northing, and the angle is the
    #        deviation from x clockwise. Convert angle to within 0<=ang<360:
    ang = (-src[3] % - 360 + 90) % 360

    # Counts
    nsrc = np.size(src[2])
    nrec = np.size(rec[0])
    nfreq = np.size(freq)
    nlay = np.size(res)

    # Write input file
    with open(rundir + 'RUNFILE', 'wb') as runfile:
        runfile.write(bytes(
            'Version:          DIPOLE1D_1.0\n'
            'Output Filename:  dipole1d.csem\n'
            'CompDerivatives:  no\n'
            'HT Filters:       kk_ht_401\n'
            'UseSpline1D:      no\n'
            'Dipole Length:    '+str(r)+'\n'
            '# integ pts:      '+str(srcpts)+'\n'
            '# TRANSMITTERS:   '+str(nsrc)+'\n'
            '          X           Y           Z    ROTATION         DIP\n',
            'UTF-8'))
        np.savetxt(runfile, np.atleast_2d(np.r_[src[1], src[0], src[2], ang,
                   src[4]]), fmt='%12.4f')
        runfile.write(bytes('# FREQUENCIES:    '+str(nfreq)+'\n',
                      'UTF-8'))
        np.savetxt(runfile, freq, fmt='%10.3f')
        runfile.write(bytes('# LAYERS:         '+str(nlay)+'\n',
                      'UTF-8'))
        np.savetxt(runfile, np.r_[[np.r_[-1000000, depth]], [res]].transpose(),
                   fmt='%12.5g')
        runfile.write(bytes('# RECEIVERS:      '+str(nrec)+'\n',
                      'UTF-8'))
        rec = np.r_[[rec[1].ravel()], [rec[0].ravel()],
                    [np.ones(np.size(rec[0]))*rec[2]]]
        np.savetxt(runfile, rec.transpose(), fmt='%12.4f')

    # Run dipole1d
    with ChDir(rundir):
        subprocess.run('DIPOLE1D', shell=True,
                       stderr=subprocess.STDOUT, stdout=subprocess.PIPE)

    # Read output-file
    skiprows = nlay + nsrc + nfreq + nrec + 6
    with open(rundir + 'dipole1d.csem', 'rb') as outfile:
        temp = np.loadtxt(outfile, skiprows=skiprows, unpack=True)
        Ex = temp[0] - 1j*temp[1]
        Ey = temp[2] - 1j*temp[3]
        Ez = temp[4] - 1j*temp[5]
        Hx = -temp[6]/mu_0 + 1j*temp[7]/mu_0
        Hy = -temp[8]/mu_0 + 1j*temp[9]/mu_0
        Hz = -temp[10]/mu_0 + 1j*temp[11]/mu_0

    return Ey, Ex, Ez, Hy, Hx, Hz

def emmod(dx, nx, dy, ny, src, rec, depth, res, freq, aniso, epermV, epermH,
          mpermV, mpermH, ab, nd=1000, startlogx=-6, deltalogx=0.5, nlogx=24,
          kmax=10, c1=0, c2=0.001, maxpt=1000, dopchip=0, xdirect=0):
    r"""Run model with emmod (Hunziker et al, 2015).

    You must have EMmod installed and it must be in your system path.

    https://software.seg.org/2015/0001

    nd        : number of integration domains
    startlogx : first integration point in space
    deltalogx : log sampling rate of integr. pts in space at first iteration
    nlogx     : amount of integration points in space at first iteration
    kmax      : largest wavenumber to be integrated
    c1        : first precision parameter
    c2        : second precision parameter
    maxpt     : maximum amount of integration points in space
    dopchip   : pchip interpolation (1) or linear interpolation (0)
    xdirect   : direct field in space domain (1) or in wavenumber domain (0)

    """

    # Create directory, overwrite existing
    rundir = join(dirname(__file__), 'tmp/emmod/')
    os.makedirs(rundir, exist_ok=True)

    # Write input-file
    with open(rundir + 'emmod.scr', 'wb') as runfile:
        runfile.write(bytes(
            '#!/bin/bash\n\nemmod \\\n'
            '  freq='+str(freq)+' \\\n'
            '  file_out=emmod.out \\\n'
            '  writebin=0 \\\n'
            '  nx='+str(nx)+' \\\n'
            '  ny='+str(ny)+' \\\n'
            '  zsrc='+str(src[2])+' \\\n'
            '  zrcv='+str(rec[2])+' \\\n'
            '  dx='+str(dx)+' \\\n'
            '  dy='+str(dy)+' \\\n'
            '  z='+','.join(map(str, np.r_[-1, depth]))+' \\\n'
            '  econdV='+','.join(map(str, 1/(res*aniso**2)))+' \\\n'
            '  econdH='+','.join(map(str, 1/res))+' \\\n'
            '  epermV='+','.join(map(str, epermV))+' \\\n'
            '  epermH='+','.join(map(str, epermH))+' \\\n'
            '  mpermV='+','.join(map(str, mpermV))+' \\\n'
            '  mpermH='+','.join(map(str, mpermH))+' \\\n'
            '  verbose=0 \\\n'
            '  component='+str(ab)+' \\\n'
            '  nd='+str(nd)+' \\\n'
            '  startlogx='+str(startlogx)+' \\\n'
            '  deltalogx='+str(deltalogx)+' \\\n'
            '  nlogx='+str(nlogx)+' \\\n'
            '  kmax='+str(kmax)+' \\\n'
            '  c1='+str(c1)+' \\\n'
            '  c2='+str(c2)+' \\\n'
            '  maxpt='+str(maxpt)+' \\\n'
            '  dopchip='+str(dopchip)+' \\\n'
            '  xdirect='+str(xdirect)+' \n',
            'UTF-8'))

    # Run EMmod
    with ChDir(rundir):
        subprocess.run('bash emmod.scr', shell=True,
                       stderr=subprocess.STDOUT, stdout=subprocess.PIPE)

    # Read output-file
    with open(rundir + 'emmod.out', 'rb') as outfile:
        temp = np.loadtxt(outfile, skiprows=1, unpack=True)

        # Get same x/y as requested (round to mm)
        tct = np.round(temp[0], 4) + 1j*np.round(temp[1], 4)
        tcr = np.round(rec[0], 4) + 1j*np.round(rec[1], 4)

        result = np.zeros(rec[0].shape, dtype=complex)
        for i in range(rec[0].size):
            itr = np.where(tct == tcr[i])[0]
            result[i] = (temp[3][itr] + 1j*temp[4][itr])[0]

    return result

max_stars_repo_name: prisae/empymod

max_issues_repo_name: agrayver/empymod

max_forks_repo_name: agrayver/empymod

reason: import numpy,from scipy


text:
"""

source https://github.com/keshik6/KITTI-2d-object-detection

"""

from __future__ import division

import gc
import os

import numpy as np
import torch
import tqdm
from torch.autograd import Variable

from utils import bbox_iou_numpy, compute_ap, non_max_suppression

def train(
    model,
    device,
    optimizer,
    scheduler,
    train_dataloader,
    csv_log_file,
    tensor_type=torch.cuda.FloatTensor,
    update_gradient_samples=16,
    freeze_darknet=False,
):
    """
        Train a deep neural yolo network model for a single epoch

        Args
            model: pytorch model object
            device: cuda or cpu
            optimizer: pytorch optimizer object
            scheduler: learning rate scheduler object that wraps the optimizer
            train_dataloader: training  images dataloader
            save_dir (string): Location to save model weights, plots and log_files
            model_num: Integer identifier for the model
            tensor_type: cuda float/ cpu float
            update_gradient_samples (int): number of samples the network sees before updating gradients
            freeze_darknet (bool): If true freeze backbone

    """

    model.train(True)
    scheduler.step()

    if freeze_darknet:
        print("Frezezing backbone...")
        for i, (name, p) in enumerate(model.named_parameters()):
            if int(name.split(".")[1]) < 75:  # if layer < 75
                p.requires_grad = False
    else:
        for i, (name, p) in enumerate(model.named_parameters()):
            if int(name.split(".")[1]) < 75:  # if layer < 75
                p.requires_grad = True

    # set gradients to zero
    optimizer.zero_grad()

    for i, (img_paths, images, labels) in enumerate(tqdm.tqdm(train_dataloader)):
        images = Variable(images.type(tensor_type))
        labels = Variable(labels.type(tensor_type), requires_grad=False)

        # Calculate loss
        loss = model(images, labels)

        # Backpropate
        loss.backward()

        # Update gradients after some batches
        if ((i + 1) % update_gradient_samples == 0) or (i + 1 == len(train_dataloader)):
            optimizer.step()
            optimizer.zero_grad()

        # Clear variables from gpu, collect garbage and clear gpu cache memory
        del images, labels
        gc.collect()
        torch.cuda.empty_cache()

        # Construct loss data
        loss_data = "%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f" % (
            model.losses["x"],
            model.losses["y"],
            model.losses["w"],
            model.losses["h"],
            model.losses["conf"],
            model.losses["cls"],
            loss.item(),
            model.losses["recall"],
            model.losses["precision"],
        )

        # Write batch_loss details to csv file
        # writer = csv.writer(csv_log_file)
        csv_log_file.writerow(loss_data.split("\t"))

    print(
        "[Losses: x %f, y %f, w %f, h %f, conf %f, cls %f, total %f, recall: %.5f, precision: %.5f]"
        % (
            model.losses["x"],
            model.losses["y"],
            model.losses["w"],
            model.losses["h"],
            model.losses["conf"],
            model.losses["cls"],
            loss.item(),
            model.losses["recall"],
            model.losses["precision"],
        )
    )

    return model

def validation(
    model,
    device,
    valid_dataloader,
    csv_log_file,
    tensor_type=torch.cuda.FloatTensor,
    num_classes=8,
):
    # Set model to evaluation mode
    model.train(False)
    model.eval()

    # Variables to store detections
    all_detections = []
    all_annotations = []

    for batch_i, (_, images, labels) in enumerate(
        tqdm.tqdm(valid_dataloader, desc="Detecting objects")
    ):

        images = Variable(images.type(tensor_type))

        with torch.no_grad():
            outputs = model(images)
            outputs = non_max_suppression(
                outputs, num_classes, conf_thres=0.8, nms_thres=0.4
            )

        for output, annotations in zip(outputs, labels):
            all_detections.append([np.array([]) for _ in range(num_classes)])
            if output is not None:
                # Get predicted boxes, confidence scores and labels
                pred_boxes = output[:, :5].cpu().numpy()
                scores = output[:, 4].cpu().numpy()
                pred_labels = output[:, -1].cpu().numpy()

                # Order by confidence
                sort_i = np.argsort(scores)
                pred_labels = pred_labels[sort_i]
                pred_boxes = pred_boxes[sort_i]

                for label in range(num_classes):
                    all_detections[-1][label] = pred_boxes[pred_labels == label]

            all_annotations.append([np.array([]) for _ in range(num_classes)])
            if any(annotations[:, -1] > 0):

                annotation_labels = annotations[annotations[:, -1] > 0, 0].numpy()
                _annotation_boxes = annotations[annotations[:, -1] > 0, 1:]

                # Reformat to x1, y1, x2, y2 and rescale to image dimensions
                annotation_boxes = np.empty_like(_annotation_boxes)
                annotation_boxes[:, 0] = (
                    _annotation_boxes[:, 0] - _annotation_boxes[:, 2] / 2
                )
                annotation_boxes[:, 1] = (
                    _annotation_boxes[:, 1] - _annotation_boxes[:, 3] / 2
                )
                annotation_boxes[:, 2] = (
                    _annotation_boxes[:, 0] + _annotation_boxes[:, 2] / 2
                )
                annotation_boxes[:, 3] = (
                    _annotation_boxes[:, 1] + _annotation_boxes[:, 3] / 2
                )
                annotation_boxes *= 416

                for label in range(num_classes):
                    all_annotations[-1][label] = annotation_boxes[
                        annotation_labels == label, :
                    ]

        # Clear variables from gpu, collect garbage and clear gpu cache memory
        del images, labels
        gc.collect()
        torch.cuda.empty_cache()

    average_precisions = {}
    for label in range(num_classes):
        true_positives = []
        scores = []
        num_annotations = 0

        for i in tqdm.tqdm(
            range(len(all_annotations)), desc=f"Computing AP for class '{label}'"
        ):
            detections = all_detections[i][label]
            annotations = all_annotations[i][label]

            num_annotations += annotations.shape[0]
            detected_annotations = []

            for *bbox, score in detections:
                scores.append(score)

                if annotations.shape[0] == 0:
                    true_positives.append(0)
                    continue

                overlaps = bbox_iou_numpy(np.expand_dims(bbox, axis=0), annotations)
                assigned_annotation = np.argmax(overlaps, axis=1)
                max_overlap = overlaps[0, assigned_annotation]

                if (
                    max_overlap >= 0.5
                    and assigned_annotation not in detected_annotations
                ):
                    true_positives.append(1)
                    detected_annotations.append(assigned_annotation)
                else:
                    true_positives.append(0)

        # no annotations -> AP for this class is 0
        if num_annotations == 0:
            average_precisions[label] = 0
            continue

        true_positives = np.array(true_positives)
        false_positives = np.ones_like(true_positives) - true_positives
        # sort by score
        indices = np.argsort(-np.array(scores))
        false_positives = false_positives[indices]
        true_positives = true_positives[indices]

        # compute false positives and true positives
        false_positives = np.cumsum(false_positives)
        true_positives = np.cumsum(true_positives)

        # compute recall and precision
        recall = true_positives / num_annotations
        precision = true_positives / np.maximum(
            true_positives + false_positives, np.finfo(np.float64).eps
        )

        # compute average precision
        average_precision = compute_ap(recall, precision)
        average_precisions[label] = average_precision

    ap_list = []
    for c, ap in average_precisions.items():
        print(f"+ Class '{c}' - AP: {ap}")
        # Write ap details to csv file
        ap_list.append(ap)

    mAP = np.mean(list(average_precisions.values()))
    ap_list.append(mAP)
    print(f"mAP: {mAP}")

    # writer = csv.writer(csv_log_file)
    csv_log_file.writerow(ap_list)

    return model, mAP

def train_model(
    model,
    device,
    optimizer,
    scheduler,
    train_dataloader,
    valid_dataloader,
    csv_log_file_train,
    csv_log_file_valid,
    weights_path,
    max_epochs=10,
    tensor_type=torch.cuda.FloatTensor,
    update_gradient_samples=16,
    freeze_darknet=False,
    freeze_epoch=-1,
):
    best_mAP = 0.0

    for i in range(0, max_epochs):
        print("--------Epoch {}--------".format(i + 1))
        if (freeze_darknet and freeze_epoch != -1) and (i + 1 >= freeze_epoch):
            model = train(
                model,
                device,
                optimizer,
                scheduler,
                train_dataloader,
                csv_log_file_train,
                tensor_type=torch.cuda.FloatTensor,
                update_gradient_samples=16,
                freeze_darknet=freeze_darknet,
            )
        else:
            model = train(
                model,
                device,
                optimizer,
                scheduler,
                train_dataloader,
                csv_log_file_train,
                tensor_type=torch.cuda.FloatTensor,
                update_gradient_samples=16,
                freeze_darknet=False,
            )

        model, mAP = validation(
            model,
            device,
            valid_dataloader,
            csv_log_file_valid,
            tensor_type=torch.cuda.FloatTensor,
            num_classes=8,
        )

        # save casual weights here
        model.save_weights(
            os.path.join(weights_path, "weights_kitti-epoch-{}.pth".format(i + 1))
        )

        # update mAP
        if mAP > best_mAP:
            best_mAP = mAP
            print("Saving best weights")
            model.save_weights(os.path.join(weights_path, "best_weights_kitti.pth"))


text:
from pathlib import Path
import numpy as np

if __name__ == '__main__':
    with Path('vocab.words.txt').open(encoding='utf-8') as f:
        word_to_idx = {line.strip(): idx for idx, line in enumerate(f)}
    with Path('vocab.words.txt').open(encoding='utf-8') as f:
        word_to_found = {line.strip(): False for line in f}

    size_vocab = len(word_to_idx)

    embeddings = np.zeros((size_vocab, 300))

    found = 0
    print('Reading W2V file (may take a while)')
    with Path('../../sgns.zhihu.bigram').open(encoding='utf-8') as f:
        for line_idx, line in enumerate(f):
            if line_idx % 100000 == 0:
                print('- At line {}'.format(line_idx))
            line = line.strip().split()
            if len(line) != 300 + 1:
                continue
            word = line[0]
            embedding = line[1:]
            if (word in word_to_idx) and (not word_to_found[word]):
                word_to_found[word] = True
                found += 1
                word_idx = word_to_idx[word]
                embeddings[word_idx] = embedding
    print('- done. Found {} vectors for {} words'.format(found, size_vocab))

    # 保存 np.array
    np.savez_compressed('w2v.npz', embeddings=embeddings)


text:
import os
import os.path as osp

import cv2
import matplotlib.cm as cm
import matplotlib.pylab as pylab
import numpy as np
import tqdm
from matplotlib import pyplot as plt
from tqdm import tqdm

def plot_single_scale(scale_lst, size):
    pylab.rcParams['figure.figsize'] = size, size / 2

    plt.figure()
    for i in range(0, len(scale_lst)):
        s = plt.subplot(1, 5, i + 1)
        plt.imshow(scale_lst[i], cmap=cm.Greys_r)
        s.set_xticklabels([])
        s.set_yticklabels([])
        s.yaxis.set_ticks_position('none')
        s.xaxis.set_ticks_position('none')
    plt.tight_layout()
    plt.show()

def convert_maps_to_masks(
        maps_path='/home/ubuntu/Data/DAVIS/Augmentation/tmp_result/',
        save_to='/home/ubuntu/Dev/davis/data/DAVIS/Results/Segmentations/480p/osvos/',
        threshold=178.5):
    """
    convert the maps outputted by gygonet to binary segmentation masks

    """
    for root, dirs, files in tqdm(os.walk(maps_path),
                                  'Sequences: Maps-->Masks'):
        for file in files:
            if file[0] == '.':
                continue # ignore hidden files
            relative_root = root[len(maps_path) + 1:]
            prob_map = cv2.imread(osp.join(root, file), 0)
            mask = np.array(prob_map > threshold, dtype=np.uint8) * 255

            dest_dir = osp.join(save_to, relative_root)
            if not osp.exists(dest_dir):
                os.makedirs(dest_dir)
            cv2.imwrite(osp.join(dest_dir, file), mask)

def convert_map_to_mask(map, threshold=178.5, low_val=0, high_val=255):
    """
    Converts map to mask,
    all elements under threshold will get low_val and the rest will get high_val
    in the output mask
    :param map: array with grayscale levels
    :param threshold:
    :param low_val:
    :param high_val:
    :return mask: array with
    """
    mask = np.array(map).copy()
    thresh = threshold / 255.0
    mask_t = (map < thresh)
    mask[mask_t] = low_val
    mask_t = (map >= thresh)
    mask[mask_t] = high_val
    return mask

def display_maps(maps, save_fig_path='',
                 show_fig=False, verbose=False):
    """
    Enables to plot images where each image is plotted three times:
        left: `gray`, middle: `Normalized RdBu`, right: `clipped gray`
        :param maps: array of images of type nparray
        :param save_fig_path: if given saves figure at this path
        :param show_fig: if True show() figure
        :param verbose: if True prints debug info
    """

    if verbose:
        print('len(maps):%d; maps[0].shape:%s' %(len(maps),maps[0].shape.__str__()) )

    maxs = [np.max(maps[x]) for x in range(len(maps))]
    if verbose:
        print maxs
    mins = [np.min(maps[x]) for x in range(len(maps))]
    if verbose:
        print mins

    blob_max = np.max(maxs)
    blob_min = np.min(mins)
    if verbose:
        print blob_max
        print blob_min

    edge = max(blob_max, -blob_min)

    f, axarr = plt.subplots(len(maps), 3,
                            figsize=(14, 2 + 2 * len(maps)))
    if len(maps)>1:
        for x in range(len(maps)):
            axarr[x, 0].imshow(maps[x], cmap="gray")
            axarr[x, 0].axis('off')
            axarr[x, 1].imshow(maps[x], cmap="RdBu", vmin=-edge,
                               vmax=edge)
            axarr[x, 1].axis('off')
            axarr[x, 2].imshow(maps[x], cmap="gray", vmin=0, vmax=1)
            axarr[x, 2].axis('off')
    else:
        axarr[0].imshow(maps[0], cmap="gray")
        axarr[0].axis('off')
        axarr[1].imshow(maps[0], cmap="RdBu", vmin=-edge,
                           vmax=edge)
        axarr[1].axis('off')
        axarr[2].imshow(maps[0], cmap="gray", vmin=0, vmax=1)
        axarr[2].axis('off')

    if save_fig_path:
        fig1 = plt.gcf()
    if show_fig:
        plt.show()
    if save_fig_path:
        fig1.savefig(save_fig_path)

    plt.close()


text:
import hashlib
from pathlib import Path
from typing import List, Optional, Tuple, Union

import numpy as np
import pandas as pd
import tensorflow as tf
from google.cloud import storage
from google.cloud.storage import Blob
from PIL import Image, ImageOps
from PIL.Image import BICUBIC, NEAREST
from pycocotools.coco import COCO
from skimage.filters import gaussian
from skimage.transform import rotate
from tqdm import tqdm

from watch_recognition.utilities import Point

def load_binary_masks_from_coco_dataset(
    data_dir: Union[str, Path],
    image_size: Optional[Tuple[int, int]] = (224, 224),
    filter_classes: Optional[List[str]] = None,
    blur_masks: bool = False,
):

    data_dir = str(data_dir)
    if data_dir.startswith("gs://"):
        bucket_name = data_dir.replace("gs://", "").split("/")[0]
        storage_client = storage.Client()
        bucket = storage_client.get_bucket(bucket_or_name=bucket_name)
        prefix = data_dir.replace(f"gs://{bucket_name}/", "")
        blobs: List[Blob] = list(bucket.list_blobs(prefix=prefix))
        for blob in tqdm(blobs):
            download_path = Path("/tmp/" + blob.name)
            if not download_path.exists() or download_path.stat().st_size == 0:
                download_path.parent.mkdir(exist_ok=True, parents=True)
                blob.download_to_filename(download_path)
        data_dir = data_dir.replace(f"gs://{bucket_name}", "/tmp/")
        print("local data dir: {data_dir}")
    annotations_file = Path(data_dir) / "result.json"

    coco = COCO(annotations_file)
    catIds = coco.getCatIds(catNms=filter_classes)
    imgIds = coco.getImgIds(catIds=catIds)
    imgs = coco.loadImgs(imgIds)
    all_images = []
    all_filenames = []
    all_masks = []
    all_hashes = []
    for img in imgs:
        filename = img["file_name"]
        all_filenames.append(filename)
        image_path = f"{data_dir}/{filename}"
        img_np = load_image(image_path, image_size=None)

        annIds = coco.getAnnIds(imgIds=img["id"], catIds=catIds, iscrowd=None)
        anns = coco.loadAnns(annIds)
        with Image.fromarray(img_np) as img_pil:
            padded_img = ImageOps.pad(img_pil, size=image_size)
            all_images.append(np.array(padded_img))
            md5hash = hashlib.md5(img_pil.tobytes())
            float_hash = int(md5hash.hexdigest(), base=16) / 16 ** 32
            all_hashes.append(float_hash)

        masks = np.array([coco.annToMask(ann) for ann in anns])
        mask = ((np.sum(masks, axis=0) > 0) * 255).astype("uint8")
        with Image.fromarray(mask) as mask_pil:
            padded_mask = ImageOps.pad(mask_pil, size=image_size, method=NEAREST)
            padded_mask = (np.array(padded_mask) > 0).astype("float32")
            padded_mask = np.expand_dims(padded_mask, axis=-1)
            if blur_masks:
                padded_mask = gaussian(padded_mask)
                padded_mask /= padded_mask.max()
            all_masks.append(padded_mask)

    all_images = np.array(all_images)
    all_masks = np.array(all_masks)
    all_filename = np.array(all_filenames)
    all_hashes = np.array(all_hashes)

    return all_images, all_masks, all_filename, all_hashes

def load_single_kp_example(
    source: Path,
    image_name: str,
    image_size: Optional[Tuple[int, int]] = (224, 224),
):
    labels_df = pd.read_csv(source / f"tags.csv")
    image_path = source / image_name

    img = tf.keras.preprocessing.image.load_img(
        image_path, "rgb", target_size=image_size, interpolation="bicubic"
    )
    data = labels_df[labels_df["crop_file"] == image_name]
    return img, data

def load_keypoints_data_as_kp(
    source: str,
    image_size: Tuple[int, int] = (224, 224),
    skip_examples_without_all_keypoints: bool = True,
    min_image_size: Optional[Tuple[int, int]] = (100, 100),
    autorotate: bool = False,
):
    if not source.endswith("/"):
        source += "/"
    with tf.io.gfile.GFile(source + "tags.csv", "r") as f:
        labels_df = pd.read_csv(f)
    all_keypoints = []
    all_images = []
    all_filenames = []
    for image_name, data in labels_df.groupby("crop_file"):
        if skip_examples_without_all_keypoints and len(data) < 4:
            continue
        if len(data["label"].unique()) != 4:
            print(f"{image_name} keypoints are not unique")
        image_path = source + f"{image_name}"

        with tf.io.gfile.GFile(image_path, "rb") as f:
            with Image.open(f) as img:
                if img.mode != "RGB":
                    img = img.convert("RGB")
                image_too_small = (
                    img.size[0] < min_image_size[0] or img.size[1] < min_image_size[1]
                )
                if min_image_size is not None and image_too_small:
                    continue

                if image_size is not None:
                    img = img.resize(image_size, BICUBIC)
                image_np = tf.keras.preprocessing.image.img_to_array(img).astype(
                    "uint8"
                )
        points = []
        for tag in ["Center", "Top", "Hour", "Minute"]:
            tag_data = data[data["label"] == tag]
            # if tag is missing, negative values will throw it outside the image
            # TODO this should probably be handled better by using the keypoint
            #  labels explicitly
            if not tag_data.empty:
                # two last coordinate values are ignored, 4 values are required to
                # correctly use albumentations for keypoints with tf.Data
                point = np.array(
                    (tag_data["x"].values[0], tag_data["y"].values[0], 0, 0)
                )
                point[0] *= image_np.shape[0]
                point[1] *= image_np.shape[1]
                int_point = np.round(point).astype(int)
                kp = tuple(int_point)
            else:
                kp = (-100, -100, 0, 0)
                # raise ValueError(f"no keypoint data for {tag} on {image_name}")

            points.append(kp)
        points = np.array(points)

        if autorotate:
            angle = keypoints_to_angle(points[0, :2], points[1, :2])
            image_np = rotate(image_np.astype("float32"), -angle).astype("uint8")
            origin_point = Point(image_np.shape[0] / 2, image_np.shape[1] / 2)
            for i in range(4):
                points[i, :2] = np.array(
                    Point(*points[i, :2])
                    .rotate_around_origin_point(origin_point, angle)
                    .as_coordinates_tuple
                )

        all_images.append(image_np)
        all_filenames.append(data["crop_file"].values[0])

        all_keypoints.append(points)
    all_images = np.array(all_images)
    all_keypoints = np.array(all_keypoints)
    all_filename = np.array(all_filenames)

    return all_images, all_keypoints, all_filename

def load_image(image_path: str, image_size: Tuple[int, int]):
    if image_path.startswith("gs://"):
        file = tf.io.gfile.GFile(image_path, "rb")
    else:
        file = open(image_path, "rb")
    with Image.open(file) as img:
        if img.mode != "RGB":
            img = img.convert("RGB")
        if image_size is not None:
            img = img.resize(image_size, BICUBIC)

        image_np = tf.keras.preprocessing.image.img_to_array(img).astype("uint8")
    file.close()
    return image_np

def keypoints_to_angle(center, top):
    top = top - center
    top = top / np.linalg.norm(top)

    reference_vector = np.array([0, -0.5])
    reference_vector = reference_vector / np.linalg.norm(reference_vector)
    angle = np.rad2deg(np.arctan2(*top) - np.arctan2(*reference_vector))
    return int(angle)

def binarize(value, bin_size):
    if value < 0:
        value += 360
    n_bins = 360 // bin_size
    b = (value + bin_size / 2) / bin_size
    b = int(b)
    return b % n_bins

max_stars_repo_name: akucia/analog-watch-recognition

max_issues_repo_name: akucia/analog-watch-recognition

max_forks_repo_name: akucia/analog-watch-recognition


text:
'''For analyzing data'''

from numpy import ones, convolve, log10, sqrt, arctan2, diff, array, unwrap, gradient
from scipy.fftpack import rfft, rfftfreq, irfft
from sklearn.preprocessing import minmax_scale

import matplotlib.pyplot as plt

# curve display
def curve(x, y, title, xlabel, ylabel, style="-k"):
    fig, ax = plt.subplots(1, 1, sharex=True, sharey=False)
    ax.set(title=title, xlabel=xlabel, ylabel=ylabel)
    if len(array(x).shape) == 1:
        ax.plot(x, y, style)
    elif len(array(x).shape) > 1:
        for x,y,s in zip(x,y,style):
            ax.plot(x, y, s)
    fig.tight_layout()
    plt.show()

# differentiation
def derivative(x, y, step=1):
    X, Y = x[::step], y[::step]
    dydx = diff(Y) / diff(X)
    return X[1::], dydx

# Extract IQ (to be changed to take in list instead of np.array)
def IQAP(I, Q):
    if I==0 and Q==0:
        Amp = -1000
        Pha = 0
    else:
        Amp = 20*log10(sqrt(I**2 + Q**2))
        Pha = arctan2(Q, I) # -pi < phase < pi
    return Amp, Pha
def IQAParray(datas): # datas
    '''
    Slice datas-array into IQ-data, then list the Amp (dB/dBm) & Pha (rad) out of it
    '''
    IQdata = datas.reshape(len(datas)//2, 2) # sort out interlaced IQ-pairs into I-list & Q-list
    Idata, Qdata = IQdata[:,0], IQdata[:,1]
    yI, yQ = [float(i) for i in Idata], [float(i) for i in Qdata]
    Amp, Pha = [], []
    for I, Q in zip(yI, yQ):
        if I==0 and Q==0:
            Amp.append(-1000)
            Pha.append(0)
        else:
            Amp.append(20*log10(sqrt(I**2 + Q**2)))
            Pha.append(arctan2(Q, I)) # -pi < phase < pi
    return yI, yQ, Amp, Pha

# moving average
def smooth(y, box_pts):
    box = ones(box_pts)/box_pts
    y_smooth = convolve(y, box, mode='same')
    return y_smooth

def FFT_deNoise(y, dx, noise_level, noise_filter=0.1):
    w = rfft(y)
    f = rfftfreq(len(y), dx)
    spectrum = w**2
    cutoff = spectrum < (spectrum.max()*noise_level*noise_filter)
    w_clean = w.copy()
    w_clean[cutoff] = 0
    y_clean = irfft(w_clean)
    return f, spectrum, w_clean, y_clean

def UnwraPhase(X, Pha, Flatten=True, Normalized=True):
    UPHA = unwrap(Pha)
    if Flatten:
        UPHA = gradient(UPHA, X)
    if Normalized:
        UPHA = minmax_scale(UPHA)
    return UPHA

def cleantrace(V):
    '''take out repeating element(s) from a trace / list in a progressing manner.
    1. Please note that it is NOT removing duplicate(s) per se
    2. Intrusive: V will be modified directly by this method'''
    turn = 0
    order = [x for x in range(len(V))] # V's indexes
    if len(V) > 1:
        while turn < len(V) - 1:
            if V[turn] == V[turn+1]:
                V.pop(turn+1)
                order.pop(turn+1)
                # print(turn)
            else:
                turn += 1
        return order
    else: return order

# Fitting

def test():
    x = [1,2,2,2,3,3,3,3,3,3,3.5,5,5,5,5,5,5,5,7,7,7,8,8,8,8,8,8,12,12,13,12,12,12,10,8,7,5,5,5,5,5,3]
    y = [1,2,2,2,3,3,3,3,3,3,3.5,5,5,5,5,5,5,5,7,7,7,8,8,8,8,8,8,12,12,13,12,12,12,10,8,7,5,5,5,5,5,3]
    print('x-before: %s' %x)
    order = cleantrace(x)
    print('x-after: %s' %x)
    ycleaned = [y[i] for i in order]
    print('order:\n%s\ny-cleaned:\n%s' %(order, ycleaned))
    return

# test()

max_stars_repo_name: takehuge/PYQUM

max_issues_repo_name: takehuge/PYQUM

max_forks_repo_name: takehuge/PYQUM

reason: from numpy,from scipy


text:
import numpy as np
import pytest

from tf_pwa.data import *
from tf_pwa.particle import BaseDecay, BaseParticle

from .common import write_temp_file

def test_index():
    data = {"s": [{"s": 1}, {"d": 2}], "p": (3, 4)}
    assert data_index(data, ("s", 1)) == {"d": 2}
    assert data_index(data, ["p", 0]) == 3
    assert data_index(data, "p") == (3, 4)
    with pytest.raises(ValueError):
        data_index(data, "t")

def test_index2():
    a = BaseParticle("A")
    b = BaseParticle("B")
    c = BaseParticle("C")
    d = BaseDecay(a, [b, c])
    data = {
        "particle": {a: {"p": 1}, b: {"m": 2}},
        "decay": [
            {d: {b: {"a": 3}, c: {"a": 4}}},
        ],
    }
    assert data_index(data, ["particle", a, "p"]) == 1
    assert data_index(data, ["particle", "B", "m"]) == 2
    assert data_index(data, ["decay", 0, str(d), b, "a"]) == 3
    assert data_index(data, ["decay", 0, d, str(c), BaseParticle("a")]) == 4

def test_flatten_dict_data():
    a = BaseParticle("A")
    b = BaseParticle("B")
    c = BaseParticle("C")
    d = BaseDecay(a, [b, c])
    data = {
        "particle": {a: {"p": 1}, b: {"m": 2}},
        "decay": [
            {d: {b: {"a": 3}, c: {"a": 4}}},
        ],
    }
    data_f = flatten_dict_data(data)
    assert len(data_f) == 4
    assert data_f["particle/A/p"] == 1
    assert data_f["particle/B/m"] == 2
    assert data_f["decay/0/{}/B/a".format(d)] == 3
    assert data_f["decay/0/{}/C/a".format(d)] == 4

def test_load_dat_file():
    s1 = "5.0 2.0 3.0 3.0\n"
    s2 = "4.0 2.0 4.0 2.0\n4.0 2.0 4.0 2.0\n"
    print(s1 + s2)
    with write_temp_file(s1 + s2) as fname:
        dat1 = load_dat_file(fname, ["a", "b", "c"])
    with write_temp_file(s1) as fname1:
        with write_temp_file(s2) as fname2:
            dat2 = load_dat_file([fname1, fname2], ["a", "b", "c"])
    assert np.allclose(dat1["a"], dat2["a"])
    assert np.allclose(dat1["b"], dat2["b"])
    assert np.allclose(dat1["c"], dat2["c"])

def test_save_load():
    b = BaseParticle("b")
    data = {"a": np.array([1.0, 2.0, 3.0]), b: np.array([2.0, 3.0, 4.0])}

    fname = "test_save_load.npy"
    save_data(fname, data)
    data2 = load_data(fname)
    assert np.allclose(data["a"], data2["a"])
    assert np.allclose(data[b], data2[b])

def test_set_seed():
    set_random_seed(100)
    a = np.random.random((10,))
    set_random_seed(100)
    b = np.random.random((10,))
    assert np.allclose(a, b)

max_stars_repo_name: ReynLieu/tf-pwa

max_issues_repo_name: ReynLieu/tf-pwa

max_forks_repo_name: ReynLieu/tf-pwa


text:
#!/usr/bin/python

# Standard imports
import cv2
import numpy as np;

# Read image
#im = cv2.imread("blob.jpg", cv2.IMREAD_GRAYSCALE)

# Start video capture
capture = cv2.VideoCapture(0)

# Setup SimpleBlobDetector parameters.
params = cv2.SimpleBlobDetector_Params()

# Change thresholds
params.minThreshold = 10
params.maxThreshold = 200

# Filter by Area.
params.filterByArea = True
params.minArea = 50

# Filter by Circularity
params.filterByCircularity = True
params.minCircularity = 0.1

# Filter by Convexity
params.filterByConvexity = True
params.minConvexity = 0.3

# Filter by Inertia
params.filterByInertia = True
params.minInertiaRatio = 0.01

# Create a detector with the parameters
ver = (cv2.__version__).split('.')
if int(ver[0]) < 3 :
	detector = cv2.SimpleBlobDetector(params)
else :
	detector = cv2.SimpleBlobDetector_create(params)

# Range for lower red
lower_redl = np.array([0, 120, 70])
upper_redl = np.array([10, 255, 255])

# Range for upper red
lower_redu = np.array([170, 120, 70])
upper_redu = np.array([180, 255, 255])

while(True):

    # Draw frame
    ret, im = capture.read()

    # Converting from BGR to HSV color space
    hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)

    # Compute mask
    mask1 = cv2.inRange(hsv, lower_redl, upper_redl)
    mask2 = cv2.inRange(hsv, lower_redu, upper_redu)

    # Bitwise NOR for two ranges
    mask2 = cv2.bitwise_not(mask1 + mask2)

    # Bitwise AND
    #res = cv2.bitwise_and(im, im, mask=mask2)

    # Detect blobs.

    keypoints = detector.detect(mask2)

    # Draw detected blobs as red circles.
    # cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS ensures
    # the size of the circle corresponds to the size of blob

    im_with_keypoints = cv2.drawKeypoints(im, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

    # Show blobs
    cv2.imshow("Keypoints", im_with_keypoints)
    if cv2.waitKey(1) == 27:
        break

capture.release()
cv2.destroyAllWindows()