text:
import random

import numpy as np
from audio_zen.acoustics.feature import norm_amplitude, tailor_dB_FS, is_clipped, load_wav, subsample
from audio_zen.dataset.base_dataset import BaseDataset
from audio_zen.utils import expand_path
from joblib import Parallel, delayed
from scipy import signal
from tqdm import tqdm

class Dataset(BaseDataset):
    def __init__(self,
                 clean_dataset,
                 clean_dataset_limit,
                 clean_dataset_offset,
                 noise_dataset,
                 noise_dataset_limit,
                 noise_dataset_offset,
                 rir_dataset,
                 rir_dataset_limit,
                 rir_dataset_offset,
                 snr_range,
                 reverb_proportion,
                 silence_length,
                 target_dB_FS,
                 target_dB_FS_floating_value,
                 sub_sample_length,
                 sr,
                 pre_load_clean_dataset,
                 pre_load_noise,
                 pre_load_rir,
                 num_workers
                 ):
        """
        Dynamic mixing for training

        Args:
            clean_dataset_limit:
            clean_dataset_offset:
            noise_dataset_limit:
            noise_dataset_offset:
            rir_dataset:
            rir_dataset_limit:
            rir_dataset_offset:
            snr_range:
            reverb_proportion:
            clean_dataset: scp file
            noise_dataset: scp file
            sub_sample_length:
            sr:
        """
        super().__init__()
        # acoustics args
        self.sr = sr

        # parallel args
        self.num_workers = num_workers

        clean_dataset_list = [line.rstrip('\n') for line in open(expand_path(clean_dataset), "r")]
        noise_dataset_list = [line.rstrip('\n') for line in open(expand_path(noise_dataset), "r")]
        rir_dataset_list = [line.rstrip('\n') for line in open(expand_path(rir_dataset), "r")]

        clean_dataset_list = self._offset_and_limit(clean_dataset_list, clean_dataset_offset, clean_dataset_limit)
        noise_dataset_list = self._offset_and_limit(noise_dataset_list, noise_dataset_offset, noise_dataset_limit)
        rir_dataset_list = self._offset_and_limit(rir_dataset_list, rir_dataset_offset, rir_dataset_limit)

        if pre_load_clean_dataset:
            clean_dataset_list = self._preload_dataset(clean_dataset_list, remark="Clean Dataset")

        if pre_load_noise:
            noise_dataset_list = self._preload_dataset(noise_dataset_list, remark="Noise Dataset")

        if pre_load_rir:
            rir_dataset_list = self._preload_dataset(rir_dataset_list, remark="RIR Dataset")

        self.clean_dataset_list = clean_dataset_list
        self.noise_dataset_list = noise_dataset_list
        self.rir_dataset_list = rir_dataset_list

        snr_list = self._parse_snr_range(snr_range)
        self.snr_list = snr_list

        assert 0 <= reverb_proportion <= 1, "reverberation proportion should be in [0, 1]"
        self.reverb_proportion = reverb_proportion
        self.silence_length = silence_length
        self.target_dB_FS = target_dB_FS
        self.target_dB_FS_floating_value = target_dB_FS_floating_value
        self.sub_sample_length = sub_sample_length

        self.length = len(self.clean_dataset_list)

    def __len__(self):
        return self.length

    def _preload_dataset(self, file_path_list, remark=""):
        waveform_list = Parallel(n_jobs=self.num_workers)(
            delayed(load_wav)(f_path) for f_path in tqdm(file_path_list, desc=remark)
        )
        return list(zip(file_path_list, waveform_list))

    @staticmethod
    def _random_select_from(dataset_list):
        return random.choice(dataset_list)

    def _select_noise_y(self, target_length):
        noise_y = np.zeros(0, dtype=np.float32)
        silence = np.zeros(int(self.sr * self.silence_length), dtype=np.float32)
        remaining_length = target_length

        while remaining_length > 0:
            noise_file = self._random_select_from(self.noise_dataset_list)
            noise_new_added = load_wav(noise_file, sr=self.sr)
            noise_y = np.append(noise_y, noise_new_added)
            remaining_length -= len(noise_new_added)

            # 如果还需要添加新的噪声，就插入一个小静音段
            if remaining_length > 0:
                silence_len = min(remaining_length, len(silence))
                noise_y = np.append(noise_y, silence[:silence_len])
                remaining_length -= silence_len

        if len(noise_y) > target_length:
            idx_start = np.random.randint(len(noise_y) - target_length)
            noise_y = noise_y[idx_start:idx_start + target_length]

        return noise_y

    @staticmethod
    def snr_mix(clean_y, noise_y, snr, target_dB_FS, target_dB_FS_floating_value, rir=None, eps=1e-6):
        """
        混合噪声与纯净语音，当 rir 参数不为空时，对纯净语音施加混响效果

        Args:
            clean_y: 纯净语音
            noise_y: 噪声
            snr (int): 信噪比
            target_dB_FS (int):
            target_dB_FS_floating_value (int):
            rir: room impulse response, None 或 np.array
            eps: eps

        Returns:
            (noisy_y，clean_y)
        """
        if rir is not None:
            if rir.ndim > 1:
                rir_idx = np.random.randint(0, rir.shape[0])
                rir = rir[rir_idx, :]

            clean_y = signal.fftconvolve(clean_y, rir)[:len(clean_y)]

        clean_y, _ = norm_amplitude(clean_y)
        clean_y, _, _ = tailor_dB_FS(clean_y, target_dB_FS)
        clean_rms = (clean_y ** 2).mean() ** 0.5

        noise_y, _ = norm_amplitude(noise_y)
        noise_y, _, _ = tailor_dB_FS(noise_y, target_dB_FS)
        noise_rms = (noise_y ** 2).mean() ** 0.5

        snr_scalar = clean_rms / (10 ** (snr / 20)) / (noise_rms + eps)
        noise_y *= snr_scalar
        noisy_y = clean_y + noise_y

        # Randomly select RMS value of dBFS between -15 dBFS and -35 dBFS and normalize noisy speech with that value
        noisy_target_dB_FS = np.random.randint(
            target_dB_FS - target_dB_FS_floating_value,
            target_dB_FS + target_dB_FS_floating_value
        )

        # 使用 noisy 的 rms 放缩音频
        noisy_y, _, noisy_scalar = tailor_dB_FS(noisy_y, noisy_target_dB_FS)
        clean_y *= noisy_scalar

        # 合成带噪语音的时候可能会 clipping，虽然极少
        # 对 noisy, clean_y, noise_y 稍微进行调整
        if is_clipped(noisy_y):
            noisy_y_scalar = np.max(np.abs(noisy_y)) / (0.99 - eps)  # 相当于除以 1
            noisy_y = noisy_y / noisy_y_scalar
            clean_y = clean_y / noisy_y_scalar

        return noisy_y, clean_y

    def __getitem__(self, item):
        clean_file = self.clean_dataset_list[item]
        clean_y = load_wav(clean_file, sr=self.sr)
        clean_y = subsample(clean_y, sub_sample_length=int(self.sub_sample_length * self.sr))

        noise_y = self._select_noise_y(target_length=len(clean_y))
        assert len(clean_y) == len(noise_y), f"Inequality: {len(clean_y)} {len(noise_y)}"

        snr = self._random_select_from(self.snr_list)
        use_reverb = bool(np.random.random(1) < self.reverb_proportion)

        noisy_y, clean_y = self.snr_mix(
            clean_y=clean_y,
            noise_y=noise_y,
            snr=snr,
            target_dB_FS=self.target_dB_FS,
            target_dB_FS_floating_value=self.target_dB_FS_floating_value,
            rir=load_wav(self._random_select_from(self.rir_dataset_list), sr=self.sr) if use_reverb else None
        )

        noisy_y = noisy_y.astype(np.float32)
        clean_y = clean_y.astype(np.float32)

        return noisy_y, clean_y

max_stars_repo_name: hit-thusz-RookieCJ/FullSubNet-plus

max_issues_repo_name: hit-thusz-RookieCJ/FullSubNet-plus

max_forks_repo_name: hit-thusz-RookieCJ/FullSubNet-plus


text:
#!/usr/bin/env python

"""

# batch2_remove_pseudo.py
# usage: python %prog infile.txt
# python2.7

# infile.txt should contain length of seq of 100 species:
'name' +'\t'+ 'hg38_length' +'\t'+ 'panTro4_length' +'\t'+ 'gorGor3_length' +'\t'+ ... 'petMar2_length'

"""

import os,sys,re
import numpy as np

f_path=sys.argv[1]

nonmammal_pseudo=[]
armadillo_pseudo=[]
non_pseudo=[]
with open(f_path, 'r') as infile:
    for line in infile:
        lsplit=line.strip().split()
        df=np.array(lsplit[1:], dtype='int64')
        df=df*100/df[0]
        n=0
        for i in range(62,100):
            if df[i] >= 50:
                n=n+1
        if n >= 14:
            nonmammal_pseudo.append(lsplit[0])
        elif df[61] >= 50:
            armadillo_pseudo.append(lsplit[0])
        else:
            non_pseudo.append(line)
with open('pseudo_non_mammal_name.txt', 'w') as out_non_mammal:
    out_non_mammal.write('\n'.join(nonmammal_pseudo))
with open('pseudo_armadillo_name.txt', 'w') as out_armadillo:
    out_armadillo.write('\n'.join(armadillo_pseudo))
with open('not_pseudo.txt', 'w') as out_not_pseudo:
    out_not_pseudo.write(''.join(non_pseudo))

ext: py

reason: import numpy


text:
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pdb

map_names = ['main_grid', 'firebreaks']
scenarios = {'main_grid': ['InitialValues', 'DefensibleSpace', 'IWUIC', 'Buildings', 'Vegetation'],
			 'firebreaks': ['InitialValues']}
hist_names = ['damage_hist', 'lives_hist', 'length_hist']
hist_titles = ['Building Damage', 'Number of Deaths', 'Fire Lengths']
hist_axis_labels = ['Building Damage ($)', 'Number of Deaths', 'Fire Length (time steps)']
ylabel = '# episodes'

n = len(hist_names)
hists = []
file_names = []
for map_name in map_names:
	for scenario in scenarios[map_name]:
		hist_set = []
		data_path = path = 'charts/histograms/' + scenario + '_' + map_name
		for hist_name in hist_names:
			with open(data_path + '/' + hist_name + '.txt', 'r') as file:
				hist_set.append(np.array([float(line.strip()) for line in file]))
		hists.append(hist_set)

		file_names.append('charts/' + scenario + '-' + map_name + '_histogram.png')

hist_maxes = [0] * 3
for scenario in hists:
	for i in range(n):
		if np.max(scenario[i]) > hist_maxes[i]:
			hist_maxes[i] = np.max(scenario[i])
hist_maxes = [hist_max * 2/3 for hist_max in hist_maxes]

for scenario, file_name in zip(hists, file_names):
	num_episodes = len(scenario[0])

	fig, axs = plt.subplots(ncols = n)

	print(file_name)
	for i in range(n):
		axs[i] = sns.distplot(a=scenario[i], ax=axs[i], hist=True, kde=False, norm_hist=False, axlabel=hist_axis_labels[i][0])
		axs[i].set_title(hist_titles[i] + ' (' + str(num_episodes) + ' episodes)')
		axs[i].set(ylabel = ylabel)
		axs[i].text(hist_maxes[i]*0.6,num_episodes*0.825,'mean : {0:.3e}\nstdev:  {1:.3e}'.format(np.mean(scenario), np.std(scenario)) ,fontsize=12)
		axs[i].set_xlim(left=0,right=hist_maxes[i])
		axs[i].set_ylim(bottom=0,top=num_episodes)
		print(round(np.mean(scenario[i]),3))

	fig.set_figwidth(20)
	fig.savefig(file_name)


text:
from pathlib import Path

import numpy as np

from src.PARAMATERS import project_dir
from src.utils.distributions import MoG

if __name__ == '__main__':

    # Spiked dataset
    mus = np.array([[-1, -1], [1, 1]])
    sigmas = np.zeros((2, 2, 2))
    sigmas[0] = 0.2 * np.eye(2)
    sigmas[1] = 0.2 * np.eye(2)
    p = np.array([0.5, 0.5])
    mog_spiked = MoG(d=2, mus=mus, sigmas=sigmas, p=p)

    # Sample n points
    n = 500  # Should refactor this to command line configurable
    X_spiked = mog_spiked.sample(n)

    # Save datasets
    mog_data_path = Path(project_dir) / 'data' / 'synthetic' / 'mog_datasets'
    if not mog_data_path.is_dir():
        mog_data_path.mkdir()

    np.save(str(mog_data_path / 'X_2d_2mog_spiked.npy'), X_spiked)


text:
import collections
from scipy.stats import norm
import matplotlib.pyplot as plt
import numpy as np

def readFile(file="input.txt"):
    try:
        f=open(file)
    except:
        print("file not found, in current directory.")
    data=[]
    for line in f:
        line=f.readline()
        line=line+" "+line[:-1]
    data=line.split(' ')
    r = collections.Counter(data)
    from context import resource_manager

    from cluster.dbscan import dbscan_chrisjmccormick
    f = input()
    r = f.split(' ')
    filename = ''
    if len(r) > 1:
        n = int(r[1])
    else:
        n = int(input())
    filename = str(r[0])
    result = readFile(file=filename)
    for k in result:
        if result[k] <= n:
            continue
        else:
            print(k + " " + str(result[k]))
    return r
    # main.save()
    import numpy as np

    a = np.random.normal(size=1000)

    bins = np.arange(-8, 9)
    # bins = np.array([-1.2,2.3,3.1,3.3,4.2,4.5,4.7,5])
    print(a)
    print(bins)

    histogram = np.histogram(a, bins=bins, normed=True)[0]

    bins = 0.5*(bins[1:] + bins[:-1])

    from scipy import stats

    b = stats.norm.pdf(bins)  # norm是正态分布
    import matplotlib.pyplot as plt
    plt.plot(bins, b)
    plt.show()
    print(b)
    #plt.plot(bins, histogram)
    plt.plot(bins, b)
    plt.show()
def _calc_ent(probs):
    """
    计算信息熵
    :param probs: numpy结构
    :return: 返回probs的信息熵
    """
    ent = - probs.dot(np.log2(probs))
    return ent

def gauss_function(data=np.array([1,2,3,4,5,6,7,8])):
    random_sample=data

    # Generate an array of 200 random sample from a normal dist with
    # mean 0 and stdv 1
    #random_sample = norm.rvs(loc=0,scale=1,size=200)
    sample=data
    from scipy import stats
    import numpy as np
    from scipy.stats import norm
    mu, std = norm.fit(data)
    print(mu)
    print(std)

if __name__ == '__main__':
    t=np.array([1,2,3,4,5])
    for i in range(0,3):
        print(i)

max_stars_repo_name: gwdgithubnom/ox-patient

max_issues_repo_name: gwdgithubnom/ox-patient

max_forks_repo_name: gwdgithubnom/ox-patient

reason: import numpy,from scipy


text:
#%%
import numpy as np
import matplotlib.pyplot as plt
from Halton_Points import haltonPoints
from augmented_system import aug_matrix
from scipy import linalg

# %%
def distanceMatrix2D(M):
    n = M.shape[0]
    my_matrix = list()
    for j in M:
        for l in M:
            aux = j-l
            my_matrix.append(linalg.norm(aux, ord=2))

    my_matrix = np.array(my_matrix)
    return my_matrix.reshape(n,n)

def GaussianRBF(M, e=1):
     return np.exp(-(e*M)**2)

def test_func(x,y):
    return (x+y)/2

# %%
n = 512
m = 3
M = haltonPoints(2, n)
x, y = M[:, 0], M[:, 1]

A = GaussianRBF(distanceMatrix2D(M))
B = aug_matrix(A, M, n, m)
F = np.hstack((test_func(x, y), np.zeros((m))))
C = linalg.solve(B, F)

# %%
error = list()
for _ in range (int(1e3)):
    n_test = 1
    M_test = np.random.random(n_test*2).reshape((n_test,2))

    M_dist_tot = list()
    for p_test in M_test:
        M_dist = list()
        for p_col in M:
            M_dist.append(linalg.norm(p_test-p_col, ord = 2))
        M_dist_tot.append(M_dist)

    M_dist = np.array(M_dist)
    res = np.dot(C[:-m], np.exp(-(M_dist)**2)) + np.dot(np.array([1, M_test[0][0], M_test[0][1]]), C[-m:])
    error.append(abs(res - test_func(M_test[:,0], M_test[:,1])[0]))

error = np.array(error)
plt.hist(error)
# %%


text:
import itertools
from   matplotlib import pyplot as plt
import numpy as np
import pandas as pd
from   trueskill import TrueSkill

class RatingsInfo:
    elo = None
    wins = None
    losses = None
    def __init__(self, env):
        self.elo = env.create_rating()
        self.wins, self.losses = 0, 0

    def get_elo(self):
        return self.elo.mu * 4.0  # scale to 100

def get_elo(d, env, player):
    '''
    Get elo from dict d, if not present, create a new RatingsInfo
    object and return its elo
    '''
    if player not in d: d[player] = RatingsInfo(env)
    return d[player].elo

def calc_elo(df):
    '''
    For a given input dataframe, calculates the running elo of each player
    in each team. Returns a dictionary of players to latest elos, and also
    returns the running df
    '''

    env = TrueSkill(draw_probability=0)
    ratings = {}
    df['elos'] = None
    df['elos'] = df['elos'].astype(object)  # avoid insertion errors later
    for idx, row in df.iterrows():
        teams = [[get_elo(ratings, env, player) for player in team]\
                for team in row['teams']]
        elos = env.rate(teams, row['ranks'])
        has_winner = sum(row['ranks']) > 0
        for team, team_elos, team_rank in zip(row['teams'], elos, row['ranks']):
            for player, player_elo in zip(team, team_elos):
                ratings[player].elo = player_elo
                if has_winner:
                    if team_rank == 0: ratings[player].wins += 1
                    else: ratings[player].losses += 1
        #df.at[idx, 'elos'] = [elos]
        df.at[idx, 'elos'] = elos
    return ratings, df

def parse_results():
    '''
    Parses results.txt and runs calc_elo on the aggregate df as well
    as per-game dfs
    '''
    results = []
    with open('results.txt') as f:
        for line in f.readlines():
            if line.startswith('DATE'): continue
            date, game, teams, ranks = line.split('|')
            teams, ranks = teams.split(';'), ranks.split(';')
            teams = [team.split(',') for team in teams]
            ranks = [int(rank) for rank in ranks]
            results.append((date, game, teams, ranks))

    res_df = pd.DataFrame(results, columns=['date', 'game', 'teams', 'ranks'])

    elos = {}
    for game in res_df.game.unique():
        elos[game], _ = calc_elo(res_df.loc[res_df.game == game])
        # (DEBUG) if game == "Secret Hitler": _.to_csv('./test.csv')
    return calc_elo(res_df) + (elos,)  # Tuple concatenation

def main():
    '''
    Calls parse_results() and then updates README.md based on return value.
    '''

    all_ratings, all_df, per_game_ratings = parse_results()

    markdown = '''
![Image](https://media.architecturaldigest.com/photos/618036966ba9675f212cc805/16:9/w_2560%2Cc_limit/SquidGame_Season1_Episode1_00_44_44_16.jpg)

### Total Rankings

| Player | ELO | Wins | Losses | Win % | Best Game |
| --- | --- | --- | --- | --- | --- |'''

    best_game = {}
    for game, ratings in per_game_ratings.items():
        for player, rating in ratings.items():
            if player not in best_game or rating.get_elo() > best_game[player][1]:
                best_game[player] = (game, rating.get_elo())

    infrequent_players = []  # players with <10 games
    for player, rating in sorted(
            all_ratings.items(),
            key=lambda item: item[1].get_elo(),
            reverse=True):
        markdown += '''
| {} | {:.2f} | {} | {} | {:.2f} | {} |'''.format(
        player, rating.get_elo(), rating.wins, rating.losses,
        rating.wins / (rating.wins + rating.losses), best_game[player][0])
        if rating.wins + rating.losses < 10: infrequent_players.append(player)

    chart_df = pd.DataFrame()
    for _, row in all_df.iterrows():
        players = list(np.array(row['teams']).flat)
        ratings = [elo.mu * 4 for elo in \
                itertools.chain.from_iterable(row['elos'])]
        #ratings = [elo.mu * 4 for elo in np.array(row['elos']).flat]
        for player, rating in zip(players, ratings):
            chart_df.at[row['date'], player] = rating
    chart_df.ffill(inplace=True)
    chart_df.drop(columns=infrequent_players, inplace=True)
    chart_df.plot.line().legend(
            loc='lower center', ncol=5, bbox_to_anchor=(0.5, -0.3))
    plt.subplots_adjust(bottom=0.25)
    plt.savefig('rankings.png')

    markdown += '''

### Rankings over Time
![Image](rankings.png)'''

    for game, ratings in per_game_ratings.items():
        markdown += '''

### {}

| Player | ELO | Wins | Losses | Win % |
| --- | --- | --- | --- | --- |'''.format(game)
        for player, rating in sorted(ratings.items(), key=lambda item: item[1].get_elo(), reverse=True):
            markdown += '''
| {} | {:.2f}  | {} | {} | {:.2f} |'''.format(
            player, rating.get_elo(), rating.wins, rating.losses,
            rating.wins / (rating.wins + rating.losses))

    with open('README.md', 'w') as f:
        f.write(markdown)

if __name__ == "__main__":
    main()

max_stars_repo_name: oliffur/gamenightrankings

max_issues_repo_name: oliffur/gamenightrankings

max_forks_repo_name: oliffur/gamenightrankings


text:
import numpy as np
import matplotlib.pyplot as plt
from condensate.core import gpcore

# constants
hbar = 1.054571817e-34
a0 = 5.29177210903e-11

class Environment():
    """
    The Environment class contains everything that is not directly related to the wavefunction.
    This includes potentials, spoons, dimensions, numbers of atoms, and more
    Many of the parameters do not change over an experiment. Others are settings for various experiments.
    Fixed parameters:
        - DIM: dimension of the square grid
        - fov: field of view of the calculation, in meters
        - N: number of atoms

    Variable parameters:
        - V: external potential
        - reference frame: this sets whether the calculation is in the lab frame or in the rotating frame
        - edge: this adds an edge potential to the calculation
        - spoon: this adds a spoon, controlled by either the mouse or a Leap Motion controller

    """

    def __init__(self, DIM=512, fov=400e-6, N=1e6, omegaz=10, a_s=100*a0, mass=3.8175406e-26):
        self.DIM = DIM
        self.fov = fov
        self.dx = fov/DIM
        self.N = N
        self._omegaz = omegaz
        self.a_s = a_s
        self.mass = mass
        self.g = N* 4 * np.pi * (hbar**2) * (a_s  / mass)
        self.g *= np.sqrt(mass * omegaz / (2*np.pi*hbar))
        self.omega = 2*np.pi
        self.epsilon = 0
        self.lb = np.sqrt(hbar / (2*mass *self.omega))

        self.V = np.zeros((DIM,DIM))
        self.use_custom_V= False

        # when doing dynamics with time varying harmonic trap, the ramp_trap is useful, but the g will not change,
        self.ramp_trap = {'ramp': False, 'omega': [self.omega], 'epsilon': [self.epsilon]}
        self.ramp_scattering_length = {'ramp': False, 'g': [self.g]}
        self.reference_frame = {'rotating': False, 'omegaR': [self.omega]}
        self.absorber = {'on': False, 'strength': 1, 'radius': self.fov/2}
        self.edge = {'on': False, 'strength': 5, 'radius': self.fov/2, 'width':self.fov/20}
        self.spoon = {
            'type': None, 'strength':1e5, 'radius': 20e-6,
            'leapx': -250, 'leapy': 500, 'leapxscale': 1, 'leapyscale': 1, 'zcontrol': False
        }

        print(f'''
            Running condensate on {DIM}x{DIM} grid with:
            atom number: {N:.0e}
            mass:        {mass:.2e}
        ''')

    def show_potential(self, frame='auto'):
        '''
        Show a contour map of the potential
        '''
        DIM = self.DIM
        fov = self.fov
        plotV = self.V * 1.
        if not self.use_custom_V:
            # Harmonic potential
            for i in range(DIM):
                for j in range(DIM):
                    x = (i-DIM//2)*fov / DIM
                    y = (j-DIM//2)*fov / DIM
                    rsq = x**2 + y**2
                    esq = x**2 - y**2
                    centrif = 0.5 * self.mass * (self.omega**2) * rsq
                    saddle = 0.5 * self.mass * self.epsilon * (self.omega**2) * esq
                    plotV[i,j] = ( centrif - saddle)/hbar

        if (frame=='auto' and self.reference_frame['rotating']) or (frame=='rotating'):
            # Rotating frame
            omega = self.reference_frame['omegaR'][-1]
            for i in range(DIM):
                for j in range(DIM):
                    x = (i-DIM//2)*fov / DIM
                    y = (j-DIM//2)*fov / DIM
                    rsq = x**2 + y**2
                    centrif = 0.5 * self.mass * (omega**2) * rsq
                    plotV[i,j] -= centrif/hbar
        a = plt.contour(plotV)
        plt.gca().set_aspect('equal', 'box')
        plt.show()

    def harmonic_potential(self, omega, epsilon=0):
        if type(omega) == list and len(omega) != 0 and type(epsilon) == list and len(epsilon) != 0:
            if len(omega) != len(epsilon):
                raise ValueError('Unmatching the ramp sequence of omega and epsilon')
            self.omega = omega[0]
            self.epsilon = epsilon[0]
            self.lb = np.sqrt(hbar / (2* self.mass *omega[0]))
            self.omegaz(np.sqrt(8) * omega[0])
            self.ramp_trap = {'ramp': True, 'omega': omega, 'epsilon': epsilon}
        elif ((type(omega) == float) or (type(omega) == int)) and ((type(epsilon) == float) or (type(epsilon) == int)):
            self.omega = omega
            self.lb = np.sqrt(hbar / (2* self.mass *omega))
            self.epsilon = epsilon
            self.omegaz(np.sqrt(8) * omega)
            self.ramp_trap = {'ramp': False, 'omega': [self.omega], 'epsilon': [self.epsilon]}
        else:
            raise ValueError('omega and epsilon messed up')

    def Feshbach(self, a_s=100*a0):
        if type(a_s) == list and len(a_s) != 0:
            self.a_s = a_s[0]
            self.g = self.N* 4 * np.pi * (hbar**2) * (a_s[0]  / self.mass)
            self.g *= np.sqrt(self.mass * self._omegaz / (2*np.pi*hbar))
            g_array = self.N* 4 * np.pi * (hbar**2) * (np.array(a_s)  / self.mass)
            g_array *= np.sqrt(self.mass * self._omegaz / (2*np.pi*hbar))
            self.ramp_scattering_length = {'ramp': True, 'g': g_array.tolist()}
        elif type(a_s) == int or type(a_s) == float:
            self.a_s = a_s
            self.g = self.N* 4 * np.pi * (hbar**2) * (a_s  / self.mass)
            self.g *= np.sqrt(self.mass * self._omegaz / (2*np.pi*hbar))
            self.ramp_scattering_length = {'ramp': False, 'g': [self.g]}
        else:
            raise ValueError('Feshbach receives number or list as s-wave scattering length!')

    def custom_potential(self, V):
        self.V = V
        self.use_custom_V = True

    def omegaz(self, omegaz):
        self._omegaz = omegaz
        self.g = self.N* 4 * np.pi * (hbar**2) * (self.a_s  / self.mass)
        self.g *= np.sqrt(self.mass * omegaz / (2*np.pi*hbar))
        return self._omegaz

    def rotating_frame(self, omegaR):
        if type(omegaR)==float: omegaR = [omegaR]
        elif type(omegaR)==int: omegaR = [float(omegaR)]
        elif type(omegaR)==list and len(omegaR)==0:
            raise ValueError('omegaR needs to be a float or a list of length steps')
        self.reference_frame = {'rotating': True, 'omegaR': omegaR}

    def lab_frame(self):
        self.reference_frame['rotating'] = False

    def absorbing_boundaries(self, strength, radius):
        self.absorber = {'on': True, 'strength': strength, 'radius': radius}

    def no_absorbing_boundaries(self):
        self.absorber['on'] = False

    def edge(self, strength, radius, width):
        self.edge = {'on': False, 'strength': strength, 'radius': radius, 'width': width}

    def no_edge(self):
        self.edge['on'] = False

max_stars_repo_name: biswaroopmukherjee/condensate

max_issues_repo_name: biswaroopmukherjee/condensate

max_forks_repo_name: biswaroopmukherjee/condensate


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Script to run analysis on lots of files and show statistics and so on about it.
This tries to add all possible analysis about the file and is a very
developmental and possible messy script.

Syntax:

    wikibot-filemeta-log [-arguments ...]

Arguments can be:

-showinfo      Show either "cats", "info" or if left empty defaults to "all".

-limitsize     The maximum file size to download and parse in mega bytes.
               Default is 100MB.
               Example: "-limitsize:10" only analyzes files less than 10MB.

-cachefiles    If used, the files are cached into a folder called "cache" in
               current directory. The directory can be given by using
               "-cachefiles:/place/to/cache". It should already exist.

-dry           If used, the output is printed on the terminal and not to the
               User page.

-skip          If used, all the images in the pagegenerator is not processed.
               After every image it processes, it skips the number of images
               provided as the argument.
               Example: "-skip:1" can be used to only run on alternate images.
"""

from __future__ import (division, absolute_import, unicode_literals,
                        print_function)

import logging
import os
import re
import sys
import tempfile
from collections import Counter
from datetime import datetime

import numpy
from six import string_types
from six.moves.urllib.error import URLError

from file_metadata.generic_file import GenericFile
from file_metadata.wikibot.utilities import (pywikibot, download_page,
                                             stringify)

from pywikibot import pagegenerators

def make_link(val):
    if val.lower().startswith('category:'):
        return '[[:' + val + ']]'
    elif val.lower().startswith('{{'):
        template = re.sub('(^{{|}}$)', '', val)
        tlx = re.sub('=', '=', template)
        return '{{tlx|' + tlx + '}}'
    else:
        return '[[:' + val + ']]'

def dump_log(data, logname, append=False):
    if isinstance(data, (tuple, list)):
        data = "\n".join(data)
    if not isinstance(data, string_types):
        raise ValueError('Unexpected output got. Expected str, tuple, '
                         'list but got {0}.'.format(type(data)))
    page = pywikibot.Page(
        pywikibot.Site(),
        'User:{user}/logs/{name}'.format(user=pywikibot.Site().username(),
                                         name=logname))
    if append:
        data = page.text + '\n' + data
    if options.get('dry'):
        pywikibot.output(data)
    else:
        page.put(data, "Logged using file-metadata")

def handle_bulk_pages(gen):
    parsed_pages = set()
    log, categories, category_buckets = [], [], []
    count, exception_count = 0, 0
    total_start_time = datetime.now()

    mime_cats = {
        "JPEG files": {'image/jpeg'},
        "GIF files": {'image/gif'},
        "PNG files": {'image/png'},
        "TIFF files": {'image/tiff'},
        "XCF files": {'image/x-xcf', 'application/x-xcf'},
        "FLAC files": {'audio/x-flac'},
        "WAV files": {'audio/x-wav'},
        "MIDI files": {'audio/midi'},
        "DjVu files": {'image/vnd-djvu', 'image/vnd.djvu'},
        "PDF files": {'application/pdf'},
        "SVG files": {'svg'},
        "Ogg sound files": {'ogg'},
        "Ogv videos": {'ogv'},
        'WebM videos': {'video/webm'},
    }
    for ipage, page in enumerate(gen):
        if not (page.exists() and
                not page.isRedirectPage() and
                page.namespace() == "File" and
                page.title() not in parsed_pages and
                (page.latest_file_info['size'] / 1024 / 1024 <
                 options.get('limitsize', float("inf"))) and
                ipage % (options.get('skip', 0) + 1) == 0):
            continue
        parsed_pages.add(page.title())
        try:
            page_path = download_page(
                page, timeout=15 * 60,
                cache_dir=options.get('cachefiles', tempfile.gettempdir()))
        except URLError:  # Download timed out
            continue

        pywikibot.output(str(count + 1) + '. Analyzing ' +
                         page.title(underscore=False))

        start_time = datetime.now()
        _file = GenericFile.create(page_path)
        try:
            meta = _file.analyze()
        except Exception as err:
            logging.exception(err)
            exception_count += 1
            continue
        finally:
            _file.close()

        info, cats, cat_buckets, img = [], set(), set(), []
        # info - Information analyzed from the file
        # cats - The suggested categories to add to the file
        # im - Image preview with bounding boxes (if applicable)

        #################################################################
        # Mime analysis
        mime = meta.get('File:MIMEType', "ERROR")
        info.append("* '''Mime Type''': " + mime)
        if mime == 'application/ogg':
            _type = meta.get('File:FileType', 'ERROR')
            info.append("* '''File type''': " + _type)

        for cat, mimeset in mime_cats.items():
            if mime in mimeset or meta.get('Composite:FileFormat') in mimeset:
                cats.add('Category:' + cat)
                cat_buckets.add('Type')
                break

        #################################################################
        # Software analysis
        # Find more files at https://commons.wikimedia.org/wiki/
        # Category:Created_with_..._templates
        if (meta.get('SVG:Output_extension', '') ==
                'org.inkscape.output.svg.inkscape'):
            # Example: File:Db-omega.svg , Joetsu_Shinkansen_icon.png
            cats.add('{{Created with Inkscape}}')

        for sw_key in ('PNG:Software', 'EXIF:Software'):
            sw = stringify(meta.get(sw_key, ''))
            if re.match('MATLAB', sw, re.I):
                # Example: File:Fat_absoprtion.png
                cats.add('{{Created with MATLAB}}')
            elif re.match('ImageMagick', sw, re.I):
                # Example: File:Groz-01.PNG
                match = re.match(r'ImageMagick (?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with ImageMagick' + ver + '}}')
            elif re.match('Adobe ImageReady', sw, re.I):
                # Example: File:Holtz.png
                cats.add('{{Created with Adobe ImageReady}}')
            elif re.match('Adobe Photoshop', sw, re.I):
                if re.match('Elements', sw, re.I):
                    # Example: File:1010_Bazylika_archikatedralna_%C5%9
                    #               Bw_Jakuba_Szczecin_sygnaturka_0.jpg
                    cats.add('{{Created with Adobe Photoshop Elements}}')
                elif re.match('Express', sw, re.I):
                    # Example: File:Politecnico_di_Milano_Bovisa_4.jpg
                    cats.add('{{Created with Adobe Photoshop Express}}')
                elif re.match(r'Photoshop CS\d?', sw, re.I):
                    match = re.match(r'Photoshop CS\d?', sw, re.I)
                    if match:
                        # Example: File:Cervicomanubriotomie.jpg
                        cats.add('{{Created with Adobe ' +
                                 match.group().strip() + '}}')
                else:
                    # Example: File:Cervicomanubriotomie.jpg
                    cats.add('{{Created with Adobe Photoshop}}')
                # Check if photomerge was used
                if (stringify(meta.get('Photoshop:HasRealMergedData',
                                       '')).lower()
                        in ('1', 'yes')):
                    # Example:01-118_Koenigstein_Panorama.jpg
                    cats.add('{{Created with Photoshop Photomerge}}')
                    cats.add('Category:Panoramics')
            elif re.match('Picasa', sw, re.I):
                # Example: File:08_Ny_Alesund_prn.JPG
                match = re.match(r'Picasa (?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with Picasa' + ver + '}}')
            elif re.match('GIMP', sw, re.I):
                # Example: File:2013-04-25_21-09-18-ecl-lune-mosaic.jpg
                match = re.match(r'GIMP (?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with GIMP' + ver + '}}')
            elif re.match('Microsoft ICE', sw, re.I):
                # Example: File:Bochnia_kopalnia_kaplica_2.jpg
                match = re.match(r'ICE v(?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with Microsoft Image Composite '
                         'Editor' + ver + '}}')
            elif re.match(r'Paint\.NET', sw, re.I):
                # Example: File:
                match = re.match(r'Paint\.NET v(?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with Paint.NET' + ver + '}}')
            elif re.match('gnome-screenshot', sw, re.I):
                # Example: File:LibreOfficePresentationTeluguExample1.png
                cats.add('Category:Screenshots')
                cat_buckets.add('Content')

        for desc_key in ('SVG:Desc',):
            desc = stringify(meta.get(desc_key, '')).lower()
            if re.match('GNUPLOT', desc, re.I):
                # Example: File:Beta_versus_rapidity.svg
                match = re.match(r'GNUPLOT (?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with GNU Plot' + ver + '}}')
            elif re.match('Chemtool', desc, re.I):
                # Example: File:Chitobiose_glucosamine.svg
                match = re.match(r'Chemtool (?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with Chemtool' + ver + '}}')
                cats.add('Category:Chemical compounds')
                cats.add('Category:Graphics')
                cat_buckets.add('Content')
            elif re.match('VectorFieldPlot', desc, re.I):
                # Example: File:VFPt_minus.svg
                match = re.match(r'VectorFieldPlot (?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with VectorFieldPlot' + ver + '}}')
                cats.add('Category:Field diagrams')
                cats.add('Category:Graphics')
                cat_buckets.add('Content')

        for comment_key in ('PNG:Comment', 'File:Comment'):
            comment = stringify(meta.get(comment_key, '')).lower()
            if re.match('Stella4D', comment, re.I):
                # Example: File:10-3_deltohedron.png
                match = re.match(r'Stella4D (?P<ver>[\d\.]+)', sw, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                cats.add('{{Created with Stella' + ver + '}}')
            elif re.match('GIMP', comment, re.I):
                # Example: File:105_H_61-37.jpeg
                cats.add('{{Created with GIMP}}')

        for prod_key in ('PDF:Producer',):
            prod = stringify(meta.get(prod_key, ''))
            if re.match(r'Libre ?Office', prod, re.I):
                match = re.match(r'Libre ?Office (?P<ver>[\d\.]+)', prod, re.I)
                ver = '|version=' + match.groupdict()['ver'] if match else ''
                if meta.get('PDF:Creator'):
                    cats.add('{{Created with LibreOffice ' +
                             meta['PDF:Creator'] + ver + '}}')
                else:
                    cats.add('{{Created with LibreOffice' + ver + '}}')
            elif re.match(r'doPDF', prod, re.I):
                cats.add('{{Created with doPDF}}')
            elif re.match(r'ACDSee', prod, re.I):
                cats.add('{{Created with ACDSee}}')
            elif re.match(r'iText', prod, re.I):
                cats.add('{{Created with iText}}')

        #################################################################
        # Device analysis
        model = stringify(meta.get('EXIF:Model', ''))
        make = stringify(meta.get('EXIF:Make', ''))
        if model:
            # Modify the model so that it conforms to the category names
            for _make in ('NIKON', 'FUJIFILM', 'Xiaomi', 'Panasonic', 'SONY',
                          'Nokia'):
                if make == _make and not re.match(_make, model, re.I):
                    # Category is named as make + model
                    model = make + ' ' + model
            if make == 'i2S DigiBook Scanner':
                model = 'i2s Digibook ' + model
            # Simple replacements
            model = model.replace('NIKON', 'Nikon')
            model = model.replace('PENTAX', 'Pentax')
            model = model.replace('FUJIFILM', 'Fujifilm')
            model = model.replace('SONY', 'Sony')
            if 'nikon' in make.lower():
                model = model.replace('COOLPIX', 'Coolpix')
            if 'xiaomi' in make.lower():
                model = model.replace('Note3', 'Note 3')
            if 'samsung' in make.lower():
                # In samsung, the cameras have multiple names like:
                # "<Digimax S600 / Kenox S600 / Digimax Cyber 630>" or
                # "Digimax A4/Kenox D4". The categories are only named as
                # "Taken with Samsung S600", "Taken with Samsung A4" though.
                samsung_model = ''
                possible_models = re.sub('[<>]', '', model).split('/')
                for imod in possible_models:
                    imod = imod.strip()
                    if (imod.lower().startswith('digimax') and
                            not imod.lower().startswith('digimax cyber')):
                        samsung_model = imod
                        break
                if not samsung_model and len(possible_models) > 0:
                    samsung_model = possible_models[0].strip()
                # Remove digimax if present
                samsung_model = re.sub('Digimax', '', samsung_model, count=1,
                                       flags=re.IGNORECASE).strip()

                model = 'Samsung ' + samsung_model
            if 'olympus' in make.lower():
                model = 'Olympus ' + model
            if 'motorola' in make.lower():
                model = model.replace('MotoG2', 'Moto G2')
                model = model.replace('MotoG3', 'Moto G3')
                model = model.replace('MotoG4', 'Moto G4')
            if 'sanyo' in make.lower():
                model = 'Sanyo ' + model
            if 'canon' in make.lower():
                model = model.replace('EOS REBEL', 'EOS')
            if 'casio' in make.lower():
                model = 'Casio ' + model

            for prefix in ['Sanned with ', 'Taken with ',
                           'Taken or Scanned with ']:
                possible_cat = pywikibot.Page(pywikibot.Site(),
                                              "Category:" + prefix + model)
                if (possible_cat.exists() or
                        prefix == 'Taken or Scanned with '):
                    info.append("* '''Model''': " +
                                possible_cat.title(underscore=False,
                                                   textlink=True, asLink=True))
                    cats.add(possible_cat.title(underscore=False))
                    break

        #################################################################
        # Color analysis
        for key in ['AverageRGB', 'ClosestLabeledColor', 'EdgeRatio',
                    'NumberOfGreyShades', 'PercentFrequentColors',
                    'MeanSquareErrorFromGrey', 'Monochrome', 'UsesAlpha',
                    'IT8TopBar', 'IT8BottomBar', 'IT8TopBarGreyMSE',
                    'IT8BottomBarGreyMSE']:
            if 'Color:' + key in meta:
                if isinstance(meta['Color:' + key], (list, tuple)):
                    val = ', '.join(map(str, meta['Color:' + key]))
                elif isinstance(meta['Color:' + key], float):
                    val = str(round(meta['Color:' + key], 5))
                else:
                    val = stringify(meta['Color:' + key])
                info.append("* '''" + key + "''': " + val)
        greys = meta.get('Color:NumberOfGreyShades')
        edges = meta.get('Color:EdgeRatio')
        if ((greys is not None and greys < 2) or
                (edges is not None and edges < 0.13) or
                meta.get('Composite:FileFormat') == 'svg'):
            if meta.get('EXIF:Make') is None:
                cats.add('Category:Graphics')
        if meta.get('Color:MeanSquareErrorFromGrey', 999) < 20:
            if meta.get('EXIF:Make') is None:
                cats.add('Category:Black and white')
            else:  # If it's scanned or from a camera and black-white
                cats.add('Category:Black and white photographs')
        if meta.get('Color:MeanSquareErrorFromGrey', 999) == 0:
            cats.add('Category:Grayscale')
        if meta.get('Color:UsesAlpha') is True:
            cats.add('Category:Transparent background')

        for spike, greymse in [('Color:IT8TopBar',
                                'Color:IT8TopBarGreyMSE'),
                               ('Color:IT8BottomBar',
                                'Color:IT8BottomBarGreyMSE')]:
            if (18 <= meta.get(spike, -1) <= 22 and
                    meta.get(greymse, 999) < 15):
                cats.add('Category:Scans with IT8 target')
                break

        if meta.get('Misc:StereoCardMSE'):
            info.append("* '''Stereo Card MSE''': " +
                        str(meta.get('Misc:StereoCardMSE')))
        if meta.get('Misc:StereoCardHistogramMSE'):
            info.append("* '''Stereo Card Histogram MSE''': " +
                        str(meta.get('Misc:StereoCardHistogramMSE')))

        #################################################################
        # Location analysis
        for key in ['GPSCity', 'GPSState', 'GPSCountry']:
            val = meta.get('Composite:' + key)
            if val is not None:
                info.append("* '''" + key + "''': " + val)
        for val in [(meta.get('Composite:GPSCity', '') + ', ' +
                     meta.get('Composite:GPSState', '')),
                    (meta.get('Composite:GPSCity', '') + ', ' +
                     meta.get('Composite:GPSCountry', '')),
                    meta.get('Composite:GPSCity', ''),
                    (meta.get('Composite:GPSState', '') + ', ' +
                     meta.get('Composite:GPSCountry', '')),
                    meta.get('Composite:GPSState', ''),
                    meta.get('Composite:GPSCountry', '')]:
            if val:
                if pywikibot.Page(
                        pywikibot.Site(), "Category:" + val).exists():
                    cats.add('Category:' + val)
                    break

        if (meta.get('Composite:GPSLatitude') and
                meta.get('Composite:GPSLongitude')):
            cats.add('{{GPS EXIF}}')
            cat_buckets.add('Location')
            info.append("* '''GPS Coordinates''': {0}, {1}"
                        .format(meta.get('Composite:GPSLatitude'),
                                meta.get('Composite:GPSLongitude')))

        #################################################################
        # Author analysis
        if meta.get('EXIF:Artist') == "New York Public Library":
            cats.add('Category:Images from the New York Public Library')

        #################################################################
        # Image analysis
        # Fill second column (image cell) for ImageFiles (Only 2 dim images)
        if (isinstance(_file.fetch('ndarray'), numpy.ndarray) and
                _file.fetch('ndarray').ndim in (2, 3)):
            height, width = _file.fetch('ndarray').shape[:2]
            max_dim = max(width, height)
            scale = min(200, max_dim) / max_dim

            img.append('<div style="position:relative;">')
            img.append('{0}|{1}x{2}px]]'.format(
                       page.title(underscore=False, asLink=True)[:-2],
                       int(width * scale), int(height * scale)))

            img_bbox = ('<div class="position-marker file-meta-{css_class}" '
                        'style="position:absolute; left:{left}px; '
                        'top:{top}px; width:{width}px; height:{height}px; '
                        'border:2px solid #{color};"></div>')

            def _str_bbox(bbox):
                return ("Left:" + str(bbox['left']) +
                        ", Top:" + str(bbox['top']) +
                        ", Width:" + str(bbox['width']) +
                        ", Height:" + str(bbox['height']))

            def _mean_bbox(bbox):
                return (bbox['left'] + bbox['width'] / 2,
                        bbox['top'] + bbox['height'] / 2)

            def _area_bbox(bbox):
                return bbox['width'] * bbox['height']

            #################################################################
            # Analysis for very specific images: Icons, Football kits, etc
            if height == width and height in (16, 32, 48, 96):
                cats.add('Category:Icons')
            elif width == 100 and height == 36:
                cats.add('Category:Football kit shorts')
                cat_buckets.add('Content')
            elif width == 100 and height == 25:
                cats.add('Category:Football kit socks')
                cat_buckets.add('Content')
            elif width == 38 and height == 59:
                cats.add('Category:Football kit body')
                cat_buckets.add('Content')
            elif width == 31 and height == 59:
                cats.add('Category:Football kit sleeves')
                cat_buckets.add('Content')

            #################################################################
            # Barcode analysis
            bar_cats = {
                "Code 39": ('code_39', 'code39'),
                "Code 93": ('code_93', 'code93'),
                "Code 128": ('code_128', 'code128'),
                "Data Matrix": ('data_matrix',),
                "Quick Response Codes": ('qr_code', 'qrcode')}

            def print_barcode_data(bar):
                info.append("** Data: " + bar['data'])
                info.append("** Format: " + str(bar['format']))
                info.append("** Position : " + _str_bbox(bar['bounding box']))
                box_kwargs = {k: int(v * scale)
                              for k, v in bar['bounding box'].items()}
                box_kwargs["css_class"] = "barcode"
                if (bar['bounding box']['height'] < 5 and
                        bar['bounding box']['width'] < 5):
                    box_kwargs["color"] = "ff0000"
                    img.append(img_bbox.format(**box_kwargs))
                    return set()
                box_kwargs["color"] = "00ff00"
                img.append(img_bbox.format(**box_kwargs))
                for cat, formats in bar_cats.items():
                    if bar['format'].lower() in formats:
                        return {'Category:' + cat, 'Category:Barcode'}
                return {'Category:Barcode'}

            # Barcodes from zxing:
            for i, bar in enumerate(meta.get('zxing:Barcodes', [])):
                info.append("* '''Barcode''' (zxing) #" + str(i))
                barcode_cats = print_barcode_data(bar)
                cats = cats.union(barcode_cats)
                if len(barcode_cats) > 0:
                    cat_buckets.add('Content')

            # Barcodes from zbar:
            for i, bar in enumerate(meta.get('zbar:Barcodes', [])):
                info.append("* '''Barcode''' (zbar) #" + str(i))
                barcode_cats = print_barcode_data(bar)
                cats = cats.union(barcode_cats)
                if len(barcode_cats) > 0:
                    cat_buckets.add('Content')

            #################################################################
            # Face analysis
            def print_face_data(face, _type):
                _cats = set()  # These may or may not be added ...
                feats = set()
                _cats.add('Category:Human faces')
                _cats.add('Category:Unidentified people')
                if _type == 'opencv':  # Dlib always finds all features
                    if len(face.get('eyes', ())) > 0:
                        # _cats.add('Category:Human eyes')
                        feats.add('Eyes (' + str(len(face['eyes'])) + ')')
                    if face.get('ears') is not None:
                        # _cats.add('Category:Human ears')
                        feats.add('Ears')
                    if face.get('nose') is not None:
                        # _cats.add('Category:Human noses')
                        feats.add('Nose')
                    if face.get('mouth') is not None:
                        # _cats.add('Category:Human mouths')
                        feats.add('Mouth')
                    if face.get('glasses') is not None:
                        feats.add('Glasses')

                if (_area_bbox(face['position']) > 0.55 * height * width):
                    # If face is very large...
                    _cats.add('Category:Portrait')

                if _type == 'dlib':
                    info.append("** Score: " + str(round(face['score'], 3)))
                info.append("** Bounding Box: " + _str_bbox(face['position']))
                if len(feats) > 0:
                    info.append("** Other features: " + ", ".join(feats))
                box_kwargs = {k: int(v * scale)
                              for k, v in face['position'].items()}
                box_kwargs["css_class"] = "barcode"
                if (((_type == 'opencv' and len(feats) > 2) or
                        (_type == 'dlib' and face['score'] > 0.045)) and
                        _area_bbox(face['position']) > width * height * 0):
                    box_kwargs["color"] = "00ff00"
                    img.append(img_bbox.format(**box_kwargs))
                    return _cats
                box_kwargs["color"] = "ff0000"
                img.append(img_bbox.format(**box_kwargs))
                return set()

            # Save all face categories in this set, ad add it later if it
            # seems appropirate. This is because we need to delay the writing
            # of these categories based on some logic.
            valid_faces = []
            face_cats = set()

            # Faces with dlib:
            for iface, face in enumerate(meta.get('dlib:Faces', [])):
                info.append("* '''Face''' (dlib) #" + str(iface + 1))
                icats = print_face_data(face, 'dlib')
                face_cats = face_cats.union(icats)
                if 'Category:Human faces' in icats:
                    valid_faces.append(face)

            # Faces with opencv's haarcascades:
            for iface, face in enumerate(meta.get('OpenCV:Faces', [])):
                info.append("* '''Face''' (haarcascade) #" + str(iface + 1))
                icats = print_face_data(face, 'opencv')
                face_cats = face_cats.union(icats)
                if 'Category:Human faces' in icats:
                    valid_faces.append(face)

            if len(valid_faces) >= 3:
                face_cats.add('Category:Groups of people')

            if len(face_cats) > 0:
                cat_buckets.add('Content')
            cats = cats.union(face_cats)
            img.append('</div>')  # Close the image's div

        #################################################################
        # Leaf cats using Location analysis
        if 'Category:Unidentified people' in cats:
            for key in ['GPSState', 'GPSCountry']:
                val = meta.get('Composite:' + key)
                if val is None:
                    continue
                cat_val = "Category:Unidentified people in " + val
                if (pywikibot.Page(pywikibot.Site(), cat_val).exists() or
                        key == 'GPSCountry'):
                    cats.add(cat_val)
                    cats.remove('Category:Unidentified people')
                    break

        if 'Category:Groups of people' in cats:
            for key in ['GPSState', 'GPSCountry']:
                val = meta.get('Composite:' + key)
                if val is None:
                    continue
                cat_val = "Category:Groups of people in " + val
                if (pywikibot.Page(pywikibot.Site(), cat_val).exists() or
                        key == 'GPSCountry'):
                    cats.add(cat_val)
                    cats.remove('Category:Groups of people')
                    break

        categories.append(cats)
        category_buckets.append(cat_buckets)

        info.append("* '''Time taken''': {0} sec"
                    .format((datetime.now() - start_time).total_seconds()))

        count += 1
        log.append("\n==== {0} ====" .format(page.title(asLink=True,
                                                        textlink=True)))
        # Make a table for text and image

        log += ['{| class="wikitable"', '|']
        if options.get('showinfo', "all") in ('all', 'info'):
            log += info
        if options.get('showinfo', "all") in ('all', 'cats'):
            log.append("* '''Categories''' ({0}): {1}"
                       .format(len(cats),
                               ", ".join(make_link(c) for c in cats)))
        log += ['|'] + img + ['|}']
        # Clean up the downloaded file if no need to cache
        if options.get('cachefiles') is None:
            os.remove(page_path)

    if count + exception_count == 0:
        # Nothing happened, don't go on.
        pywikibot.output('Analysis was not run on any files.')
        return

    stats = ["* '''Time taken''': {0} sec"
             .format((datetime.now() - total_start_time).total_seconds())]
    stats += ["* '''Number of files successfully analyzed''': "
              "{0} ({1:0.4f} %)".format(
                  count, 100 * count / (count + exception_count))]
    stats += ["* '''Number of files with exceptions''': "
              "{0} ({1:0.4f} %)".format(
                  exception_count,
                  100 * exception_count / (count + exception_count))]
    cat_counter = Counter(cat for fcats in categories for cat in fcats)
    stats += ["* '''Number of distinct categories used''': {0}"
              .format(len(cat_counter))]

    stats += ['{| class="wikitable sortable plainrowheaders"',
              '|+ Data of files categorized by category name',
              '! Category name',
              '! Number of files',
              '! colspan=2 | Percent of files in the category',
              ]
    for icat, (catname, numfiles) in enumerate(
            sorted(cat_counter.items(), key=lambda x: x[1], reverse=True)):
        stats.append('|-')
        stats.append('! scope=row | ' + make_link(catname))
        stats.append('| ' + str(numfiles))
        stats.append('| {{bartable| ' +
                     str(round(100 * numfiles / count, 2)) + '}}')
    stats.append('|}')

    stats += ['{{User:AbdealiJK/Templates/Bar chart',
              '| title = Histogram of number of categories found per file',
              '| label_type = Number of categories found',
              '| data_type = Number of files with that many categories',
              '| data_max = ' + str(count)]
    for icat, (numcats, numfiles) in \
            enumerate(Counter(len(cats) for cats in categories).items()):
        stats += ['| label{0} = {1} ({2:.4f} %)'.format(
                  icat + 1, numcats, 100 * numfiles / count),
                  '| data{0} = {1}'.format(icat + 1, numfiles)]
    stats.append('}}')

    stats += ['{{User:AbdealiJK/Templates/Bar chart',
              '| title = Histogram of number of categories found per file '
              '(without file-type categories)',
              '| label_type = Number of categories found',
              '| data_type = Number of files with that many categories',
              '| data_max = ' + str(count)]
    file_type_cats = set('Category:' + c for c in mime_cats.keys())
    non_file_type_counter = Counter(
        len(cats - file_type_cats) for cats in categories)
    for icat, (numcats, numfiles) in enumerate(non_file_type_counter.items()):
        stats += ['| label{0} = {1} ({2:.4f} %)'.format(
                  icat + 1, numcats, 100 * numfiles / count),
                  '| data{0} = {1}'.format(icat + 1, numfiles)]
    stats.append('}}')

    stats += ['{{User:AbdealiJK/Templates/Bar chart',
              '| title = Histogram of buckets',
              '| label_type = Bucket name',
              '| data_type = Number of files in bucket',
              '| data_max = ' + str(count)]
    bucket_counter = Counter(
        b for fbuckets in category_buckets for b in fbuckets)
    for ibucket, (bucketname, numfiles) in enumerate(
            sorted(bucket_counter.items(), key=lambda x: x[1], reverse=True)):
        stats += ['| label{0} = {1} ({2:.4f} %)'.format(
                  ibucket + 1, bucketname, 100 * numfiles / count),
                  '| data{0} = {1}'.format(ibucket + 1, numfiles)]
    stats.append('}}')

    log = stats + ['\n'] + log

    # Dump all logs to the log page
    dump_log(log, logname=options.get('logname'))

options = {}

def main(*args):
    local_args = pywikibot.handle_args(args)
    gen_factory = pagegenerators.GeneratorFactory()

    for local_arg in local_args:
        if gen_factory.handleArg(local_arg):
            continue
        arg, sep, value = local_arg.partition(':')
        if arg == '-showinfo':
            options[arg[1:]] = value or "all"
            if value not in ("cats", "info", "all"):
                pywikibot.error("Invalid value for -showinfo. It can only be "
                                "cats, info, or all.\nUse -help for further "
                                "information.")
                sys.exit(1)
        elif arg == '-limitsize':
            options[arg[1:]] = int(value) if value != "" else 100
        elif arg == '-cachefiles':
            options[arg[1:]] = value or 'cache'
        elif arg == '-logname':
            options[arg[1:]] = value
        elif arg == '-dry':
            options[arg[1:]] = True
        elif arg == '-skip':
            options[arg[1:]] = int(value) or 0

    if not options.get('dry') and not options.get('logname'):
        pywikibot.error('-logname is required to decide the page to write '
                        'to.\nUse -help for further information.')
        sys.exit(2)

    gen = gen_factory.getCombinedGenerator()
    if not gen:
        pywikibot.bot.suggest_help(missing_generator=True)
        return False
    else:
        pywikibot.Site().login()
        pregenerator = pagegenerators.PreloadingGenerator(gen)
        handle_bulk_pages(pregenerator)
        return True

if __name__ == "__main__":
    main()

max_stars_repo_name: pywikibot-catfiles/file-metadata

max_issues_repo_name: AbdealiJK/file-metadata

max_forks_repo_name: pywikibot-catfiles/file-metadata


text:
import os
import math
import numpy as np
import pandas as pd
import time
from tqdm.notebook import tqdm

## Imports for plotting
import matplotlib.pyplot as plt
from IPython.display import set_matplotlib_formats
set_matplotlib_formats('svg', 'pdf')
from matplotlib.colors import to_rgba
import seaborn as sns
sns.set()

# Pytoch
import torch
import torch.nn.functional as F
from torch import nn
import torch.utils.data as data
print("Using torch", torch.__version__)
torch.manual_seed(42)

class MockDataset(data.Dataset):

    def __init__(self,
                 features=2,
                 pos_n=100,
                 neg_n=100,
                 pos_mean=100,
                 pos_std=25,
                 neg_mean=150,
                 neg_std=12):
        """
        Inputs:
            features - Number of features we want to generate
            pos_n - Numer of samples for positive class
            neg_n - Numer of samples for negative class
            pos_mean - mean value for positive class features
            neg_mean - mean value for negative class features
            pos_std - standard deviation for positive class features
            neg_std - standard deviation for negative class features
        """
        super().__init__()
        self.features = features
        self.pos_n = pos_n
        self.pos_mean = pos_mean
        self.pos_std = pos_std
        self.neg_n = neg_n
        self.neg_mean = neg_mean
        self.neg_std = neg_std
        self.size = self.pos_n + self.neg_n
        self.generate_data()

    def generate_col(self, mean, std, n):
        data_cols = np.empty(n).T
        for col in range(1,self.features):
            temp = np.random.normal(mean, std, n)
            data_cols = np.vstack((data_cols,temp))
        return data_cols.T

    def generate_data(self):
        pos_data = self.generate_col(self.pos_mean, self.pos_std, self.pos_n)
        neg_data = self.generate_col(self.neg_mean, self.neg_std, self.neg_n)
        data = np.vstack((pos_data,neg_data))
        self.data = torch.Tensor(data).to(torch.float64)

        self.pos_label = torch.ones(self.pos_n)
        self.neg_label = torch.zeros(self.neg_n)
        self.label = torch.cat((self.pos_label, self.neg_label), dim=0).to(torch.float64)

    def __len__(self):
        return self.size

    def __getitem__(self, idx):
        data_point = self.data[idx]
        data_label = self.label[idx]
        return data_point, data_label

    def to_df(self, row_ids=False, label=False):
        df = pd.DataFrame(self.data.numpy())
        df.columns = [f"col_{c}" for c in df.columns]
        if row_ids:
            df["row_ids"] = df.index.values.T
        if label:
            df["label"] = self.label
        return df

    @staticmethod
    def sort_to_sequence(df, key_col, exclude_cols):
        df = df.copy()
        cols_space = (df.shape[1]-len(exclude_cols))
        np_seq = np.empty((0, cols_space))
        for key in tqdm(df[key_col]):
            df_temp = df[df[key_col]==key].drop(exclude_cols, axis = 1). \
            T.sort_values(by=key).reset_index()
            np_temp = df_temp.T.values[0].reshape(1,-1)
            np_seq = np.concatenate([np_seq,np_temp],axis=0)
        return np_seq

class MyModule(nn.Module):

    def __init__(self, n_input):
        super().__init__()
        self.fc1 = nn.Linear(n_input, 8)
        self.b1 = nn.BatchNorm1d(8)
        self.fc2 = nn.Linear(8, 4)
        self.b2 = nn.BatchNorm1d(4)
        self.fc3 = nn.Linear(4,3)
        self.b3 = nn.BatchNorm1d(3)
        self.fc4 = nn.Linear(3,1)

    def swish(self, x):
        return x * torch.sigmoid(x)

    def forward(self, x):
        x = self.swish(self.fc1(x))
        x = self.b1(x)
        x = self.swish(self.fc2(x))
        x = self.b2(x)
        x = self.swish(self.fc3(x))
        x = self.b3(x)
        x = self.fc4(x)
        return x

def visualize_data(data, label, x1:int=1, x2:int=2):
    if isinstance(data, torch.Tensor):
        data = data.cpu().numpy()
    if isinstance(label, torch.Tensor):
        label = label.cpu().numpy()
    data_0 = data[label == 0]
    data_1 = data[label == 1]
    size = np.min([data_0.shape[0], data_1.shape[0]])

    plt.figure(figsize=(4,4))
    plt.scatter(data_0[:size,x1], data_0[:size,x2], edgecolor=".6", label="Negative Class")
    plt.scatter(data_1[:size,x1], data_1[:size,x2], edgecolor=".2", label="Positive Class")
    plt.title("Dataset samples")
    plt.ylabel(r"$x_2$")
    plt.xlabel(r"$x_1$")
    plt.legend()

def train_model(model, optimizer, data_loader, loss_module, num_epochs=100):
    model.train()

    # Training loop
    for epoch in tqdm(range(num_epochs)):
        for data_inputs, data_labels in data_loader:

            preds = model(data_inputs.float())
            preds = preds.squeeze(dim=1)

            loss = loss_module(preds, data_labels.float())
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()

def eval_model(model, data_loader):
    model.eval()
    true_preds, num_preds = 0., 0.

    with torch.no_grad():
        for data_inputs, data_labels in data_loader:

            preds = model(data_inputs.float())
            preds = preds.squeeze(dim=1)
            preds = torch.sigmoid(preds)
            pred_labels = preds.round()

            true_preds += (pred_labels.float() == data_labels.float()).sum().float()
            num_preds += data_labels.shape[0]

    acc = true_preds / num_preds
    print(f"Accuracy of the model: {100.0*acc:4.2f}%")


text:
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

#Added some imports and deleted some
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from PIL import Image
import numpy as np
import cv2

"""[summary]
This code is modified from the orginal use to work with our validation programs.
To see the orignal code from the tensorflow authors see below:
https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/examples/label_image/label_image.py
Author to edits in the code: Daniel Persson 2019-05-22
"""

#This method has not been modified
def load_graph(model_file):
  graph = tf.Graph()
  graph_def = tf.GraphDef()

  with open(model_file, "rb") as f:
    graph_def.ParseFromString(f.read())
  with graph.as_default():
    tf.import_graph_def(graph_def)

  return graph

#This method has not been modified
def read_tensor_from_image_file(file_name,
                                input_height=299,
                                input_width=299,
                                input_mean=0,
                                input_std=255):
  input_name = "file_reader"
  output_name = "normalized"
  file_reader = tf.read_file(file_name, input_name)
  if file_name.endswith(".png"):
    image_reader = tf.image.decode_png(
        file_reader, channels=3, name="png_reader")
  elif file_name.endswith(".gif"):
    image_reader = tf.squeeze(
        tf.image.decode_gif(file_reader, name="gif_reader"))
  elif file_name.endswith(".bmp"):
    image_reader = tf.image.decode_bmp(file_reader, name="bmp_reader")
  else:
    image_reader = tf.image.decode_jpeg(
        file_reader, channels=3, name="jpeg_reader")
  float_caster = tf.cast(image_reader, tf.float32)
  dims_expander = tf.expand_dims(float_caster, 0)
  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
  sess = tf.Session()
  result = sess.run(normalized)

  return result

#Removed load_labels method and the arguments

#Modified this part to create a method to make it easier to call from any program
def result(file_name,graph):
  """[summary]
  Returns our results from the google models
  Args:
      file_name ([STR]): [The name of the model]
      graph ([PB]): [The retrained model in pb format]
  """

  #Changed to values that are constant
  input_height = 299
  input_width = 299
  input_name = "file_reader"
  output_name = "normalized"
  input_layer = "Placeholder"
  output_layer = "final_result"

  #Modify so it only takes in the input we need for the operation
  t = read_tensor_from_image_file(
      file_name,
      input_height=input_height,
      input_width=input_width)

  #This has not been modified
  input_name = "import/" + input_layer
  output_name = "import/" + output_layer
  input_operation = graph.get_operation_by_name(input_name)
  output_operation = graph.get_operation_by_name(output_name)

  #This has not been modified
  with tf.Session(graph=graph) as sess:
    results = sess.run(output_operation.outputs[0], {
        input_operation.outputs[0]: t
    })
  results = np.squeeze(results)
  #Removed code here that was related to the outputlabels that was not useful for our need
  return results


text:
# Contributor: piergiaj
# Sampling rate modifications by Samuel Kwong

import torch
import torch.utils.data as data_utl
from torch.utils.data.dataloader import default_collate

import numpy as np
import json
import csv
import pickle

import os
import os.path

from PIL import Image

def video_to_tensor(pic):
    """Convert a ``numpy.ndarray`` to tensor.
    Converts a numpy.ndarray (T x H x W x C)
    to a torch.FloatTensor of shape (C x T x H x W)

    Args:
         pic (numpy.ndarray): Video to be converted to tensor.
    Returns:
         Tensor: Converted video.
    """
    return torch.from_numpy(pic.transpose([3,0,1,2]))

def pil_loader(path):
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGB')

def load_rgb_frames(image_dir, vid, start, num, stride, transforms):
    frames = []
    for i in range(start, start+num, stride):
        try:
            img = pil_loader(os.path.join(image_dir, vid, vid+'-'+str(i).zfill(6)+'.jpg'))
            img = transforms(img)
            img = torch.unsqueeze(img, 0)
        except: # duplicate last frame
            img = frames[-1]
        frames.append(img)
    return frames

def load_flow_frames(image_dir, vid, start, num):
    frames = []
    for i in range(start, start+num):
        imgx = cv2.imread(os.path.join(image_dir, vid, vid+'-'+str(i).zfill(6)+'x.jpg'), cv2.IMREAD_GRAYSCALE)
        imgy = cv2.imread(os.path.join(image_dir, vid, vid+'-'+str(i).zfill(6)+'y.jpg'), cv2.IMREAD_GRAYSCALE)

        w,h = imgx.shape
        if w < 224 or h < 224:
            d = 224.-min(w,h)
            sc = 1+d/min(w,h)
            imgx = cv2.resize(imgx,dsize=(0,0),fx=sc,fy=sc)
            imgy = cv2.resize(imgy,dsize=(0,0),fx=sc,fy=sc)

        imgx = (imgx/255.)*2 - 1
        imgy = (imgy/255.)*2 - 1
        img = np.asarray([imgx, imgy]).transpose([1,2,0])
        frames.append(img)
    return np.asarray(frames, dtype=np.float32)

def make_dataset(split_file, split, root, mode='rgb', stride=4, num_span_frames=32, is_sife=False):
    num_actions, num_scenes = 157, 16
    dataset = []
    with open(split_file, 'r') as f1:
        data = json.load(f1)

    if is_sife:
        with open('./data/annotations/charades_train_scene_map.pkl', 'rb') as f2:
            train_scene_map = pickle.load(f2)

        with open('./data/annotations/charades_test_scene_map.pkl', 'rb') as f3:
            test_scene_map = pickle.load(f3)

    for i, vid in enumerate(data.keys()):
        if (i % 500 == 0):
            print('{}/{}'.format(i, len(data.keys())))

        if data[vid]['subset'] != split:
            continue

        if not os.path.exists(os.path.join(root, vid)):
            continue

        # Sample clips with temporal stride of 4, having input clips spanning total of 125 frames (~5.2 seconds)
        # Follows approach from Long-Term Feature Banks for Detailed Video Understanding (https://arxiv.org/pdf/1812.05038.pdf)

        num_avail_frames = len(os.listdir(os.path.join(root, vid)))
        if mode == 'flow':
            num_span_frames=num_span_frames//2

        action_label = np.zeros((num_actions, num_span_frames), np.float32)

        if is_sife:
            scene_label = np.zeros(num_span_frames, np.int64) # scene label goes from [0, num_class-1]
            if split == 'training':
                scene_label.fill(train_scene_map[vid])
            else:
                scene_label.fill(test_scene_map[vid])

        fps = num_avail_frames/data[vid]['duration'] # we use 24fps as provided by Charades website
        for ann in data[vid]['actions']:
            most_recent = None
            for fr in range(0, num_span_frames*stride, stride): # sample every 4 frames
                if fr < num_avail_frames:
                    if fr/fps > ann[1] and fr/fps < ann[2]:
                        action_label[ann[0], int(fr/stride)] = 1 # binary classification
                        most_recent = 1
                    else:
                        most_recent = 0
                else: # repeat last frame
                    action_label[ann[0], int(fr/stride)] = most_recent # binary classification
        if is_sife:
            dataset.append((vid, action_label, scene_label, num_span_frames*stride, stride))
        else:
            dataset.append((vid, action_label, num_span_frames*stride, stride))

    return dataset

class Charades(data_utl.Dataset):

    def __init__(self, split_file, split, root, mode='rgb', transforms=None, stride=4, num_span_frames=32, is_sife=False):

        self.data = make_dataset(split_file, split, root, mode='rgb', stride=4, num_span_frames=32, is_sife=False)
        self.split_file = split_file
        self.transforms = transforms
        self.mode = mode
        self.root = root
        self.is_sife = is_sife

    def __getitem__(self, index):
        """
        Args:
            index (int): Index

        Returns:
            tuple: (image, target) where target is class_index of the target class.
        """
        if self.is_sife:
            vid, action_label, scene_label, nf, stride = self.data[index]
        else:
            vid, action_label, nf, stride = self.data[index]

        if self.mode == 'rgb':
            imgs = load_rgb_frames(self.root, vid, 1, nf, stride, self.transforms)
        else:
            imgs = load_flow_frames(self.root, vid, 1, nf, stride)

        inputs = torch.cat(imgs)
        inputs = inputs.permute(1, 0, 2, 3)

        if self.is_sife:
            return inputs, torch.from_numpy(action_label), torch.from_numpy(scene_label), vid
        else:
            return inputs, torch.from_numpy(action_label), vid

    def __len__(self):
        return len(self.data)

if __name__ == '__main__':
    from torchvision import transforms
    import pickle

    split = 'data/annotations/charades.json'
    root = ''
    mode = 'rgb'
    stride = 4
    num_span_frames = 32
    batch_size = 2
    train_scene_map_pkl = './data/annotations/charades_train_scene_map.pkl'
    test_scene_map_pkl = './data/annotations/charades_test_scene_map.pkl'

    train_transforms = transforms.Compose([transforms.Resize((224,224)),
                                           transforms.ToTensor()
                                          ])
    test_transforms = transforms.Compose([transforms.Resize((224,224)),
                                          transforms.ToTensor()
                                         ])

    print('Getting train dataset...')
    train_path = './data/train_dataset_{}_{}.pickle'.format(stride, num_span_frames)
    if os.path.exists(train_path):
        pickle_in = open(train_path, 'rb')
        train_dataset = pickle.load(pickle_in)
    else:
        train_dataset = Charades(split, train_scene_map_pkl, test_scene_map_pkl, 'training', root, mode, train_transforms, stride, num_span_frames)
        pickle_out = open(train_path, 'wb')
        pickle.dump(train_dataset, pickle_out)
        pickle_out.close()
    print('Got train dataset.')
    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size, shuffle=True, num_workers=0, pin_memory=True)

    for data in train_dataloader:
        inputs, actions, scenes, vid = data
        print('input shape = {}'.format(inputs.shape))
        print('actions shape = {}'.format(actions.shape))
        print('scenes shape = {}'.format(scenes.shape))
        print('scenes = {}'.format(scenes))
        print('vid = {}'.format(vid))


text:
import numpy as np
import copy

from util.util import *
from heuristics.greedy_tsp import greedy as greedy_tsp
from graph.graph import Graph
from util.prim import *

# prefer concorde
try:
    from concorde.tsp import TSPSolver
    CONCORDE_AVAILABLE = True
    #CONCORDE_AVAILABLE = False
except ImportError:
    CONCORDE_AVAILABLE = False

if CONCORDE_AVAILABLE == False:
    try:
        from python_tsp.heuristics import solve_tsp_simulated_annealing as solve_tsp_lib
        SOLVER_AVAILABLE = False
    except ImportError:
        SOLVER_AVAILABLE = False

def generate_random_tour(nvert, rng_seed=None, kind="Euclidean"):
    # set up tour
    gen = np.random.default_rng(seed=rng_seed)

    graph = Graph.fully_connected([f"v{i}" for i in range(nvert)])

    if kind == "Euclidean":
        # randomly place vertices in the unit square
        positions = {v: gen.uniform(size=2) for v in graph.vertices}
        graph.place_vertices(positions)

    elif kind == "Unit":
        graph.weights = {e: gen.uniform(low=0, high=1) for e in graph.edges}
        # symmetrize
        #for u, v in graph.edges:
        #    graph.weights[frozenset((u, v))] = graph.weights[frozenset((v, u))]

    # randomly generate initial tour
    tour = copy.deepcopy(graph.vertices)
    tour = list(map(frozenset, zip(tour, tour[1:] + [tour[0]])))

    return graph, tour

def generate_ktree(nvert, k, rng_seed=None, kind="Euclidean"):
    gen = np.random.RandomState(seed=rng_seed)

    assert k <= nvert + 1

    base_vertices = [f"v{i}" for i in range(k+1)]
    # identify the subset of vertices to which all added
    # vertices will be neighbors
    subset = base_vertices[:k-2]

    graph = Graph.fully_connected(copy.deepcopy(base_vertices))

    for i in range(k+1, nvert):
        # add new vertex
        graph.vertices.append(f"v{i}")

        # add edges to create a k-tree
        # for each new vertex v, connect it to exactly k-2 vertices S
        # from the base graph in such a way that S and {v} together
        # form a clique.

        # connect each new vertex to exactly k-2 of the base vertices
        graph.edges += [frozenset({f"v{i}", v}) for v in subset]

    # finally, connect all added vertices in a cycle
    added_vs = [f"v{i}" for i in range(k+1, nvert)]
    cycle = copy.deepcopy(added_vs)

    graph.edges += [frozenset({u, v}) for u, v in zip(cycle, cycle[1:])] + [frozenset({cycle[-1], cycle[0]})]

    # a hamiltonian cycle is given by the outer cycle excepting one edge, plus an edge to v0, plus a hamiltonian
    # path through the original graph ending at any vertex of the neighbor subset
    tour = cycle

    start = subset[0]
    end = subset[1]

    tour.append(start)
    remaining = subset[2:]
    tour += remaining
    tour += base_vertices[k-2:]
    tour.append(end)

    tour = [frozenset({u, v}) for u, v in zip(tour, tour[1:])] + [frozenset({tour[-1], tour[0]})]

    if kind == "Unit":
        for edge in graph.edges:
            graph.set_weight(gen.uniform(low=0, high=1), edge)
    elif kind == "Euclidean":
        positions = {v: gen.uniform(size=2) for v in graph.vertices}
        graph.place_vertices(positions)
    else:
        raise ValueError("Weight distribution not supported")

    return graph, tour

def solve_tsp(graph, tour, optimizer, it_max=1000): #tour, weights, it_max=1000):

    for i in range(it_max):

        new_tour = optimizer(tour, graph.weights)

        if path_length(tour, graph.weights) <= path_length(new_tour, graph.weights):
            break
        else:
            tour = new_tour

    return new_tour, i + 1

def solve_ensemble(optimizer, num_tours, nvert, seeds=None, greedy=False, compute_lb=True, kind="Euclidean", tour_generator=generate_random_tour):

    if not seeds:
        seeds = [None]*num_tours

    len_original = np.zeros(num_tours)
    len_optimal = np.zeros(num_tours)
    dts = np.zeros(num_tours)
    lower_bounds = np.zeros(num_tours)

    for i in range(num_tours):
        graph, tour = tour_generator(nvert, rng_seed=seeds[i], kind=kind)

        if compute_lb:
            if CONCORDE_AVAILABLE:
                lb = path_length(tsp_concorde(graph), graph.weights)
            elif SOLVER_AVAILABLE:
                lb = path_length(tsp_solver(graph), graph.weights)
            else:
                lb = tsp_lower_bound(graph)
        else:
            lb = np.inf

        if greedy:
            tour = greedy_tsp(graph.vertices, graph.weights)

        len_original[i] = path_length(tour, graph.weights)

        optimal_tour, dt = solve_tsp(graph, tour, optimizer)
        len_optimal[i] = path_length(optimal_tour, graph.weights)
        lower_bounds[i] = lb

        dts[i] = dt

    return len_original, len_optimal, dts, lower_bounds

def average_case_time(num_tours, nverts, optimizer, greedy=False, kind="Euclidean", tour_generator=generate_random_tour):
    dts = np.zeros(nverts.shape)

    for i, nv in enumerate(nverts):
        seeds = list(range(i*num_tours, (i+1)*num_tours))
        len_orig, len_optimal, dt, lower_bounds \
            = solve_ensemble(optimizer, num_tours, nv, seeds=seeds, greedy=greedy, compute_lb=False, kind=kind, tour_generator=tour_generator)
        dts[i] = np.mean(dt)

    return dts

def average_case_ratio(num_tours, nverts, optimizer, greedy=False, kind="Euclidean"):
    ratios = np.zeros(nverts.shape)

    for i, nv in enumerate(nverts):
        seeds = list(range(i*num_tours, (i+1)*num_tours))
        len_orig, len_optimal, dt, lbs = solve_ensemble(optimizer, num_tours, nv, seeds=seeds, greedy=greedy, kind=kind)
        ratios[i] = np.mean(len_optimal / lbs)

    return ratios

def tsp_concorde(graph):
    # solve with concorde
    pos = np.array(list(graph.positions.values()))
    xs = pos[:, 0]
    ys = pos[:, 1]

    solver = TSPSolver.from_data(xs, ys, norm="GEO")
    sol = solver.solve(verbose=False)

    optimal_tour = [graph.vertices[i] for i in sol.tour]
    return list(map(frozenset, list(zip(optimal_tour, optimal_tour[1:])) + [(optimal_tour[-1], optimal_tour[0])]))

def tsp_solver(graph):
    distance_matrix = np.zeros((len(graph.vertices), len(graph.vertices)))
    for i, u in enumerate(graph.vertices):
        for j, v in enumerate(graph.vertices):
            if i == j:
                distance_matrix[i, i] == 0
            else:
                distance_matrix[i, j] = graph.weights[frozenset({u, v})]

    perm, distance = solve_tsp_lib(distance_matrix)

    tour = [graph.vertices[i] for i in perm]
    tour = [frozenset({u, v}) for u, v in zip(tour, tour[1:])] + [frozenset({tour[-1], tour[0]})]
    return tour

max_stars_repo_name: Jvanrhijn/tsp-local-search

max_issues_repo_name: Jvanrhijn/tsp-local-search

max_forks_repo_name: Jvanrhijn/tsp-local-search


text:
#!/usr/bin/env python3
# Generates an example R-Tree with some trajectory units.

import json
import matplotlib
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D, proj3d
import numpy as np
import subprocess

from itertools import combinations, product
from common import RESULT_PATH
from common import compile

def color(a, b, c):
    return (a / 255, b / 255, c / 255.0)

# Format: min, max, color, label, labelpos
cubes = [
    # N1
    ((13, 5, 0), (30, 20, 5), color(132, 141, 197), "$R_1$", (12, 3, 0)),
    # N2
    ((4, 5, 5), (15, 18, 15), color(147, 156, 72), "$R_2$", (2, 4, 4)),
    # N3
    ((15, 10, 5), (25, 25, 15), color(159, 98, 198), "$R_3$", (26, 25, 15)),
    # N4
    ((5, 15, 10), (15, 25, 20), color(203, 88, 96), "$R_4$", (3, 14, 20)),
]

trajectories = [
    {
        "points": [(30, 20, 0), (15, 18, 5), (4, 10, 10), (5, 5, 15)],
        "label": "Trajectory 1",
        "color": "red",
    },
    {
        "points": [(20, 5, 0), (25, 10, 5), (25, 20, 10), (15, 25, 15), (5, 25, 20)],
        "label": "Trajectory 2",
        "color": "green",
    },
    {
        "points": [(15, 9, 0), (13, 12, 5), (13, 15, 10), (10, 18, 15), (6, 17, 20)],
        "label": "Trajectory 3",
        "color": "blue",
    },
]

def pairwise(iterable):
    """iterate over adjecent pairs"""
    it = iter(iterable)
    a = next(it, None)
    for b in it:
        yield (a, b)
        a = b

def plot_cube(ax, min, max, color, label, labelpos):
    # All cube corners
    points = np.array(list(product(*zip(min, max))))

    backcorner = np.array([min[0], max[1], min[2]])

    # All cube corner pairs
    for a, b in combinations(points, 2):
        # Differing coordinates
        diffs = (1 for i in range(3) if a[i] != b[i])
        # Draw a line if they differ in exactly one coordinate
        if sum(diffs) == 1:
            #‘solid’ | ‘dashed’, ‘dashdot’, ‘dotted’ | (offset, on-off-dash-seq)
            back = (a == backcorner).all() or (b == backcorner).all()
            linestyle = "dotted" if back else "solid"
            ax.plot3D(*zip(a, b), color=color, linestyle=linestyle)

    ann_x, ann_y, _ = proj3d.proj_transform(*labelpos, ax.get_proj())
    ax.text(*labelpos, label, color=color)

def plot_trajectory(ax, points, color, label):
    ax.plot3D(*zip(*points), color=color, marker="o", label=label)

fig = plt.figure(figsize=(8, 8))

ax = fig.add_subplot(111, projection="3d")
ax.set_xlabel("x")
ax.set_ylabel("y")
ax.set_zlabel("t")
ax.set_aspect("equal")
ax.locator_params(integer=True)

ax.set_xlim(0, 30)
ax.set_xticks([0, 10, 20, 30])
ax.set_ylim(0, 25)
ax.set_yticks([0, 10, 20])
ax.set_zlim(0, 20)
ax.set_zticks([0, 10, 20])

for traj in trajectories:
    plot_trajectory(ax, traj["points"], traj["color"], traj["label"])

for cube in cubes:
    plot_cube(ax, *cube)

ax.legend(loc="upper left")

fig.savefig(str(RESULT_PATH / "irwi_example_boxes.pdf"), bbox_inches="tight")

max_stars_repo_name: mbeckem/msc

max_issues_repo_name: mbeckem/msc

max_forks_repo_name: mbeckem/msc


text:
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import gzip
import logging
import argparse
from PIL import Image
import numpy as np

import paddle.v2 as paddle
from paddle.utils.dump_v2_config import dump_v2_config

logger = logging.getLogger("paddle")
logger.setLevel(logging.INFO)

def multilayer_perceptron(img, layer_size, lbl_dim):
    for idx, size in enumerate(layer_size):
        hidden = paddle.layer.fc(input=(img if not idx else hidden),
                                 size=size,
                                 act=paddle.activation.Relu())
    return paddle.layer.fc(input=hidden,
                           size=lbl_dim,
                           act=paddle.activation.Softmax())

def network(input_dim=784, lbl_dim=10, is_infer=False):
    images = paddle.layer.data(
        name='pixel', type=paddle.data_type.dense_vector(input_dim))

    predict = multilayer_perceptron(
        images, layer_size=[128, 64], lbl_dim=lbl_dim)

    if is_infer:
        return predict
    else:
        label = paddle.layer.data(
            name='label', type=paddle.data_type.integer_value(lbl_dim))
        return paddle.layer.classification_cost(input=predict, label=label)

def main(task="train", use_gpu=False, trainer_count=1, save_dir="models"):
    if task == "train":
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)

        paddle.init(use_gpu=use_gpu, trainer_count=trainer_count)
        cost = network()
        parameters = paddle.parameters.create(cost)
        optimizer = paddle.optimizer.Momentum(
            learning_rate=0.1 / 128.0,
            momentum=0.9,
            regularization=paddle.optimizer.L2Regularization(rate=0.0005 * 128))

        trainer = paddle.trainer.SGD(cost=cost,
                                     parameters=parameters,
                                     update_equation=optimizer)

        def event_handler(event):
            if isinstance(event, paddle.event.EndIteration):
                if event.batch_id % 100 == 0:
                    logger.info("Pass %d, Batch %d, Cost %f, %s" %
                                (event.pass_id, event.batch_id, event.cost,
                                 event.metrics))
            if isinstance(event, paddle.event.EndPass):
                with gzip.open(
                        os.path.join(save_dir, "params_pass_%d.tar" %
                                     event.pass_id), "w") as f:
                    trainer.save_parameter_to_tar(f)

        trainer.train(
            reader=paddle.batch(
                paddle.reader.shuffle(
                    paddle.dataset.mnist.train(), buf_size=8192),
                batch_size=128),
            event_handler=event_handler,
            num_passes=5)
    elif task == "dump_config":
        predict = network(is_infer=True)
        dump_v2_config(predict, "trainer_config.bin", True)
    else:
        raise RuntimeError(("Error value for parameter task. "
                            "Available options are: train and dump_config."))

def parse_cmd():
    parser = argparse.ArgumentParser(
        description="PaddlePaddle MNIST demo for CAPI.")
    parser.add_argument(
        "--task",
        type=str,
        required=False,
        help=("A string indicating the taks type. "
              "Available options are: \"train\", \"dump_config\"."),
        default="train")
    parser.add_argument(
        "--use_gpu",
        type=bool,
        help=("A bool flag indicating whether to use GPU device or not."),
        default=False)
    parser.add_argument(
        "--trainer_count",
        type=int,
        help=("This parameter is only used in training task. It indicates "
              "how many computing threads are created in training."),
        default=1)
    parser.add_argument(
        "--save_dir",
        type=str,
        help=("This parameter is only used in training task. It indicates "
              "path of the directory to save the trained models."),
        default="models")
    return parser.parse_args()

if __name__ == "__main__":
    args = parse_cmd()
    main(args.task, args.use_gpu, args.trainer_count, args.save_dir)