text:
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import time
import math
import numpy as np
import torch
from torch.autograd import Variable
from torch.nn.parameter import Parameter
from torch.multiprocessing import Queue, Pipe
import torch.multiprocessing as mp
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch_scatter import scatter
from functools import partial
from abc import ABC, abstractmethod
from smore.common.embedding.sparse_embed import SparseEmbedding
from smore.common.embedding.embed_rw import EmbeddingRW
from smore.common.util import eval_tuple
from smore.common.consts import eps

def merge_grad_indices(idx_list):
    concatenated_indices = torch.cat(idx_list, dim=0)
    merged_indices, inverse_indices = torch.unique(concatenated_indices, return_inverse=True)
    return merged_indices, inverse_indices

def merge_grad_list(grad_list, inverse_indices, reduce_type="sum"):
    concatenated_grad = torch.cat(grad_list, dim=0)
    merged_grad = scatter(concatenated_grad, inverse_indices, dim=0, reduce=reduce_type)
    return merged_grad

class EmbedLookupFunc(torch.autograd.Function):

    @staticmethod
    def forward(ctx, embed_opt, dummy_tensor, indices, name):
        ctx.embed_opt = embed_opt
        ctx.indices = indices
        ctx.idx_fwd = embed_opt.n_fwd
        embed_opt.n_fwd += 1
        with torch.no_grad():
            submat = embed_opt.embed_rw.read(indices, name)
        return submat

    @staticmethod
    def backward(ctx, grad_output):
        ctx.embed_opt.accumulate_grad(ctx.indices, grad_output, ctx.idx_fwd)
        return None, None, None, None

class EmbeddingOptimizer(ABC):

    @classmethod
    @abstractmethod
    def prepare_embed(cls, args, optim_mode, embed, fn_register):
        pass

    @abstractmethod
    def share_memory(self):
        pass

    @abstractmethod
    def prepare_forward(self, indices, name):
        pass

    @abstractmethod
    def update_step(self, indices, inverse_indices, grad, lr):
        pass

    def zero_grad(self):
        self.grad_list = []
        self.n_fwd = 0
        self.has_grad_list = []
        self.grad_stats = []

    def filter_stats(self, list_stat_full):
        list_st = []
        for i in self.has_grad_list:
            e = list_stat_full[i]
            e.job_handle.sync()
            list_st.append(e)
        return list_st

    @classmethod
    def prepare_optimizers(cls, args, list_sparse_embeds):
        if not args.share_optim_stats:  # no need to have global optimizer stats
            return
        optim_mode = eval_tuple(args.optim_mode)
        for e in list_sparse_embeds:
            assert isinstance(e, SparseEmbedding)
            fn_register = lambda n, t: e.register_tensor(n, t)
            cls.prepare_embed(args, optim_mode, e, fn_register)

    def set_local_rw(self, name, tensor, gpu_id):
        cur_rw = EmbeddingRW(tensor, gpu_id=gpu_id)
        setattr(self, name + '_rw', cur_rw)

    def __init__(self, args, sp_embed, gpu_id):
        self.args = args
        self.grad_stats = []
        self.embed_rw = EmbeddingRW(sp_embed.embedding, gpu_id=gpu_id)
        self.sparse_embed = sp_embed
        sp_embed.fwd_delegate = self.forward

        self.dummy_tensor = nn.Parameter(torch.randn(1, 1))
        self.grad_list = []
        self.n_fwd = 0
        self.step = 0
        self.gpu_id = gpu_id

        optim_mode = eval_tuple(args.optim_mode)
        self.optim_mode = optim_mode
        assert self.optim_mode[0] in ['fast', 'aggr']
        self.optimizer_name = optim_mode[1]
        self.optimizer_device = optim_mode[2]
        assert self.optimizer_device in ['gpu', 'cpu']
        self.squeeze = self.optim_mode[3]
        self.queue_size = self.optim_mode[4]

        self.share_optim_stats = args.share_optim_stats
        fn_local_rw = partial(self.set_local_rw, gpu_id=self.gpu_id)
        if not self.share_optim_stats:
            self.prepare_embed(args, optim_mode, sp_embed, fn_local_rw)
        else:
            for name, buf in sp_embed.named_buffers(recurse=False):
                fn_local_rw(name, buf)

    def accumulate_grad(self, indices, grad_submat, fwd_idx):
        self.grad_list.append((indices, grad_submat))
        self.has_grad_list.append(fwd_idx)

    def forward(self, indices, name):
        if self.sparse_embed.training and self.optimizer_device == 'cpu':
            self.prepare_forward(indices, name)
        return EmbedLookupFunc.apply(self, self.dummy_tensor, indices, name)

    def apply_grad(self, lr):
        self.step += 1
        if len(self.grad_list) == 0:
            return
        merged_indices, inverse_indices = merge_grad_indices([indices for indices, _ in self.grad_list])
        grad_list = [grad for _, grad in self.grad_list]
        inverse_indices = inverse_indices.to(grad_list[0].device)
        merged_grad = merge_grad_list(grad_list, inverse_indices)
        self.update_step(merged_indices, inverse_indices, merged_grad, lr)

class SGDEmbedOptimizer(EmbeddingOptimizer):

    @classmethod
    def prepare_embed(cls, args, optim_mode, embed, fn_register):
        pass

    def share_memory(self):
        pass

    def zero_grad(self):
        super(SGDEmbedOptimizer, self).zero_grad()

    def prepare_forward(self, indices, name):
        pass

    def update_step(self, indices, inverse_indices, grad, lr):
        with torch.no_grad():
            # calculate final grad and update embed_mat
            tmp = -lr * grad
            self.embed_rw.write(indices, tmp, name="embed_gd", additive=True)

class AdamEmbedOptimizer(EmbeddingOptimizer):

    @classmethod
    def prepare_embed(cls, args, optim_mode, embed, fn_register):
        fn_register("state_sum_fo", torch.zeros(embed.embedding.size()))
        squeeze = optim_mode[3]
        if squeeze:
            fn_register("state_sum_sq", torch.zeros(embed.embedding.size(0), 1))
        else:
            fn_register("state_sum_sq", torch.zeros(embed.embedding.size()))

    def share_memory(self):
        self.state_sum_fo_rw.embed.share_memory_()
        self.state_sum_sq_rw.embed.share_memory_()

    def prepare_forward(self, indices, name):
        fo = self.state_sum_fo_rw.read(indices, name)
        sq = self.state_sum_sq_rw.read(indices, name)
        self.grad_stats.append((fo, sq))

    def update_step(self, indices, inverse_indices, grad, lr):
        beta1 = 0.9
        beta2 = 0.999 # this might further improve the performance if we keep track of the optimization steps for each embedding rather than using a global optimization step for all embeddings
        # indices, grad on gpu
        stats_device = self.state_sum_sq_rw.embed.device
        with torch.no_grad():
            if self.optimizer_device == 'gpu':
                indices = indices.to(stats_device)
            # for indices, grad in zip(indices_list, grad_list):
            bias_correction1 = 1 - beta1 ** self.step
            bias_correction2 = 1 - beta2 ** self.step
            indices = indices.to(stats_device)
            if self.squeeze:
                grad_sum_sq = torch.mean(grad * grad, dim=1, keepdim=True) # gpu

            # retreive old momentum
            if self.optimizer_device == 'cpu':
                list_fo_full, list_sq_full = zip(*self.grad_stats)
                list_fo = self.filter_stats(list_fo_full)
                grad_avg_fo = merge_grad_list(list_fo, inverse_indices, reduce_type="max")
                list_sq = self.filter_stats(list_sq_full)
                grad_avg_sq = merge_grad_list(list_sq, inverse_indices, reduce_type="max")
            else:
                grad_avg_fo = self.state_sum_fo_rw.read(indices)
                grad_avg_sq = self.state_sum_sq_rw.read(indices)
            bak_fo = grad_avg_fo.clone()
            bak_sq = grad_avg_sq.clone()

            # update momentum
            if self.squeeze:
                grad_avg_sq.mul_(beta2).add_(grad_sum_sq * (1-beta2))
            else:
                grad_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1-beta2)
            grad_avg_fo.mul_(beta1).add_(grad, alpha=1-beta1)

            # calculate avg
            denom = (grad_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)

            # calculate final grad and update embed_mat
            tmp = -lr/bias_correction1 * grad_avg_fo/denom
            self.embed_rw.write(indices, tmp, name="embed_gd", additive=True)

            # copy new momentum to state_sum
            self.state_sum_fo_rw.write(indices, grad_avg_fo - bak_fo, name="update_fo", additive=True)
            self.state_sum_sq_rw.write(indices, grad_avg_sq - bak_sq, name="update_sq", additive=True)

class RMSpropEmbedOptimizer(EmbeddingOptimizer):

    @classmethod
    def prepare_embed(cls, args, optim_mode, embed, fn_register):
        squeeze = optim_mode[3]
        if squeeze:
            fn_register("state_sum", torch.zeros(embed.embedding.size(0)))
        else:
            fn_register("state_sum", torch.zeros(embed.embedding.size()))

    def share_memory(self):
        self.state_sum.share_memory_()

    def __init__(self, args, sp_embed, gpu_id):
        super(RMSpropEmbedOptimizer, self).__init__(args, sp_embed, gpu_id)

    def update_step(self, indices, grad, lr):
        alpha = 0.99
        # indices, grad on gpu
        with torch.no_grad():
            # for indices, grad in zip(indices_list, grad_list):
            indices = indices.to(self.state_sum.device)
            if self.squeeze:
                grad_sum = (grad * grad).mean(1) # gpu
            if self.optimizer_device == 'cpu':
                grad = grad.to(self.state_sum.device)
                if self.squeeze:
                    grad_sum = grad_sum.to(self.state_sum.device)

            # retreive old momentum
            square_avg = self.state_sum[indices]
            if self.optimizer_device == 'gpu':
                square_avg = square_avg.to(grad.device)

            # update momentum
            if self.squeeze:
                square_avg.mul_(alpha).add_(grad_sum * (1-alpha))
            else:
                square_avg.mul_(alpha).addcmul_(grad, grad, value=1-alpha)

            # calculate avg
            avg = square_avg.sqrt().add_(eps)
            if self.squeeze:
                avg = avg.unsqueeze(1)

            # copy new momentum to state_sum
            if self.optimizer_device == 'gpu':
                square_avg = square_avg.to(self.state_sum.device)
            self.state_sum.index_copy_(0, indices, square_avg)

            # calculate final grad and update embed_mat
            tmp = -lr * grad/avg
            if self.optimizer_device == 'gpu':
                tmp = tmp.to(self.sp_embed.device)
            self.sp_embed.embedding.index_add_(0, indices, tmp)

class AdaGradEmbedOptimizer(EmbeddingOptimizer):

    @classmethod
    def prepare_embed(cls, args, optim_mode, embed, fn_register):
        squeeze = optim_mode[3]
        if squeeze:
            fn_register("state_sum", torch.zeros(embed.embedding.size(0), 1))
        else:
            fn_register("state_sum", torch.zeros(embed.embedding.size()))

    def share_memory(self):
        self.state_sum_rw.embed.share_memory_()

    def prepare_forward(self, indices, name):
        fo = self.state_sum_rw.read(indices, name)
        self.grad_stats.append(fo)

    def update_step(self, indices, inverse_indices, grad, lr):
        # indices, grad on gpu
        stats_device = self.state_sum_rw.embed.device
        with torch.no_grad():
            if self.optimizer_device == 'gpu':
                indices = indices.to(stats_device)
            if self.squeeze:
                grad_sum = torch.mean(grad * grad, dim=1, keepdim=True) # gpu
            else:
                grad_sum = grad * grad

            if self.optimizer_device == 'cpu':
                list_sq = self.filter_stats(self.grad_stats)
                square_avg = merge_grad_list(list_sq, inverse_indices, reduce_type="max")
            else:
                square_avg = self.state_sum_rw.read(indices)
            square_avg = square_avg + grad_sum
            avg = square_avg.sqrt().add_(eps)
            tmp = -lr * grad/avg
            self.embed_rw.write(indices, tmp, name="embed_gd", additive=True)
            self.state_sum_rw.write(indices, grad_sum, name="update_sq", additive=True)

def get_optim_class(args):
    optim_mode = eval_tuple(args.optim_mode)
    optimizer_name = optim_mode[1]
    if optimizer_name == 'adam':
        return AdamEmbedOptimizer
    elif optimizer_name == 'rmsprop':
        return RMSpropEmbedOptimizer
    elif optimizer_name == 'adagrad':
        return AdaGradEmbedOptimizer
    elif optimizer_name == 'sgd':
        return SGDEmbedOptimizer
    else:
        raise NotImplementedError

max_issues_repo_name: google-research/smore

max_forks_repo_name: pyg-team/smore


text:
import numpy as np

class GAPScorer(object):
    def __init__(self, top_k=20):
        self.scores_list = []
        self.targets_list = []
        self.top_k = top_k
        self.n_pos = 0

    def add(self, pred, target):
        top_idxs = np.argsort(pred, axis=1)[:, ::-1][:, :self.top_k]
        self.n_pos += target.sum()
        n_repeats = min(self.top_k, top_idxs.shape[1])
        rows_idxs = np.arange(len(top_idxs))[..., np.newaxis].repeat(n_repeats, axis=1)
        top_targets = target[rows_idxs, top_idxs]
        top_preds = pred[rows_idxs, top_idxs]
        self.scores_list.append(top_preds.reshape(-1))
        self.targets_list.append(top_targets.reshape(-1))

    @property
    def current_value(self):
        scores_arr = np.concatenate(self.scores_list)
        targets_arr = np.concatenate(self.targets_list)
        idxs = np.argsort(scores_arr)[::-1]
        sorted_targets = targets_arr[idxs]
        ranks = np.arange(len(idxs)) + 1
        n_pos_at_rank = np.cumsum(sorted_targets)
        AP = (n_pos_at_rank * sorted_targets / ranks).sum() / self.n_pos
        return AP

class GAPScorerFilteredColumns(GAPScorer):
    def __init__(self, from_col, to_col, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._from_col = from_col
        self._to_col = to_col

    def add(self, pred, target):
        filtered_pred = pred[:, self._from_col:self._to_col]
        filtered_target = target[:, self._from_col:self._to_col]
        super().add(filtered_pred, filtered_target)

ext: py

reason: import numpy


text:
from YGRW.run import generate_trajectory
import numpy as np

def test_basic_run():
    """
    Verify that runs can be taken
    Returns
    -------

    """

    traj = generate_trajectory(timesteps=1, write_after=True)

    assert len(traj) == 2
    assert not np.array_equal(traj.position, traj.positions[0])

    impossible_traj = generate_trajectory(timesteps=1, nuclear_radius=0)
    assert len(impossible_traj) == 2
    assert np.array_equal(impossible_traj.positions, [[0, 0], [0, 0]])

max_stars_repo_name: MCnu/YGRW


text:
#!/usr/bin/env python

#
# REV3 Project
# Copyright 2016, - All Rights Reserved
#
# Team BPWALSH
#

import numpy as np

#
# generate simulated sensor data of phone laying flat
#

NS2S = 1.0 / 1000000000.0

times = np.arange(0.0, 200.0, 0.05)

for t in times:
    print "%d,%d,%f,%f,%f" % (t/NS2S,1, 0.0, 0.0, 9.81) # accelerometer inputs
    print "%d,%d,%f,%f,%f" % (t/NS2S,4, 0.0, 0.0, 0.0) # gyro inputs


text:
# Universidade Federal de Viçosa - Campus Rio Paranaíba
# Sistemas de Informação - Processamento Digital de Imagens
#
# Professor: Joao Mari
# Autores:
#          - MatheusRV (3929)
#          - iguit0 (3902)
#          - ThiagoMunich (3628)
#
# Transformações de intensidade -  Construção de histogramas
# Como Executar:
#  $ python hist.py img_1.tif
#
# Instruções do Projeto:
#  - OBS: Não gera imagem de saída!
#

import sys
import matplotlib.pyplot as plt
from scipy import misc

def loadImg(arg):
    return misc.imread(arg)

# Lê a imagem a partir de um arquivo
img_1 = loadImg(sys.argv[1])
saida = 'histograma.tif'

# Transforma os níveis de intensidade numa imagem de uma dimensão
histograma = img_1.flatten()

'''
# Organiza o plote das imagens
plt.figure()
plt.subplot(221)
plt.hist(histograma, bins=256, range=(0,255))
plt.title('histograma')
plt.savefig(saida)
plt.subplot(222)
plt.imshow(img_1, cmap='gray', interpolation='nearest')
plt.title('img_entrada')

# Plota as imagens de entrada e saída na tela
plt.show()
'''

'''
from scipy import misc
import matplotlib.pyplot as plt
from skimage import img_as_float

# Carrega a imagem a partir do arquivo.
im = misc.imread('ascent.tif')

# Histograma da imagem (uint-8)
plt.figure()
plt.subplot(2,3,1)
plt.imshow(im, cmap='gray')
plt.colorbar()
plt.title('Imagem original - dtype=''uint8''.')
plt.subplot(2,3,2)
plt.hist(im.flatten(), bins=256, range=(0,255))
plt.title('Histograma nao normalizado.')
plt.subplot(2,3,3)
plt.hist(im.flatten(), bins=256, range=(0,255), normed=True)
plt.title('Histograma normalizado.')
# Converte a imagem de uint-8 [0..255] para float [0..1].

im = img_as_float(im)
# Histograma da imagem (float)
plt.subplot(2,3,4)
plt.imshow(im, cmap='gray')
plt.colorbar()
plt.title('Imagem original - dtype=''float''.')
plt.subplot(2,3,5)
plt.hist(im.flatten(), bins=256, range=(0,1))
plt.title('Histograma nao normalizado.')
plt.subplot(2,3,6)
plt.hist(im.flatten(), bins=256, range=(0,1), normed=True)
plt.title('Histograma normalizado.')

# Mostra as figuras na tela
plt.show()
'''

max_stars_repo_path: algorithms/2 - Intensity Transformations/4 - Histograms Building/hist.py

max_stars_repo_name: MatheusRV/Image-Processing-Course

max_issues_repo_name: MatheusRV/Image-Processing-Course

max_forks_repo_name: MatheusRV/Image-Processing-Course

reason: from scipy


text:
#!/usr/bin/python3
# -*- coding=utf-8 -*-

import numpy as np
from common.yolo_postprocess_np import yolo_decode, yolo_handle_predictions, yolo_correct_boxes, yolo_adjust_boxes

def yolo3_decode(predictions, anchors, num_classes, input_dims, elim_grid_sense=False):
    """
    YOLOv3 Head to process predictions from YOLOv3 models

    :param num_classes: Total number of classes
    :param anchors: YOLO style anchor list for bounding box assignment
    :param input_dims: Input dimensions of the image
    :param predictions: A list of three tensors with shape (N, 19, 19, 255), (N, 38, 38, 255) and (N, 76, 76, 255)
    :return: A tensor with the shape (N, num_boxes, 85)
    """
    assert len(predictions) == len(anchors)//3, 'anchor numbers does not match prediction.'

    if len(predictions) == 3: # assume 3 set of predictions is YOLOv3
        anchor_mask = [[6,7,8], [3,4,5], [0,1,2]]
        scale_x_y = [1.05, 1.1, 1.2] if elim_grid_sense else [None, None, None]
    elif len(predictions) == 2: # 2 set of predictions is YOLOv3-tiny
        anchor_mask = [[3,4,5], [0,1,2]]
        scale_x_y = [1.05, 1.05] if elim_grid_sense else [None, None]
    else:
        raise ValueError('Unsupported prediction length: {}'.format(len(predictions)))

    results = []
    for i, prediction in enumerate(predictions):
        results.append(yolo_decode(prediction, anchors[anchor_mask[i]], num_classes, input_dims, scale_x_y=scale_x_y[i], use_softmax=False))

    return np.concatenate(results, axis=1)

def yolo3_postprocess_np(yolo_outputs, image_shape, anchors, num_classes, model_image_size, max_boxes=100, confidence=0.1, iou_threshold=0.4, elim_grid_sense=False):
    predictions = yolo3_decode(yolo_outputs, anchors, num_classes, input_dims=model_image_size, elim_grid_sense=elim_grid_sense)
    predictions = yolo_correct_boxes(predictions, image_shape, model_image_size)

    boxes, classes, scores = yolo_handle_predictions(predictions,
                                                     image_shape,
                                                     max_boxes=max_boxes,
                                                     confidence=confidence,
                                                     iou_threshold=iou_threshold)

    boxes = yolo_adjust_boxes(boxes, image_shape)

    return boxes, classes, scores

max_stars_repo_name: kimbfs/Proyecto

max_issues_repo_name: kimbfs/Proyecto

max_forks_repo_name: kimbfs/Proyecto


text:
import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

max_stars_repo_path: Binary Classification using ANN/imports.py


text:
#!/usr/bin/env python
# encoding: utf-8

import pandas as pd
from pandas import ExcelWriter
import numpy as np
import csv
import matplotlib.pyplot as plt
from urllib.parse import urlparse
from collections import Counter
import math
import requests
import httplib2
import logging

DATAFILENAME = "./data/Software&TechnicalProducts - ResearchFish.xlsx"
DATASHEETNAME = "Software_TechnicalProducts"
CHART_STORE_DIR = "./charts/"
EXCEL_RESULT_STORE = "./data/researchfish_results.xlsx"
IMPACT_RESULT_STORE = "./data/impact.txt"
LOGGERLOCATION = "./log/ResearchFishLog.log"

def import_xls_to_df(filename, name_of_sheet):
    """
    Imports an Excel file into a Pandas dataframe
    :params: get an xls file and a sheetname from that file
    :return: a df
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Importing data...')

    return pd.read_excel(filename,sheetname=name_of_sheet)

def add_column(dataframe,newcol):
    """
    Adds a new column of NaNs called newcol
    :params: a dataframe and column name
    :return: a dataframe with a new column
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Adding a column...')
    dataframe[newcol] = np.nan
    return dataframe

def clean_data(dataframe,colname):
    """
    Cleans the dataframe based on advice we have been given by EPSRC:
    1. Remove the tech products that don't actually relate to software
    2. Remove duplicate records (where a duplicate is defined as a record with the same 'Tech product' as another record)
    3. Removes years before 2012. Only 2012-2016 are reliable enough to include in the study
    4. Removes years that include text (i.e. "Pre-2016") because it's ambiguous
    5. Drops any years that include NaN in the year category
    :params: a dataframe and a colname of the column in which the years are stored
    :return: a dataframe with only int years and NaNs
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Cleaning data...')

    # Want some metrics on how many records are being dropped. Set up a variable to store length of dataframe before each cleaning operation
    length_start = len(dataframe)

    # Drop all outputs that aren't related to the four products that are classed as software
    dataframe = dataframe[(dataframe['Type of Tech Product'] == 'Software') | (dataframe['Type of Tech Product'] == 'Grid Application') | (dataframe['Type of Tech Product'] == 'e-Business Platform') | (dataframe['Type of Tech Product'] == 'Webtool/Application')]

    length_tech_product = len(dataframe)

    # Remove duplicate entries where duplication occurs in the 'Impact' AND the 'Tech Product' fields
    dataframe.drop_duplicates(subset = ['Tech Product'], keep = 'first', inplace = True)

    length_dupes = len(dataframe)

    # Remove data from years where EPSRC are less certain that the data is accurate
    lost_years = ['2006', '2007', '2008', '2009', '2010', '2011']
    for year in lost_years:
        dataframe.drop(dataframe[dataframe[colname] == year].index, inplace = True)

    length_years = len(dataframe)

    # Go through the rows, if you can't convert the year into an int (i.e. the entry includes the text "Pre-"), write a NaN back into the dataframe
    for i, row in dataframe.iterrows():
        try:
            int(dataframe[colname][i])
        except:
            dataframe[colname][i] = np.nan

    # Drop any data which lacks a info on Year First Provided because we're unsure of its provenance
    dataframe.dropna(subset=[colname], inplace=True)

    length_final = len(dataframe)

    # Print details on the changes to the dataframe length during cleaning
    logger.info("Records dropped during tech product cleaning: " + repr(length_start - length_tech_product))
    logger.info("Records dropped during duplicate cleaning: " + repr(length_tech_product - length_dupes))
    logger.info("Records dropped when cleaning years outside 2012-2016: " + repr(length_dupes - length_years))
    logger.info("Records dropped during non-valid-year cleaning: " + repr(length_years - length_final))
    logger.info("Records left in cleaned data set: " + repr(length_final))

    return dataframe

def produce_count(dataframe, colname):
    """
    When given a column, returns a count of its unique values
    This DOES NOT count blank entries - unlike produce_count_and_na()
    :params: a data frame and a column name in the dataframe
    :return: a table of unique names and their count
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Producing a count...')

    dataframe = pd.DataFrame(dataframe[colname].value_counts())

    # Add a column for percentages
    dataframe['percentage'] = dataframe[colname]/dataframe[colname].sum()

    return dataframe

def produce_count_and_na(dataframe, colname):
    """
    When given a column, returns a count of its unique values
    This ALSO DOES count blank entries - unlike produce_count()
    Special measures need to be employed for the 'Open source' column, because that question is only asked for 'Tech product' of the
    'Software variety' hence the if statement
    :params: a data frame and a column name in the dataframe
    :return: a table of unique names and their count
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Producing a count and including na...')

    # Employ special measures as discussed above for 'Open Source?' field
    if colname == 'Open Source?':
        temp_dataframe = dataframe[dataframe['Type of Tech Product'] == 'Software']
        dataframe = pd.DataFrame(temp_dataframe[colname].value_counts(dropna = False))
    # pd.DataFrame(dataframe[(dataframe['Tech Product'] == 'Software') & (dataframe[colname])].value_counts(dropna = False))
    else:
        dataframe = pd.DataFrame(dataframe[colname].value_counts(dropna = False))

    # Add a column for percentages
    dataframe['percentage'] = dataframe[colname]/dataframe[colname].sum()

    return dataframe

def get_root_domains(dataframe,colname):
    """
    Takes a df and a column in it which contains urls, then extracts the main domain part (the 'netloc')
    of the url and writes the results into a new df
    :params: a data frame and a column name in the dataframe
    :return: a new df of URL netlocs
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Getting root domains...')

    # initialise list
    list_of_rootdomains = list()

    # Take the colname column of df, strip out the nans (which break urlparser) and add it to urls
    urls = dataframe[colname].dropna()

    logger.info('This many URLs found: ' + str(len(urls)))

    # User urlparse() to strip out the rootdomain (i.e, netloc) and write it to a list
    for i in urls:
        current_url = urlparse(i)
        list_of_rootdomains.append(current_url.netloc)

    # Convert the list into a df so we can use the same functions as are being used to summarise other data
    dataframeurl = pd.DataFrame({'rootdomains': list_of_rootdomains})

    logger.info('This many rootdomains found: ' + str(len(dataframeurl)))

    logger.info('This many unique rootdomains found: ' + str(len(dataframeurl.rootdomains.unique())))

    return dataframeurl

def plot_bar_charts(dataframe,filename,title,xaxis,yaxis,truncate):
    """
    Takes a two-column dataframe and plots it
    :params: a dataframe with two columns (one labels, the other a count), a filename for the resulting chart, a title, and titles for the
    two axes (if title is None, then nothing is plotted), and a truncate variable which cuts down the number of
    rows plotted (unless it's 0 at which point all rows are plotted)
    :return: Nothing, just prints a chart
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Plotting charts...')

    if truncate > 0:
        # This cuts the dataframe down to the number of rows given in truncate
        dataframe = dataframe.ix[:truncate]

    dataframe.plot(kind='bar', legend=None)
    plt.title(title)
    if xaxis != None:
        plt.xlabel(xaxis)
    if yaxis != None:
        plt.ylabel(yaxis)
    # This provides more space around the chart to make it prettier
    plt.tight_layout(True)
    plt.savefig(CHART_STORE_DIR + filename + '.png', format = 'png', dpi = 150)
    plt.show()
    return

def impact_to_txt(dataframe,colname):
    """
    Takes a dataframe, collates all content in the colname column and writes it to a textfile
    :params: a dataframe and a column (colname) containing impact statements
    :return: write a text file containing all impact statements combined
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Creating impact text file...')

    # Don't want any of the NaNs, so drop them
    impact_dataframe = dataframe.dropna(subset=[colname])
    # Open file for writing
    file_for_impacts = open(IMPACT_RESULT_STORE, 'w')
    # Go through dataframe row by row and write the text from the colname column to as a separate line to the text file
    for i, row in impact_dataframe.iterrows():
        file_for_impacts.write("%s\n" % impact_dataframe[colname][i])
    return

def check_url_status(dataframe, colname, statuscol):
    """
    Takes a dataframe and a column in that dataframe that contains URLs. Pings the URLs to see if they throw
    an exception, then writes the result to another column of the dataframe
    :params: a dataframe, a column (colname) in which URLs are stored and a column (statuscol) in which the
    URL status will be recorded
    :return: a dataframe with the status of the URLs recorded in a column
    """
    # Set up logging
    logger = logging.getLogger(__name__)
    logger.info('Checking URLs...')

    # Don't want any of the NaNs, so drop the rows in which NaN was entered for the URL
    dataframe.dropna(subset=[colname], inplace=True)

    h = httplib2.Http()
    for i, row in dataframe.iterrows():
        try:
            print("Checking " + dataframe[colname][i])
            response, content = h.request(dataframe[colname][i])
            if response.status < 400:
                dataframe[statuscol][i] = response['date']
        except:
            dataframe[statuscol][i] = 'No response'

    #When the URL checker times out without an error, it skips to the next URL and leave a blank in the statuscol
    # which I want replaced with a 'No response' to making counting easier
    dataframe[statuscol] = dataframe[statuscol].fillna('No response')

    #Record no. of dead sites
    dead_sites = len(dataframe[dataframe[statuscol] == 'No response'])

    #Log how many dead URLs were found
    logger.info('This many dead URLs found: ' + str(dead_sites))

    return dataframe

def main():
    """
    Main function to run program
    """

    # Set up logging
    logging.basicConfig(level=logging.DEBUG)
    logger = logging.getLogger(__name__)
    handler = logging.FileHandler(LOGGERLOCATION)
    handler.setLevel(logging.INFO)
    # create a logging format
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(handler)

    logger.info('Starting...')

    # I write back to the original dataframe and pandas warns about that, so turning off the warning
    pd.options.mode.chained_assignment = None

    # Import dataframe from original xls
    df = import_xls_to_df(DATAFILENAME, DATASHEETNAME)

    print(len(df))

    logger.info('Raw dataframe length before any processing: ' + repr(len(df)))

    # Add a column for URL pinging response
    add_column(df,'URL status')

    # Clean the dataframe
    df = clean_data(df,'Year First Provided')

    # Get a list of rootdomains (i.e. netloc) of URLs
    rootdomainsdf = get_root_domains(df,'URL')

    # Adds data into df about status of the URL at which software is stored
    url_check = check_url_status(df,'URL','URL status')
    url_df = pd.concat([url_check['URL'], url_check['URL status']], axis=1, keys=['URL', 'URL status'])

    # Count the unique values in columns to get summaries of open/closed/no licence, which university released outputs, where outputs are being stored and in which year outputs were recorded
    open_source_licence = produce_count_and_na(df,'Open Source?')
    open_source_licence.index = open_source_licence.index.fillna('No response')
    universities = produce_count_and_na(df,'RO')
    unique_rootdomains = produce_count_and_na(rootdomainsdf,'rootdomains')
    year_of_return = produce_count(df,'Year First Provided')
    url_status = produce_count(df,'URL status')

    # Want this to be sorted in year order rather than in order of largest count
    year_of_return.sort_index(inplace = True)

    # Collate all impact statements into a text file for later word cloud generation
    impact_to_txt(df,'Impact')

    # Plot results and save charts
    plot_bar_charts(open_source_licence,'opensource','Is the output under an open-source licence?',None,'No. of outputs',0)
    plot_bar_charts(universities,'universities','Top 30 universities that register the most outputs',None,'No. of outputs',30)
    plot_bar_charts(unique_rootdomains,'rootdomain','30 most popular domains for storing outputs',None,'No. of outputs',30)
    plot_bar_charts(year_of_return,'returnyear','When was output first registered?',None,'No. of outputs',0)

    # Write results to Excel spreadsheet for the shear hell of it
    writer = ExcelWriter(EXCEL_RESULT_STORE)
    open_source_licence.to_excel(writer,'opensource')
    universities.to_excel(writer,'universities')
    unique_rootdomains.to_excel(writer,'rootdomain')
    year_of_return.to_excel(writer,'returnyear')
    url_df.to_excel(writer,'urlstatus')
    url_status.to_excel(writer,'urlstatus_summ')
    df.to_excel(writer,'Resulting_df')
    writer.save()

if __name__ == '__main__':
    main()


text:
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer as CV
import string
exclude = set(string.punctuation)

def basic_sanitize(in_string):
    '''Returns a very roughly sanitized version of the input string.'''
    return_string = ' '.join(in_string.encode('ascii', 'ignore').strip().split())
    return_string = ''.join(ch for ch in return_string if ch not in exclude)
    return_string = return_string.lower()
    return_string = ' '.join(return_string.split())
    return return_string

def bayes_compare_language(l1, l2, ngram = 1, prior=.01, cv = None):
    '''
    Arguments:
    - l1, l2; a list of strings from each language sample
    - ngram; an int describing up to what n gram you want to consider (1 is unigrams,
    2 is bigrams + unigrams, etc). Ignored if a custom CountVectorizer is passed.
    - prior; either a float describing a uniform prior, or a vector describing a prior
    over vocabulary items. If you're using a predefined vocabulary, make sure to specify that
    when you make your CountVectorizer object.
    - cv; a sklearn.feature_extraction.text.CountVectorizer object, if desired.

    Returns:
    - A list of length |Vocab| where each entry is a (n-gram, zscore) tuple.'''
    if cv is None and type(prior) is not float:
        print "If using a non-uniform prior:"
        print "Please also pass a count vectorizer with the vocabulary parameter set."
        quit()
    l1 = [basic_sanitize(l) for l in l1]
    l2 = [basic_sanitize(l) for l in l2]
    if cv is None:
        cv = CV(decode_error = 'ignore', min_df = 10, max_df = .5, ngram_range=(1,ngram),
                binary = False,
                max_features = 15000)
    counts_mat = cv.fit_transform(l1+l2).toarray()
    # Now sum over languages...
    vocab_size = len(cv.vocabulary_)
    print "Vocab size is {}".format(vocab_size)
    if type(prior) is float:
        priors = np.array([prior for i in range(vocab_size)])
    else:
        priors = prior
    z_scores = np.empty(priors.shape[0])
    count_matrix = np.empty([2, vocab_size], dtype=np.float32)
    count_matrix[0, :] = np.sum(counts_mat[:len(l1), :], axis = 0)
    count_matrix[1, :] = np.sum(counts_mat[len(l1):, :], axis = 0)
    a0 = np.sum(priors)
    n1 = 1.*np.sum(count_matrix[0,:])
    n2 = 1.*np.sum(count_matrix[1,:])
    print "Comparing language..."
    for i in range(vocab_size):
        #compute delta
        term1 = np.log((count_matrix[0,i] + priors[i])/(n1 + a0 - count_matrix[0,i] - priors[i]))
        term2 = np.log((count_matrix[1,i] + priors[i])/(n2 + a0 - count_matrix[1,i] - priors[i]))
        delta = term1 - term2
        #compute variance on delta
        var = 1./(count_matrix[0,i] + priors[i]) + 1./(count_matrix[1,i] + priors[i])
        #store final score
        z_scores[i] = delta/np.sqrt(var)
    index_to_term = {v:k for k,v in cv.vocabulary_.iteritems()}
    sorted_indices = np.argsort(z_scores)
    return_list = []
    for i in sorted_indices:
        return_list.append((index_to_term[i], z_scores[i]))
    return return_list

max_stars_repo_name: jmhessel/FIghtingWords

max_issues_repo_name: jmhessel/FIghtingWords

max_forks_repo_name: jmhessel/FIghtingWords


text:
### simple tools for randomization
import random
import numpy as np
import research_toolbox.tb_utils as tb_ut

def shuffle(xs):
    idxs = list(range(len(xs)))
    random.shuffle(idxs)
    return [xs[i] for i in idxs]

def random_permutation(n):
    idxs = list(range(n))
    random.shuffle(idxs)
    return idxs

def argsort(xs, fns, increasing=True):
    """The functions in fns are used to compute a key which are then used to
    construct a tuple which is then used to sort. The earlier keys are more
    important than the later ones.
    """

    def key_fn(x):
        return tuple([f(x) for f in fns])

    idxs, _ = tb_ut.zip_toggle(
        sorted(enumerate(xs),
               key=lambda x: key_fn(x[1]),
               reverse=not increasing))
    return idxs

def sort(xs, fns, increasing=True):
    idxs = argsort(xs, fns, increasing)
    return apply_permutation(xs, idxs)

def apply_permutation(xs, idxs):
    assert len(set(idxs).intersection(list(range(len(xs))))) == len(xs)
    return [xs[i] for i in idxs]

def apply_inverse_permutation(xs, idxs):
    assert len(set(idxs).intersection(list(range(len(xs))))) == len(xs)

    out_xs = [None] * len(xs)
    for i_from, i_to in enumerate(idxs):
        out_xs[i_to] = xs[i_from]
    return out_xs

def shuffle_tied(xs_lst):
    assert len(xs_lst) > 0 and len(list(map(len, xs_lst))) == 1

    n = len(xs_lst[0])
    idxs = random_permutation(n)
    ys_lst = [apply_permutation(xs, idxs) for xs in xs_lst]
    return ys_lst

def set_random_seed(x=None):
    np.random.seed(x)
    random.seed(x)

def uniform_sample_product(lst_lst_vals, num_samples):
    n = len(lst_lst_vals)
    components = []
    for i, lst in enumerate(lst_lst_vals):
        idxs = np.random.randint(len(lst), size=num_samples)
        components.append([lst[j] for j in idxs])

    samples = []
    for i in range(num_samples):
        samples.append([components[j][i] for j in range(n)])
    return samples

def uniformly_random_indices(num_indices, num_samples, with_replacement):
    return np.random.choice(num_indices,
                            num_samples,
                            replace=with_replacement,
                            p=probs)

def random_indices_with_probs(num_indices, num_samples, with_replacement,
                              probs):
    return np.random.choice(num_indices,
                            num_samples,
                            replace=with_replacement,
                            p=probs)


text:
from abc import abstractmethod

from kernel_exp_family.estimators.estimator_oop import EstimatorBase
try:
    from kernel_exp_family.estimators.parameter_search_bo import BayesOptSearch
except ImportError:
    print("Could not import BayesOptSearch.")
from kernel_exp_family.kernels.kernels import gaussian_kernel, \
    gaussian_kernel_grad, theano_available
from kernel_exp_family.tools.assertions import assert_array_shape
from kernel_exp_family.tools.log import Log
import numpy as np

if theano_available:
    from kernel_exp_family.kernels.kernels import gaussian_kernel_hessian_theano, \
        gaussian_kernel_third_order_derivative_tensor_theano

logger = Log.get_logger()

def compute_b(X, Y, K_XY, sigma):
    assert X.shape[1] == Y.shape[1]
    assert K_XY.shape[0] == X.shape[0]
    assert K_XY.shape[1] == Y.shape[0]

    NX = len(X)
    D = X.shape[1]

    b = np.zeros(NX)
    K1 = np.sum(K_XY, 1)
    for l in np.arange(D):
        x_l = X[:, l]
        y_l = Y[:, l]

        s_l = x_l ** 2
        t_l = y_l ** 2

        # Replaces dot product with np.diag via broadcasting
        # See http://mail.scipy.org/pipermail/numpy-discussion/2007-March/026809.html
        D_s_K = s_l[:, np.newaxis] * K_XY
        D_x_K = x_l[:, np.newaxis] * K_XY

        b += 2. / sigma * (K_XY.dot(t_l) \
                        + np.sum(D_s_K, 1) \
                        - 2 * D_x_K.dot(y_l)) - K1

    return b

def compute_C(X, Y, K, sigma):
    assert X.shape[1] == Y.shape[1]
    assert K.shape[0] == X.shape[0]
    assert K.shape[1] == Y.shape[0]

    D = X.shape[1]
    NX = X.shape[0]

    C = np.zeros((NX, NX))
    for l in np.arange(D):
        x_l = X[:, l]
        y_l = Y[:, l]

        # Replaces dot product with np.diag via broadcasting
        # See http://mail.scipy.org/pipermail/numpy-discussion/2007-March/026809.html
        D_x_KXY = x_l[:, np.newaxis] * K
        KXY_D_y = K * y_l
        KXY_T_D_x = K.T * x_l
        D_y_KXY_T = y_l[:, np.newaxis] * K.T

        C += (D_x_KXY - KXY_D_y).dot(KXY_T_D_x - D_y_KXY_T)

    return C

def fit(X, Y, sigma, lmbda, K=None, reg_f_norm=True, reg_alpha_norm=True):
        # compute kernel matrix if needed
        if K is None:
            K = gaussian_kernel(X, Y, sigma=sigma)

        b = compute_b(X, Y, K, sigma)
        C = compute_C(X, Y, K, sigma)

        reg_mat = np.zeros(np.shape(K))
        if reg_f_norm:
            reg_mat += K

        if reg_alpha_norm:
            reg_mat += np.eye(len(K))

        if (reg_f_norm or reg_alpha_norm) and (lmbda>0):
            C += reg_mat * lmbda

        # solve (potentially regularised) linear system
        a = -sigma / 2. * np.linalg.solve(C, b)

        return a

def objective(X, Y, sigma, lmbda, alpha, K=None, K_XY=None, b=None, C=None):
    if K_XY is None:
        K_XY = gaussian_kernel(X, Y, sigma=sigma)

    if K is None and lmbda > 0:
        if X is Y:
            K = K_XY
        else:
            K = gaussian_kernel(X, sigma=sigma)

    if b is None:
        b = compute_b(X, Y, K_XY, sigma)

    if C is None:
        C = compute_C(X, Y, K_XY, sigma)

    NX = len(X)
    first = 2. / (NX * sigma) * alpha.dot(b)
    second = 2. / (NX * sigma ** 2) * alpha.dot((C).dot(alpha))
    J = first + second
    return J

class KernelExpLiteGaussian(EstimatorBase):
    def __init__(self, sigma, lmbda, D, N,
                 reg_f_norm=True, reg_alpha_norm=True):

        assert lmbda>0
        assert sigma>0

        self.sigma = sigma
        self.lmbda = lmbda
        self.D = D
        self.N = N
        self.reg_f_norm = reg_f_norm
        self.reg_alpha_norm = reg_alpha_norm

        # initial RKHS function is flat
        self.alpha = np.zeros(0)
        self.X = np.zeros((0, D))

    def fit(self, X):
        assert_array_shape(X, ndim=2, dims={1: self.D})

        # sub-sample if data is larger than previously set N
        if len(X) > self.N:
            logger.info("Sub-sampling %d/%d data." % (self.N, len(X)))
            inds = np.random.permutation(len(X))[:self.N]
            self.X = X[inds]
        else:
            self.X = np.copy(X)

        self.alpha = self.fit_wrapper_()

    @abstractmethod
    def fit_wrapper_(self):
        self.K = gaussian_kernel(self.X, sigma=self.sigma)
        return fit(X=self.X, Y=self.X, sigma=self.sigma, lmbda=self.lmbda,
                   K=self.K,
                   reg_f_norm=self.reg_f_norm, reg_alpha_norm=self.reg_alpha_norm)

    def log_pdf(self, x):
        assert_array_shape(x, ndim=1, dims={0: self.D})

        k = gaussian_kernel(self.X, x.reshape(1, self.D), self.sigma)[:, 0]
        return np.dot(self.alpha, k)

    def grad(self, x):
        assert_array_shape(x, ndim=1, dims={0: self.D})

        k = gaussian_kernel_grad(x, self.X, self.sigma)
        return np.dot(self.alpha, k)

    if theano_available:
        def hessian(self, x):
            """
            Computes the Hessian of the learned log-density function.

            WARNING: This implementation slow, so don't call repeatedly.
            """
            assert_array_shape(x, ndim=1, dims={0: self.D})

            H = np.zeros((self.D, self.D))
            for i, a in enumerate(self.alpha):
                H += a * gaussian_kernel_hessian_theano(x, self.X[i], self.sigma)

            return H

        def third_order_derivative_tensor(self, x):
            """
            Computes the third order derivative tensor of the learned log-density function.

            WARNING: This implementation is slow, so don't call repeatedly.
            """
            assert_array_shape(x, ndim=1, dims={0: self.D})

            G3 = np.zeros((self.D, self.D, self.D))
            for i, a in enumerate(self.alpha):
                G3 += a * gaussian_kernel_third_order_derivative_tensor_theano(x, self.X[i], self.sigma)

            return G3

    def log_pdf_multiple(self, X):
        assert_array_shape(X, ndim=2, dims={1: self.D})

        k = gaussian_kernel(self.X, X, self.sigma)
        return np.dot(self.alpha, k)

    def objective(self, X):
        assert_array_shape(X, ndim=2, dims={1: self.D})

        return objective(self.X, X, self.sigma, self.lmbda, self.alpha, self.K)

    def get_parameter_names(self):
        return ['sigma', 'lmbda']

class KernelExpLiteGaussianAdaptive(KernelExpLiteGaussian):
    def __init__(self, sigma, lmbda, D, N,
                 num_initial_evaluations=3, num_evaluations=3, minimum_size_learning=100,
                 num_initial_evaluations_relearn=1, num_evaluations_relearn=1,
                 param_bounds={'sigma': [-3, 3]}):
        KernelExpLiteGaussian.__init__(self, sigma, lmbda, D, N)

        self.bo = None
        self.param_bounds = param_bounds
        self.num_initial_evaluations = num_initial_evaluations
        self.num_iter = num_evaluations
        self.minimum_size_learning = minimum_size_learning

        self.n_initial_relearn = num_initial_evaluations_relearn
        self.n_iter_relearn = num_evaluations_relearn

        self.learning_parameters = False

    def fit(self, X):
        # avoid infinite recursion from x-validation fit call
        if not self.learning_parameters and len(X) >= self.minimum_size_learning:
            self.learning_parameters = True
            if self.bo is None:
                logger.info("Bayesian optimisation from scratch.")
                self.bo = BayesOptSearch(self, X, self.param_bounds, num_initial_evaluations=self.num_initial_evaluations)
                best_params = self.bo.optimize(self.num_iter)
            else:
                logger.info("Bayesian optimisation using prior model.")
                self.bo.re_initialise(X, self.n_initial_relearn)
                best_params = self.bo.optimize(self.n_iter_relearn)

            self.set_parameters_from_dict(best_params)
            self.learning_parameters = False
            logger.info("Learnt %s" % str(self.get_parameters()))

        # standard fit call from superclass
        KernelExpLiteGaussian.fit(self, X)


text:
import os
import Config
import datetime
import tensorflow as tf
import sys
import dataset
import pandas as pd
import numpy as np

sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
from mrcnn import visualize

class Trainer:
	def __init__(self):
		self.config = None
		self.trainDataset = None
		self.testDataset = None

	def train(self):
		# load data
		train_iterator = self.trainDataset.make_initializable_iterator()
		next_element_train = train_iterator.get_next()

		label_df = pd.read_csv("./label.csv")

		# training cycle
		with tf.Session() as sess:
			# Initialize all variables
			sess.run(tf.global_variables_initializer())
			print("{}: Start training...".format(datetime.datetime.now()))

			for epoch in range(self.config.getConfigValue("EPOCHS")):
				# initialize iterator in each new epoch
				sess.run(train_iterator.initializer)

				print("{}: Epoch {} starts".format(datetime.datetime.now(),epoch+1))

				# Training phase
				while True:
					try:
						[filename, image, findings, bboxes] = sess.run(next_element_train)

						# print(findings, bboxes)
						# print(findings.shape, bboxes.shape)
						# exit()

						findings_ = []
						for finding in findings:
							findings_.append(dataset.label_to_name(label_df,finding[0,...]))

						visualize.draw_boxes(image[0,...],boxes=bboxes[0,...],captions=findings_)

						# print(findings)
						# print(bboxes)

						# findings_.append("test")
						# bboxes = np.append(bboxes,[[[0,0,256,512]]],axis=1)
						# visualize.draw_boxes(image[0,...],boxes=bboxes[0,...],captions=findings_, visibilities=[1,3])

					except tf.errors.OutOfRangeError:
						break

			return


text:
from __future__ import annotations
from typing import Any, Union, List

import numpy as np
from nptyping import NDArray

from .core.ElementInGFpn import ElementInGFpn
from .core import types, modulus, validator, primitive_roots as pr

class GFpn:
    """Galois Field: GF(p^n).

    Args:
        p (int): A prime number.
        mod_coeffs (Union[List[int], NDArray[Any, int]]):
            Coefficients of a monic irreducible polynomial.

    Examples:
        >>> from galois_field import GFpn

        In this case, p = 5, a monic irreducible polynomial = x^4 + 2.
        >>> gf = GFpn(5, [1, 0, 0, 0, 2])

        Generate the element in GF(5^4).
        >>> gf.elm([1, 2]) # 1x + 2
        ElementInGFpn([1, 2], 5, [1, 0, 0, 0, 2])

        >>> gf.elm([1, 2, 3, 4, 5]) # 2x^3 + 3x^2 + 4x + 3
        ElementInGFpn([2, 3, 4, 3], 5, [1, 0, 0, 0, 2])
    """

    def __init__(self, p: int,
                 mod_coeffs: Union[List[int], NDArray[Any, int]]):
        self.__p = p

        modulus_mod_coeffs = modulus.modulus_coeffs(np.array(mod_coeffs), p)
        self.__mod_poly = np.poly1d(modulus_mod_coeffs)

    @property
    def p(self) -> int:
        """A prime number. Read-only."""
        return self.__p

    @property
    def mod_coeffs(self) -> types.Fpn:
        """Coefficients of the monic irreducible polynomial. Read-only."""

        return self.__mod_poly.coeffs

    @property
    def mod_poly(self) -> np.poly1d:
        """The monic irreducible polynomial. Read-only."""
        return self.__mod_poly

    def __str__(self) -> str:
        return f'GF({self.p}^{len(self.mod_coeffs) - 1})'

    def is_valid(self) -> bool:
        """Determine if this field is valid.

        Returns:
            bool: Is valid ?
        """
        return validator.is_prime(self.p) and \
            validator.is_irreducible_poly(self.mod_poly, self.p)

    def elm(self, coeffs: Union[NDArray[Any, int], List[int]]) -> ElementInGFpn:
        """Generate the Element from coeffs in GF(p^n).

        Returns:
            ElementInGFpn: The element in GF(p^n).
        """
        return ElementInGFpn(np.array(coeffs), self.p, self.mod_poly)

    def random_primitive_elm(self) -> ElementInGFpn:
        """Return a primitive element in GF(p^n) randomly.

        Returns:
            ElementInGFpn: A primitive root in GF(p^n)
        """
        return pr.random_primitive_root_over_Fpn(self.__p, self.mod_coeffs)

max_stars_repo_name: syakoo/galois-field

max_issues_repo_name: syakoo/galois-field

max_forks_repo_name: syakoo/galois-field


text:
import cv2
import numpy as np
from PIL import Image
import os

path = "./photo/myobject"

filelist = os.listdir(path)
total_num = len(filelist)

n = 6
for i in range(1,total_num):
    n = 6 - len(str(i))
    filepath = "./photo/myobject/"+str(0)*n + str(i)+".jpg"
    imgsize = Image.open(filepath)#開啟圖片
    print('開啟檔案：' + filepath)
    for a in range(1,101):
        for b in range(1,101):
            print('round %s_%s_%s' % (str(i),str(a),str(b)))
            img = cv2.imread(filepath)
            mask = np.zeros(img.shape[:2],np.uint8)
            bgdModel = np.zeros((1,65),np.float64)
            fgdModel = np.zeros((1,65),np.float64)
            rect = (a,b,imgsize.size[0],imgsize.size[1])
            cv2.grabCut(img,mask,rect,bgdModel,fgdModel,5,cv2.GC_INIT_WITH_RECT)
            mask2 = np.where((mask==2)|(mask==0),0,1).astype('uint8')
            img = img*mask2[:,:,np.newaxis]
            print('輸出檔案：%s_%s_%s.jpg' %(str(0)*n + str(i),str(a),str(b)))
            cv2.imwrite('./save/%s_%s_%s.jpg' %(str(0)*n + str(i),str(a),str(b)),img)


text:
from typing import Optional, Sequence, Tuple

import numpy
from numpy.random import rand
from scipy.ndimage import gaussian_filter

from aydin.features.groups.correlation import CorrelationFeatures

class RandomFeatures(CorrelationFeatures):
    """
    Random Feature Group class

    Generates features by convolving the image with a random set of filters.
    """

    def __init__(self, size: int, num_features: Optional[int] = None):
        """
        Constructor that configures these features.

        Parameters
        ----------

        size : int
            Size of the kernels used.

        num_features : Optional[int]
            Number of features to generate. If None the maximum number of features
            that may be linearly independent is used.
        """
        super().__init__(kernels=None)
        self.size = size
        self._num_features = num_features

        self.image = None
        self.exclude_center: Sequence[Tuple[int, ...]] = []

    def _ensure_random_kernels_available(self, ndim: int):
        # Ensures that the kernels are available for subsequent steps.
        # We can't construct the kernels until we know the dimension of the image
        if self.kernels is None or self.kernels[0].ndim != ndim:
            rnd_kernels = []

            num_features = (
                self.size**ndim if self._num_features is None else self._num_features
            )

            shape = tuple((self.size,) * ndim)

            for k in range(num_features):
                numpy.random.seed(k)
                kernel = numpy.where(rand(*shape) > 0.5, 1.0, 0.0)
                kernel = gaussian_filter(kernel, sigma=0.5)
                kernel /= kernel.sum()
                kernel = kernel.astype(numpy.float32, copy=False)

                # import napari
                # with napari.gui_qt():
                #      from napari import Viewer
                #      viewer = Viewer()
                #      viewer.add_image(kernel, name='kernel')

                rnd_kernels.append(kernel)

            self.kernels = rnd_kernels

    @property
    def receptive_field_radius(self) -> int:
        return self.size // 2

    def num_features(self, ndim: int) -> int:
        self._ensure_random_kernels_available(ndim)
        return super().num_features(ndim)

    def prepare(self, image, excluded_voxels=None, **kwargs):
        if excluded_voxels is None:
            excluded_voxels = []
        self._ensure_random_kernels_available(image.ndim)
        super().prepare(image, excluded_voxels, **kwargs)

max_stars_repo_name: AhmetCanSolak/aydin

max_issues_repo_name: AhmetCanSolak/aydin

max_forks_repo_name: AhmetCanSolak/aydin

reason: import numpy,from numpy,from scipy


text:
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, accuracy_score
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from ccgnet.Dataset import Dataset, DataLoader
import numpy as np
from hyperopt import fmin, tpe, Trials, hp, STATUS_OK
import hyperopt.pyll.stochastic
import random

fold_10 = eval(open('Fold_10.dir').read())
def make_dataset():
    data1 = Dataset('CC_Table/CC_Table.tab', mol_blocks_dir='Mol_Blocks.dir')
    data1.make_graph_dataset(Desc=1, A_type='OnlyCovalentBond', hbond=0, pipi_stack=0, contact=0, make_dataframe=True)
    return data1

def black_box_function(args_dict):
    clf = RandomForestClassifier(**args_dict, n_jobs=8)
    clf.fit(x_train, y_train)
    valid_pred_score = clf.predict_proba(x_valid)
    valid_pred_labels = [np.argmax(i) for i in valid_pred_score]
    valid_acc = accuracy_score(y_valid, valid_pred_labels)
    print((str(valid_acc)+': {}').format(args_dict))
    return {'loss': 1-valid_acc, 'status': STATUS_OK}

space4rf = {
            'max_depth': hp.choice('max_depth', range(1, 21)),
            'max_features': hp.choice('max_features', range(1,24)),
            'n_estimators': hp.choice('n_estimators', list(range(10, 501, 10))),
            'criterion': hp.choice('criterion', ["gini", "entropy"]),
            'min_samples_leaf': hp.choice('min_samples_leaf', list(range(1, 50))),
            'min_samples_split': hp.choice('min_samples_split', list(range(2, 201))),
            'bootstrap': hp.choice('bootstrap', [0, 1])
           }

Samples = fold_10['fold-0']['train'] + fold_10['fold-0']['valid']
random.seed(10)
random.shuffle(Samples)
num_sample = len(Samples)
train_num = int(0.9 * num_sample)
train_samples = Samples[:train_num]
valid_samples = Samples[train_num:]

data = make_dataset()

train_data, valid_data = data.split(train_samples=train_samples, valid_samples=valid_samples)
x_train, y_train = train_data[-2], train_data[2]
x_valid, y_valid = valid_data[-2], valid_data[2]

trials = Trials()
best = fmin(black_box_function, space4rf, algo=tpe.suggest, max_evals=100, trials=trials)
print(best)


text:
import cv2
import numpy as np

file_name = "testm1.jpg"

src = cv2.imread("../thresh_inv.png", 1)
tmp = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)
_,alpha = cv2.threshold(tmp,0,255,cv2.THRESH_BINARY)
b, g, r = cv2.split(src)
rgba = [b,g,r, alpha]
dst = cv2.merge(rgba,4)
cv2.imwrite("testm1.png", dst)


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Nov 22 00:33:23 2019

@author: Sampritha H M
"""

import numpy as np
import os
import sys
import Utils.common_utilities as utils

os.chdir("../")
os.chdir("data")
file = sys.argv[1]
train_input,train_output,test_input,test_output = utils.json_reader(file)

def solve(inputData):
    array = inputData
    print("\n Input: \n",array)
    output = []
    for i in range(len(array)):
        arr = np.array(array[i])
        for j in range(len(arr)):
            if arr[j] != 0:
                output.append(arr[j])
    return output

print("\n\nTRANING")
for i in range(len(train_input)):
    result = []
    inputData = train_input[i]
    result.append(solve(inputData))
    print(" Output:\n",result)
    if(train_output[i] == result):
        print("Training Successfull for training input ", i)

print("\n\nTESTING")

output = []
output.append(solve(test_input[0]))
print(" Output:\n",output)
if(test_output[0] == output):
    print(" Testing Successfull")

max_stars_repo_name: samprithahm/ARC

max_issues_repo_name: samprithahm/ARC

max_forks_repo_name: samprithahm/ARC


text:
#!/usr/bin/env python

import numpy as np
from swignifit.swignifit_raw import vector_double
from pypsignifit import psignidata

__doc__ = """This module is intended to perform bayesian model comparison using model posterior probabilities

The rationale is the following:
Imaginge a number of candidate psychometric function models M1,M2,...Mn. We proceed by treating
M as another model parameter and perform gibbs sampling in the parameter space of this metamodel.
The function gibbs_model will then give the marginal posterior distribution in model space using gibbs sampling.
Interestingly, for this particular gibbs sampler, we can write out an analytical expression for the marginal stationary
distribution with respect to M. The function gibbs_analytical determines this marginal distribution.

In some applications, we might be interested in fitting more than one psychometric function in each
of the models to be compared. This is for example relevant if the feature that distinguishes between
the different models is e prior distribution. In this case, we can have each model consist of a sequence
of PsiInference objects.

Note that the logic of this module only works for Bayesian inference and *not* for a Bootstrap setting.
Also be aware that the data that were used for the different models should be the same across M1,M2,...,Mn.
"""

__all__ = ["gibbs_model","gibbs_analytical"]

def sample_model ( theta, inference_objects ):
    """sample_model ( theta, inference_objects )

    Sample a model from the full conditional f(M|theta)

    :Parameters:
    *theta* :
        list of arrays of parameters
    *inference_objects* :
        a sequence of inference objects to be sampled from.
    """
    p = np.zeros ( len(inference_objects), 'd' )
    th = [ vector_double ( th_ ) for th_ in theta ]
    for i,model in enumerate ( inference_objects ):
        if isinstance ( model, psignidata.PsiInference ):
            p[i] = -model._pmf.neglpost ( th[0], model._data )
        elif getattr ( inference_objects, "__iter__", False ):
            for M,th_ in zip ( model, th ):
                p[i] -= M._pmf.neglpost ( th_, M._data )
    p -= p.mean() # This should stabilize the exponential but should otherwise not change the result (constant factor)
    p = np.exp ( p )
    p /= np.sum ( p )
    return int ( np.where ( np.random.multinomial ( 1, p ) )[0] )

def sample_parameter ( inference_object ):
    """sample_parameter ( inference_object )

    reSample a single parameter setting from the model posterior.
    This is done by randomly selecting one of the samples stored in
    the inference object.

    :Parameters:
    *inference_object* :
        an inference object from which a parameter sample should be taken
    """
    if isinstance ( inference_object, psignidata.PsiInference ):
        i = np.random.randint ( len(inference_object.mcdeviance) )
        return [inference_object.mcestimates[i,:]]
    elif getattr ( inference_object, "__iter__", False ):
        out = []
        for model in inference_object:
            i = np.random.randint ( len(model.mcdeviance) )
            out.append ( model.mcestimates[i,:] )
        return out
    else:
        raise ValueError, "Incorrect input"

def gibbs_model ( inference_objects, nsamples=1000, M0=0 ):
    """gibbs_model ( inference_objects, nsamples=1000 )

    Perform Gibbs sampling in the full model P(M,theta).
    This function will return the marginal distribution across models.

    :Parameters:
    *inference_objects* :
        psychometric function models to be considered. Note that these inference objects
        have to be based on the same data set in order to obtain valid results.
        At the moment this is *not* checked!
    *nsamples* :
        how many samples are to be generated
    *M0* :
        Model to start with
    """

    mpost = np.zeros ( len(inference_objects), 'd' )
    chain = np.zeros ( nsamples )

    M = M0
    for i in xrange ( nsamples ):
        th = sample_parameter ( inference_objects[M] )
        M  = sample_model ( th, inference_objects )
        mpost[M] += 1
        chain[i] = M

    return mpost,chain

def gibbs_analytical ( inference_objects ):
    """gibbs_analytical ( inference_objects )

    Analytically determine the marginal stationary distribution of M in the
    full model P(M,theta).

    :Parameters:
        *inference_objects* :
            psychometric function models to be compared. Note that these inference objects
            have to be based on the same data set in order to obtain valid results.
    """

    n = len ( inference_objects )
    T = np.zeros ( (n,n), 'd' )

    # Set up the transition matrix
    for k,M in enumerate ( inference_objects ):
        if isinstance ( M, psignidata.PsiInference ):
            M = [M]
        elif getattr ( M, "__iter__", False ):
            pass
        else:
            raise ValueError, "Elements of inference_objects should be either a PsiInference object or a sequence"

        for j in xrange ( M[0].mcestimates.shape[0] ):
            th = [ m.mcestimates[j] for m in M ]
            p =  np.exp ( np.array ( [ eval_post ( th, M_l ) for M_l in inference_objects ] ) )
            p /= p.sum()
            T[k] += p
        T[k] /= T[k].sum()

    # Now solve the stochastic matrix equation pi*T=pi
    A = (T-np.eye(n)).T
    A[0,:] = 1.
    p = np.zeros ( n, 'd' )
    p[0] = 1.

    return np.linalg.solve ( A, p ),T

def eval_post ( theta, M ):
    if isinstance ( M, psignidata.PsiInference ):
        M = [M]
    elif getattr ( M, "__iter__", False ):
        pass
    else:
        raise ValueError, "Elements of inference_objects should be either a PsiInference object or a sequence"

    return reduce ( lambda x,y: x+y, [ -m._pmf.neglpost ( th, m._data ) for th,m in zip(theta,M) ] )


text:
import unittest
from bs4 import BeautifulSoup
import requests
import pandas as pd
from pandas.testing import assert_frame_equal, assert_series_equal
import sys
sys.path.append('..')
from scripts.ProFootballReference import player_href_list_grab , player_id_transform, player_soup_grab, player_name_and_pos_grab, player_table_grab, player_table_transform, player_table_variable_add, player_table_reindex
import scripts.config
import numpy as np

class TestProFootballReference(unittest.TestCase):
    def test_player_href_list_grab(self):
        year = 2019
        expected = [
            '/players/M/McCaCh01.htm',
            '/players/J/JackLa00.htm',
            '/players/H/HenrDe00.htm',
            '/players/J/JoneAa00.htm',
            '/players/E/ElliEz00.htm',
            '/players/C/CookDa01.htm'
        ]
        self.assertListEqual(expected, player_href_list_grab(year)[:6])

    def test_player_id_transform(self):
        string = '/players/M/McCaCh01.htm'
        expected = 'M', 'McCaCh01'
        self.assertEqual(expected, player_id_transform(string))

    def test_player_soup_grab(self):
        last_name_letter = 'M'
        player_id = 'McCaCh01'
        year = 2019
        self.assertIsInstance(player_soup_grab(last_name_letter, player_id, year), BeautifulSoup)

    def test_player_table_transform(self):
        df = pd.DataFrame(
            {('Unnamed: 0_level_0', 'Rk'): {0: 1.0, 1: 2.0},
            ('Unnamed: 1_level_0', 'Date'): {0: '2019-09-08', 1: '2019-09-12'},
            ('Unnamed: 2_level_0', 'G#'): {0: 1.0, 1: 2.0},
            ('Unnamed: 3_level_0', 'Week'): {0: 1.0, 1: 2.0},
            ('Unnamed: 4_level_0', 'Age'): {0: 23.093000000000004, 1: 23.096999999999998},
            ('Unnamed: 5_level_0', 'Tm'): {0: 'CAR', 1: 'CAR'},
            ('Unnamed: 6_level_0', 'Unnamed: 6_level_1'): {0: '@', 1: np.nan},
            ('Unnamed: 7_level_0', 'Opp'): {0: 'LAR', 1: 'TAM'},
            ('Unnamed: 8_level_0', 'Result'): {0: 'L 27-30', 1: 'L 14-20'},
            ('Unnamed: 9_level_0', 'GS'): {0: '*', 1: '*'},
            ('Rushing', 'Att'): {0: 19, 1: 16},
            ('Rushing', 'Yds'): {0: 128, 1: 37},
            ('Rushing', 'Y/A'): {0: 6.74, 1: 2.31},
            ('Rushing', 'TD'): {0: 2, 1: 0},
            ('Receiving', 'Tgt'): {0: 11, 1: 6},
            ('Receiving', 'Rec'): {0: 10, 1: 2},
            ('Receiving', 'Yds'): {0: 81, 1: 16},
            ('Receiving', 'Y/R'): {0: 8.1, 1: 8.0},
            ('Receiving', 'TD'): {0: 0, 1: 0},
            ('Receiving', 'Ctch%'): {0: '90.9%', 1: '33.3%'},
            ('Receiving', 'Y/Tgt'): {0: 7.36, 1: 2.67},
            ('Passing', 'Cmp'): {0: 0, 1: 0},
            ('Passing', 'Att'): {0: 0, 1: 0},
            ('Passing', 'Cmp%'): {0: 0, 1: 0},
            ('Passing', 'Yds'): {0: 0, 1: 0},
            ('Passing', 'TD'): {0: 0, 1: 0},
            ('Passing', 'Int'): {0: 0, 1: 0},
            ('Passing', 'Rate'): {0: 0, 1: 0},
            ('Passing', 'Sk'): {0: 0, 1: 0},
            ('Passing', 'Yds.1'): {0: 0, 1: 0},
            ('Passing', 'Y/A'): {0: 0, 1: 0},
            ('Passing', 'AY/A'): {0: 0, 1: 0},
            ('Scoring', '2PM'): {0: 0, 1: 0},
            ('Scoring', 'TD'): {0: 2, 1: 0},
            ('Scoring', 'Pts'): {0: 12, 1: 0},
            ('Fumbles', 'Fmb'): {0: 0, 1: 0},
            ('Fumbles', 'FL'): {0: 0, 1: 0},
            ('Fumbles', 'FF'): {0: 0, 1: 0},
            ('Fumbles', 'FR'): {0: 0, 1: 0},
            ('Fumbles', 'Yds'): {0: 0, 1: 0},
            ('Fumbles', 'TD'): {0: 0, 1: 0},
            ('Off. Snaps', 'Num'): {0: 66.0, 1: 75.0},
            ('Off. Snaps', 'Pct'): {0: '100%', 1: '100%'},
            ('Def. Snaps', 'Num'): {0: 0.0, 1: 0.0},
            ('Def. Snaps', 'Pct'): {0: '0%', 1: '0%'},
            ('ST Snaps', 'Num'): {0: 0.0, 1: 0.0},
            ('ST Snaps', 'Pct'): {0: '0%', 1: '0%'}
        })

        expected = pd.DataFrame(
            {'rk': {0: 1.0, 1: 2.0},
            'date': {0: '2019-09-08', 1: '2019-09-12'},
            'g#': {0: 1.0, 1: 2.0},
            'week': {0: 1.0, 1: 2.0},
            'age': {0: 23.093000000000004, 1: 23.096999999999998},
            'tm': {0: 'CAR', 1: 'CAR'},
            'home/away': {0: 'Away', 1: 'Home'},
            'opp': {0: 'LAR', 1: 'TAM'},
            'result': {0: 'L 27-30', 1: 'L 14-20'},
            'gs': {0: '*', 1: '*'},
            'rushing_att': {0: 19, 1: 16},
            'rushing_yds': {0: 128, 1: 37},
            'rushing_y/a': {0: 6.74, 1: 2.31},
            'rushing_td': {0: 2, 1: 0},
            'receiving_tgt': {0: 11, 1: 6},
            'receiving_rec': {0: 10, 1: 2},
            'receiving_yds': {0: 81, 1: 16},
            'receiving_y/r': {0: 8.1, 1: 8.0},
            'receiving_td': {0: 0, 1: 0},
            'receiving_ctch%': {0: '90.9%', 1: '33.3%'},
            'receiving_y/tgt': {0: 7.36, 1: 2.67},
            'passing_cmp': {0: 0, 1: 0},
            'passing_att': {0: 0, 1: 0},
            'passing_cmp%': {0: 0, 1: 0},
            'passing_yds': {0: 0, 1: 0},
            'passing_td': {0: 0, 1: 0},
            'passing_int': {0: 0, 1: 0},
            'passing_rate': {0: 0, 1: 0},
            'passing_sk': {0: 0, 1: 0},
            'passing_yds.1': {0: 0, 1: 0},
            'passing_y/a': {0: 0, 1: 0},
            'passing_ay/a': {0: 0, 1: 0},
            'scoring_2pm': {0: 0, 1: 0},
            'scoring_td': {0: 2, 1: 0},
            'scoring_pts': {0: 12, 1: 0},
            'fumbles': {0: 0, 1: 0},
            'fumbles_lost': {0: 0, 1: 0},
            'fumbles_ff': {0: 0, 1: 0},
            'fumbles_fr': {0: 0, 1: 0},
            'fumbles_yds': {0: 0, 1: 0},
            'fumbles_td': {0: 0, 1: 0}
        })
        assert_frame_equal(expected, player_table_transform(df))

    def test_player_table_reindex(self):
        df = pd.DataFrame(
            {'rk': {0: 1.0, 1: 2.0},
            'date': {0: '2019-09-08', 1: '2019-09-12'},
            'g#': {0: 1.0, 1: 2.0},
            'week': {0: 1.0, 1: 2.0},
            'age': {0: 23.093000000000004, 1: 23.096999999999998},
            'tm': {0: 'CAR', 1: 'CAR'},
            'home/away': {0: 'Home', 1: 'Home'},
            'opp': {0: 'LAR', 1: 'TAM'},
            'result': {0: 'L 27-30', 1: 'L 14-20'},
            'gs': {0: '*', 1: '*'},
            'rushing_att': {0: 19, 1: 16},
            'rushing_yds': {0: 128, 1: 37},
            'rushing_y/a': {0: 6.74, 1: 2.31},
            'rushing_td': {0: 2, 1: 0},
            'receiving_tgt': {0: 11, 1: 6},
            'receiving_rec': {0: 10, 1: 2},
            'receiving_yds': {0: 81, 1: 16},
            'receiving_y/r': {0: 8.1, 1: 8.0},
            'receiving_td': {0: 0, 1: 0},
            'receiving_ctch%': {0: '90.9%', 1: '33.3%'},
            'receiving_y/tgt': {0: 7.36, 1: 2.67},
            'passing_cmp': {0: 0, 1: 0},
            'passing_att': {0: 0, 1: 0},
            'passing_cmp%': {0: 0, 1: 0},
            'passing_yds': {0: 0, 1: 0},
            'passing_td': {0: 0, 1: 0},
            'passing_int': {0: 0, 1: 0},
            'passing_rate': {0: 0, 1: 0},
            'passing_sk': {0: 0, 1: 0},
            'passing_yds.1': {0: 0, 1: 0},
            'passing_y/a': {0: 0, 1: 0},
            'passing_ay/a': {0: 0, 1: 0},
            'scoring_2pm': {0: 0, 1: 0},
            'scoring_td': {0: 2, 1: 0},
            'scoring_pts': {0: 12, 1: 0},
            'fumbles_fmb': {0: 0, 1: 0},
            'fumbles_fl': {0: 0, 1: 0},
            'fumbles_ff': {0: 0, 1: 0},
            'fumbles_fr': {0: 0, 1: 0},
            'fumbles_yds': {0: 0, 1: 0},
            'fumbles_td': {0: 0, 1: 0},
            'player_id': {0: 'McCaCh01', 1: 'McCaCh01'},
            'pos': {0: 'RB', 1: 'RB'},
            'year': {0: 2019, 1: 2019},
            'player_name': {0: 'Christian McCaffrey', 1: 'Christian McCaffrey'}
        })

        expected = pd.DataFrame(
            {'player_id': {0: 'McCaCh01', 1: 'McCaCh01'},
            'player_name': {0: 'Christian McCaffrey', 1: 'Christian McCaffrey'},
            'pos': {0: 'RB', 1: 'RB'},
            'year': {0: 2019, 1: 2019},
            'date': {0: '2019-09-08', 1: '2019-09-12'},
            'week': {0: 1.0, 1: 2.0},
            'age': {0: 23.093000000000004, 1: 23.096999999999998},
            'tm': {0: 'CAR', 1: 'CAR'},
            'home/away': {0: 'Home', 1: 'Home'},
            'opp': {0: 'LAR', 1: 'TAM'},
            'result': {0: 'L 27-30', 1: 'L 14-20'},
            'passing_cmp': {0: 0, 1: 0},
            'passing_att': {0: 0, 1: 0},
            'passing_yds': {0: 0, 1: 0},
            'passing_td': {0: 0, 1: 0},
            'passing_int': {0: 0, 1: 0},
            'passing_rate': {0: 0, 1: 0},
            'passing_sk': {0: 0, 1: 0},
            'passing_y/a': {0: 0, 1: 0},
            'passing_ay/a': {0: 0, 1: 0},
            'rushing_att': {0: 19, 1: 16},
            'rushing_yds': {0: 128, 1: 37},
            'rushing_y/a': {0: 6.74, 1: 2.31},
            'rushing_td': {0: 2, 1: 0},
            'receiving_tgt': {0: 11, 1: 6},
            'receiving_rec': {0: 10, 1: 2},
            'receiving_yds': {0: 81, 1: 16},
            'receiving_y/r': {0: 8.1, 1: 8.0},
            'receiving_td': {0: 0, 1: 0},
            'receiving_y/tgt': {0: 7.36, 1: 2.67},
            'scoring_2pm': {0: 0, 1: 0},
            'fumbles': {0: 0, 1: 0},
            'fumbles_lost': {0: 0, 1: 0}}
        )
        assert_frame_equal(expected, player_table_reindex(df))

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: Robert-F-Mulligan/fantasy-football

max_issues_repo_name: Robert-F-Mulligan/fantasy-football

max_forks_repo_name: Robert-F-Mulligan/fantasy-football


text:
# In[1]:

import sys
sys.path.append("/Users/rbiswas/src/interfacecosmology")

import halomass as hm
from interfaces import FCPL
import psutils as psu
import numpy as np
import matplotlib.ticker as ticker
import matplotlib.pyplot as plt
import camb_utils.cambio as cio
import psutils as psu
from interfaces import FCPL
import massfunctions as mf
import utils.plotutils as pu
import halomass as hmf
import sys
import matplotlib.gridspec as gridspec

# In[2]:

# INPUT FILES (CAMB outputs)
dirn  = "./"
#M000 (LCDM)
tkfile  = dirn +"example_data/M000/m000n0_transfer_fin_out.dat"
pkfile = dirn + "example_data/M000/m000n0_matterpower_fin.dat"
#M000n1 (same LCDM, with some fraction of CDM replaced by massive neutrinos)
ntkfile = dirn +"example_data/M000n1/m000n1_transfer_fin_out.dat"
npkfile = dirn +"example_data/M000n1/m000n1_matterpower_fin.dat"

# In[3]:

#Cosmological Models:
#               Same cosmology represented differently
M000 = FCPL(H0 = 71. ,Om0  = 0.26479, Ob0 = 0.044793, ns = 0.963, As = 2.16e-9)
M000s  = FCPL(H0 = 71., Om0 = 0.26479, Ob0 = 0.044793, ns = 0.963, sigma8 = 0.8)
#
M000n1 = FCPL(H0 = 71., Om0 = 0.26479, Ob0 = 0.044793, ns = 0.963, sigma8 = 0.8, sigmamnu=0.94)

# In[4]:

#powerspectrum
#use power spectrum from CAMB power spectrum output
psfrompk = psu.powerspectrum(koverh = None, asciifile = pkfile)
#use matter power spectrum from CAMB transfer function output, with As
psfromtkm = psu.powerspectrum(koverh = None, pstype ="matter", asciifile =tkfile, cosmo = M000)
#use matter power spectrum from CAMB transfer function output, with sigma8
psfromtkms = psu.powerspectrum(koverh = None, pstype ="matter", sigma8type = "matter", asciifile =tkfile, cosmo = M000s)
#use cb power spectrum from CAMB transfer function output, normalized to matter
psfromtkcbs = psu.powerspectrum(koverh = None, pstype ="cb", sigma8type = "matter", asciifile =tkfile, cosmo = M000s)
#use cb power spectrum from CAMB transfer function output, normalized to cb
psfromtkcbscb = psu.powerspectrum(koverh = None, pstype ="cb", sigma8type = "cb", asciifile =tkfile, cosmo = M000s)
#
#psfrompkn = psu.powerspectrum(koverh = None, asciifile  = npkfile)
#psfromtknm = psu.powerspectrum(koverh = None, asciifile  = ntkfile, pstype ="matter", cosmo = M000n1)

# In[5]:

simdir = "/Users/rbiswas/doc/projects/NeutrinoMF/doc/files/paperfiles/"
fname0 = simdir + "m000.499.fof_mf"

# In[13]:

#M000_z0 = np.loadtxt(simdir + "m000.499.fof_mf")

# In[15]:

#MinhinvMsun = M000_z0[:,0]

# In[25]:

def fracPoissonerrors(num ,asymmetric = True):

	num  = np.asarray(num)
	sig  = np.sqrt(num + 1.0/4.)

	if asymmetric:
		return np.array([(sig +0.5)/num, (sig - 0.5)/num])
	else :
		return 1./np.sqrt(num)

def get_errorbars(fname  ):

    data  = np.loadtxt(fname)

    MininvhMsun = data[:,0]
    numcluster  = data[:,1]
    massfn      = data[:,2]
    calcB       = data[:,-2]
    calcM       = data[:,-1]

    return MininvhMsun , massfn, fracPoissonerrors(numcluster )*massfn, calcB, calcM

#MininvhMsun , massfn, massfnyerr = get_errorbars(fname)
#ax = plt.subplot()
def plotnormalized(fname , normals= None, normalnames=None, styles=None, usefile =  True):#, ps = None, cosmo = None):

    ax = plt.gca()
    #if axesobj != None:
    #       ax = axesobj

    MininvhMsun , massfn , massfnyerr,calcB, calcM = get_errorbars(fname)

    #print "YERR", massfnyerr
    #print np.shape(massfnyerr[0]), np.shape(massfnyerr[1]), np.shape( MininvhMsun), np.shape(massfn), np.shape(normals[0])
    #Mass = MininvhMsun*cosmo.h
    #normalB = hm.dndlnM(Mass , ps = ps , z= z, cosmo
    #print yerrba
    if normals == None:
        normval = 1.
        #ax.errorbar(MininvhMsun , massfn /normval , yerr = yerrbars/normval)
    if usefile:
        ax.errorbar( MininvhMsun, massfn/calcB , yerr = massfnyerr/calcB,  fmt  = "ks", label="Bhattacharya etal.")
        ax.errorbar( MininvhMsun, massfn/calcM , yerr = massfnyerr/calcM,  fmt  = "ro", label="MICE")

    else:
        for i, normval in enumerate(normals):
            name = normalnames [i]
            stylename = styles[i]
            #print name, np.shape(normval), np.shape(massfn), np.shape(MininvhMsun), np.shape(normval), name
            ax.errorbar( MininvhMsun, massfn/normval , yerr = massfnyerr/normval, label = name, fmt  = stylename)
            #ax.errorbar( MininvhMsun, massfn/calcB , yerr = massfnyerr/calcB,  fmt  = "bd")

            #ax.errorbar( MininvhMsun, massfn/hm.dndlnM(M = masses, ps = psfromtkcbscb , z = 0.,bgtype="cb",cosmo= M000s), yerr = massfnyerr/normval, label = name, fmt  = stylename)
    ax.set_xscale('log')
    ax.grid(True)
    xvals = np.linspace(1.0e13,1.1e15,2)
    refval = 1.0
    bandvals = [-0.1,0.1]
    ax.plot(xvals, np.ones(len(xvals)),'k-',lw =2.0)
    ax.fill_between (xvals , refval + bandvals[0], refval + bandvals[1],color = 'gray', alpha =0.25)
    ax.set_xlim(1.0e13,1.0e15)
    ax.xaxis.set_ticklabels("",visible=False)
    ax.yaxis.set_ticks([0.9,1.0,1.1])
    #ax.yaxis.set_ticklabels("",visible=False)
    ax.set_ylim(0.85,1.15)
    ax.grid(True)

    return 0

# In[33]:

gs  = gridspec.GridSpec(3,1, height_ratios=[0.33,0.33,0.33], width_ratios=[1.,0.,0.])
ax0 = plt.subplot(gs[0])
fname = simdir+ "m000.499.fof_mf"
masses = get_errorbars(fname)[0]/M000s.h
#sigmaM_0 = psu.
dndlnm_B0 = hmf.dndlnM0(M = masses, ps = psfromtkcbscb , z = 0.,bgtype="cb",cosmo= M000s)/M000s.h**3
dndlnm_M0 = hmf.dndlnM0(M = masses, ps = psfromtkcbscb , z = 0.,bgtype="cb",cosmo = M000s, fittingform="MICE")/M000s.h**3 #*M000s.h**3

plotnormalized(fname, normals = [dndlnm_B0,dndlnm_M0],normalnames = ["Bhattacharya", "MICE"],styles =["ks","ro"])
#ax0.set_xscale('log')
#ax0.set_yscale('log')
ax1 = plt.subplot(gs[1])
psfromtkcbscb1 = psu.powerspectrum(koverh = None, pstype ="cb", sigma8type = "cb", asciifile =tkfile, cosmo = M000s,z = 1.0, method = "CAMBoutgrowth")

fname1 = simdir+ "m000.247.fof_mf"
masses1 = get_errorbars(fname1)[0]/M000s.h
dndlnm_B1 = hmf.dndlnM0(M = masses1, ps = psfromtkcbscb , z = 1.,bgtype="cb",cosmo= M000s,deltac= 1.684)/M000s.h**3/2.**3.
dndlnm_M1 = hmf.dndlnM0(M = masses1, ps = psfromtkcbscb , z = 1.,bgtype="cb",cosmo = M000s, fittingform="MICE")/M000s.h**3./2**3.0 #*M000s.h**3

#dndlnm_B1 = hmf.dndlnM(M = masses1, ps = psfromtkcbscb1 , z = 1.,bgtype="cb",cosmo= M000s,deltac= 1.684)/M000s.h**3
#dndlnm_M1 = hmf.dndlnM(M = masses1, ps = psfromtkcbscb1 , z = 1.,bgtype="cb",cosmo = M000s, fittingform="MICE")/M000s.h**3. #*M000s.h**3
plotnormalized(fname1, normals = [dndlnm_B1,dndlnm_M1],normalnames = ["Bhattacharya", "MICE"],styles =["ks","ro"])

#plotnormalized(simdir+ "m000.247.fof_mf")
ax2 = plt.subplot(gs[2])
psfromtkcbscb2 = psu.powerspectrum(koverh = None, pstype ="cb", sigma8type = "cb", asciifile =tkfile, cosmo = M000s,z = 2.0, method = "CAMBoutgrowth")
fname2 = simdir+ "m000.163.fof_mf"
masses2 = get_errorbars(fname2)[0]/M000s.h

#print "Masses", masses2
#print "Eavltest", np.shape(masses2)
dndlnm_B2 = hmf.dndlnM0(M = masses2, ps = psfromtkcbscb , z = 2.,bgtype="cb",cosmo= M000s, deltac=1.686)/M000s.h**3/3**3.0
dndlnm_M2 = hmf.dndlnM0(M = masses2, ps = psfromtkcbscb , z = 2.,bgtype="cb",cosmo = M000s, fittingform="MICE")/M000s.h**3/3.0**3 #*M000s.h**3

#dndlnm_B2 = hmf.dndlnM(M = masses2, ps = psfromtkcbscb2 , z = 2.,bgtype="cb",cosmo= M000s, deltac=1.686)/M000s.h**3/27.0
#dndlnm_M2 = hmf.dndlnM(M = masses2, ps = psfromtkcbscb2 , z = 2.,bgtype="cb",cosmo = M000s, fittingform="MICE")/M000s.h**3/27. #*M000s.h**3
#print "Hello what is the problem?"
#print np.shape(masses2), np.shape(psfromtkcbscb2), np.shape(dndlnm_B2),np.shape(dndlnm_M2)
#plotnormalized(fname2, normals = [dndlnm_B1,dndlnm_M1],normalnames = ["Bhattacharya", "MICE"],styles =["ks","ro"])

#print "Bye"
#print "Eavl"#, np.shape(masses2) #,np.shape(psfromtkcbscb2), np.shape(dndlnm_B2) , np.shape(dndlnm_M2)

#plotnormalized(fname2, normals = [dndlnm_B2,dndlnm_M2],normalnames = ["Bhattacharya", "MICE"],styles =["ks","ro"])

plotnormalized(fname2, normals = [dndlnm_B2,dndlnm_M2],normalnames = ["Bhattacharya", "MICE"],styles =["ks","ro"])

#plotnormalized(simdir + "m000.163.fof_mf")
ax2.set_xscale('log')
ax2.set_xlabel(r'Mass ($h^{-1}M_\odot$)' )
ax1.set_ylabel("ratio of massfn")
ax2.legend(loc='upper right', numpoints =1)
ax1.text(1.0, 0.0,"z = 2.")
#axt2 = ax2.twiny()
#axt2.set_xlabel(r"z=2")
#axt1 = ax1.twiny()
#axt1.set_xlabel(r"z=1")
#axt0 = ax0.twiny()
#axt0.set_xlabel(r"z=0")

plt.tight_layout(h_pad = -0.1, w_pad = -0.1)
#plt.savefig("M000_Formula.pdf")
plt.savefig("M000_SumanFormula.pdf")

# Out[33]:

#     0.333 0.788 0.807 1.795
#     0.308553824609 0.782556886404 0.807 1.795
#     0.295094350742 0.779390315288 0.807 1.795
#

# image file:

# In[17]:

plt.plot(masses,1.0/hmf.dndlnM0(M = masses, ps = psfromtkcbscb , z = 2.,bgtype="cb",cosmo= M000s, deltac=1.686)*hmf.dndlnM(M = masses, ps = psfromtkcbscb2, z= 2.0, bgtype = "cb", cosmo= M000s, deltac=1.686),"ks")
plt.xscale('log')

# Out[17]:

#     0.295094350742 0.779390315288 0.807 1.795
#     0.295094350742 0.779390315288 0.807 1.795
#

# image file:

# In[ ]:

dndlnm_B1c = dndlnm_B1*0.12
plotnormalized(fname1, normals = [dndlnm_B1c,dndlnm_M1],normalnames = ["Bhattacharya", "MICE"],styles =["ks","ro"])

print dndlnm_B1 , dndlnm_B1c

# In[38]:

dndlnm_B2 = hm.dndlnM(M = masses2, ps = psfromtkcbscb2 , z = 2.,bgtype="cb",cosmo= M000s, deltac=1.686)/M000s.h**3
dndlnm_M2 = hm.dndlnM(M = masses2, ps = psfromtkcbscb2 , z = 2.,bgtype="cb",cosmo = M000s, fittingform="MICE")/M000s.h**3
print dndlnm_B2, dndlnm_M2
print np.shape(dndlnm_B2)
print 1./8.

# Out[38]:

#     0.295094350742 0.779390315288 0.807 1.795
#     [  8.69840997e-03   5.74194925e-03   3.62316898e-03   2.18669780e-03
#        1.27152405e-03   7.13534935e-04   3.82356121e-04   1.92706374e-04
#        9.00314410e-05   3.81860774e-05   1.49713792e-05   5.08301266e-06
#        1.40307142e-06   3.45923170e-07   6.94611686e-08   1.67266650e-08] [  8.09230756e-03   5.33340159e-03   3.36490446e-03   2.03425732e-03
#        1.18760520e-03   6.71018535e-04   3.63331163e-04   1.85832054e-04
#        8.85716560e-05   3.85895834e-05   1.56552636e-05   5.55795206e-06
#        1.62826867e-06   4.31161409e-07   9.46077053e-08   2.47736866e-08]
#     (16,)
#     0.125
#

# In[109]:

plt.plot(masses, hm.dndlnM(M = masses, ps = psfromtkcbscb, cosmo = M000s)/psu.dndlnM(masses, ps = psfromtkcbscb,cosmo = M000))
plt.xscale('log')
plt.ylim(0.9,1.1)
print M000s.h**3


text:
"""
Circuito RC (Clase y solución para distintas frecuencas)

Luis Eduardo Sánchez González

Facultad de Ciencias Físico Matemáticas
Física Computacional

mié 10 mar 2021 09:12:48 CST

Repositorio: https://github.com/Luis2501/Fisica-Computacional-1
"""
import numpy as np
import sys
sys.path.append("../")
from PhysicsPy.ODEsMethods import *

class RC_Circuit:

	"""
	Clase que permite resolver un circuito RC,
	tanto para una fuente de voltaje alterna
	como fuente de voltaje directa.
	"""

	def __init__(self, R, C, Vs):

		self.R, self.C = R, C

		if isinstance(Vs, (int, float)):

			self.Vs = lambda t: Vs						#Voltaje para corriente directa,
											#El circuito es estacionario si Vs = 0

		elif callable(Vs):

			self.Vs = Vs							#Fuente de voltaje alterno

		else:

			raise ValueError("Voltage is not defined")			#No se definio algún tipo de voltaje

	def __call__(self, u, t):

		Q = u
		R, C = self.R, self.C
		Vs = self.Vs

		return (Vs(t)/R) - (Q/(R*C))

def SolveCircuit(frecuencia, Periodo, R, C):

	"""
	Función que resuelve el circuito para distintas
	frecuencias de una fuenta de voltaje.
	"""

	QQ, tt, VVs = [], [], []

	for f, T in zip(frecuencia, Periodo):

		Vs = lambda t: np.cos((2*np.pi*f)*t)					#Fuente de voltaje

		Circuito = RC_Circuit(R, C, Vs)						#Instancia, creamos un circuito

		Solucion = Euler(Circuito)						#Resolvemos con el método de Euler
		Solucion.InitialConditions(1e-6, [0, 2*T], (1/f)*1e-3)			#Condiciones iniciales
		Q, t = Solucion.SolveODE()						#Obtenemos la solución

		QQ.append(Q[:,0]) ; tt.append(t)
		VVs.append(Vs(t))

	return QQ, tt, VVs


text:
"""
Tests for `bx.seqmapping`.
"""
import unittest
from io import StringIO

from numpy import (
    allclose,
    array,
)

import bx.seqmapping

class CharMappingTests(unittest.TestCase):
    __test__ = False

    def test_DNA(self):
        assert(allclose(
            bx.seqmapping.DNA.translate("ACGTacgt-?X"),
            [0, 1, 2, 3, 0, 1, 2, 3, 4, -1, -1]))

    def test_DNA_list(self):
        assert(allclose(
            bx.seqmapping.DNA.translate_list(["ACGTA", "TGCAX"]),
            [0 + 3*6, 1 + 2*6, 2 + 1*6, 3 + 0*6, -1]))

    def test_other(self):
        m = bx.seqmapping.CharToIntArrayMapping()
        m.set_mapping("A", 0)
        m.set_mapping("B", 7)
        assert(allclose(m.translate("ABCCBA"), [0, 7, -1, -1, 7, 0]))

class IntMappingTests(unittest.TestCase):
    __test__ = False

    def test_simple(self):
        m = bx.seqmapping.IntToIntMapping(4)
        m.set_mapping(0, 0)
        m.set_mapping(2, 0)
        m.set_mapping(1, 1)
        m.set_mapping(3, 1)
        assert(allclose(m.translate(array([0, 1, 2, 3, 4], 'i')), array([0, 1, 0, 1, -1])))

eight_species_mapping = """TTTTTTTT 0
CCCCCCCC 4
AAAAAAAA 1
GGGGGGGG 5
AAAAA*AA 2
TTTTT*TT 0
GGGGG*GG 5
CCCCC*CC 4
GGGG*GGG 5
TTTT*TTT 2
GGGAAGGG 5
AAAA*AAA 2
TTTTT*T* 2
CCCCC*C* 4
CCCTTCCC 4
CCCC*CCC 4
TTTT**TT 2
AAAA**AA 2
AAAAA*A* 2
GGGGG*G* 5
AAAAAGAA 2
TTTTTCTT 2
GGGAA*GG 5
TTTT**T* 2
TTTCCTTT 0
AAAAAAA* 1
CCCTT*CC 3
TTTTTTT* 2
CC*CC*CC 3
AAAGGAAA 2
------G- 2
"""

rows = ["AAATTGT-----ATGTCCATCCTTTAAAGGTCATTCCTTTAATGGTCTTTTCTGGACACCACTAGGGGTCAGAAGTAGTTCATCAAAC-----------------TTTCTTCCCTCCC-TACTTCAGTG",
        "AAATTGT-----ATGTCCATCCTTTAAAGGTCATTCCTTTAATGGTCTTTTCTGGACACCACTAGGGGTCAGAAGTAGTTCATCAAAC-----------------TTTCTTCCCTCCC-TACTTCAGTG",
        "AAATTTT-----ATGTCTATCCTTTAAAGGTCATTCCTCTAATAGTCTTTTCTGGACACCACTAGGGGTCAGAAGTAGTTCATTAAAC-----------------TTTCTTCCCTCCC-TACCTCAGTG",
        "AAACTGT-----ATCACCACCTTTTTAAGGTCATTTCTCTAATGATCCTGTT-GCATACCAGTAGGGGGCAGAAGTGTTCCGCTGATTTCCGCCCTCCTCCCCACCCCCCCACCCCCC-TTATTCAAAG",
        "*********************************************************************************************************************************",
        "-TATTAT-----ATGGCCATGTTCAAAAGGTTGTTTCTCTAATGATTCCTTC-TGATACCAGTAGGGGTCAGAAGTGGTCCATTGATT---------------------CTTTTCCTC-TGATTC-AAG",
        "AAATTGA--AAGATCTCACTCTTTGCCAGGTAGTCCATCTAAGGGTCACATATGGATACCAGCAGGGCCT-GAAGAAGCCCATTGAAT------------------------TTTCCC-ATCTTCAAGG",
        "AAATTCATGATAGTGTCACTCTTAAATAGATGATTC--------TTCACAT---GATGCCAGCAGGGGGC-AGAGCAGGCTGTGAAAT------------------------TTTCCCTTTCTTCAAAG"]

class AlignmentMappingTests(unittest.TestCase):
    __test__ = False

    def test_largescale(self):
        f = StringIO(eight_species_mapping)
        n, m = bx.seqmapping.alignment_mapping_from_file(f)
        t = bx.seqmapping.DNA.translate_list(rows)
        m.translate(t)

max_stars_repo_name: lldelisle/bx-python

max_issues_repo_name: lldelisle/bx-python

max_forks_repo_name: lldelisle/bx-python

reason: from numpy


text:
"""
Upset plot Example
==========================

Here shows how to draw upset plot

"""
import numpy as np
import pandas as pd

import milkviz as mv

# %%
# Data input for upset plot
# -------------------------------------

# A list of any sequence, intersections between lists will be computed for you, duplicates will not considred
#
# Noted that this is the difference from input in venn diagram.
#
mammals = ['Cat', 'Dog', 'Horse', 'Sheep', 'Pig', 'Cattle', 'Rhinoceros', 'Moose']
herbivores = ['Horse', 'Sheep', 'Cattle', 'Moose', 'Rhinoceros']
domesticated = ['Dog', 'Chicken', 'Horse', 'Sheep', 'Pig', 'Cattle', 'Duck']

# %%
# Create the upset plot
# ----------------------------
#
mv.upset([mammals, herbivores, domesticated], names=['mammals', 'herbivores', 'domesticated'])

max_stars_repo_name: Mr-Milk/milkviz

max_issues_repo_name: Mr-Milk/milkviz

max_forks_repo_name: Mr-Milk/milkviz


text:
# from http://docs.pymc.io/notebooks/GLM-logistic.html

import pandas as pd
import numpy as np
import pymc3 as pm
import matplotlib.pyplot as plt
import seaborn
import warnings

from sklearn.metrics import accuracy_score

warnings.filterwarnings('ignore')
from collections import OrderedDict
from time import time

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from scipy.optimize import fmin_powell
from scipy import integrate

import theano as thno
import theano.tensor as T

def run_models(df, upper_order=5):
    '''
    Convenience function:
    Fit a range of pymc3 models of increasing polynomial complexity.
    Suggest limit to max order 5 since calculation time is exponential.
    '''

    models, traces = OrderedDict(), OrderedDict()

    for k in range(1,upper_order+1):

        nm = 'k{}'.format(k)
        fml = create_poly_modelspec(k)

        with pm.Model() as models[nm]:

            print('\nRunning: {}'.format(nm))
            pm.glm.GLM.from_formula(fml, df, family=pm.glm.families.Normal())

            traces[nm] = pm.sample(2000, init=None)

    return models, traces

def plot_traces(traces, retain=1000):
    '''
    Convenience function:
    Plot traces with overlaid means and values
    '''

    ax = pm.traceplot(traces[-retain:], figsize=(12,len(traces.varnames)*1.5),
        lines={k: v['mean'] for k, v in pm.df_summary(traces[-retain:]).iterrows()})

    for i, mn in enumerate(pm.df_summary(traces[-retain:])['mean']):
        ax[i,0].annotate('{:.2f}'.format(mn), xy=(mn,0), xycoords='data'
                    ,xytext=(5,10), textcoords='offset points', rotation=90
                    ,va='bottom', fontsize='large', color='#AA0022')

def create_poly_modelspec(k=1):
    '''
    Convenience function:
    Create a polynomial modelspec string for patsy
    '''
    return ('income ~ educ + hours + age ' + ' '.join(['+ np.power(age,{})'.format(j) for j in range(2,k+1)])).strip()

data = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data", header=None, names=['age', 'workclass', 'fnlwgt',
                'education-categorical', 'educ',
                'marital-status', 'occupation',
                'relationship', 'race', 'sex',
                'captial-gain', 'capital-loss',
                'hours', 'native-country',
                'income'])

income = 1 * (data['income'] == " >50K")
age2 = np.square(data['age'])

data = data[['age', 'educ', 'hours']]
data['age2'] = age2
data['income'] = income

with pm.Model() as logistic_model:
    pm.glm.GLM.from_formula('income ~ age + age2 + educ + hours', data, family=pm.glm.families.Binomial())
    trace_logistic_model = pm.sample(500)

sd = np.mean(trace_logistic_model['sd'])
intercept = np.mean(trace_logistic_model['Intercept'])
age = np.mean(trace_logistic_model['age'])
age2 = np.mean(trace_logistic_model['age2'])
edu = np.mean(trace_logistic_model['education'])
hours = np.mean(trace_logistic_model['hours'])

eval_set = data
Y = eval_set['income'].tolist()
yp = eval_set['age'].values * age + eval_set['age2'].values * age2 \
    + eval_set['education'].values * edu + eval_set['hours'].values \
    * hours + intercept

th = np.mean(yp)
yp_b = [1 if p >= th else 0 for p in yp]

print(accuracy_score(Y, yp_b))

plt.plot(Y[:80], color="green")
plt.plot(yp[:80], color="pink")
plt.show()

max_stars_repo_name: Nadheesh/ml-benchmark

max_issues_repo_name: Nadheesh/ml-benchmark

max_forks_repo_name: Nadheesh/ml-benchmark

reason: import numpy,from scipy,import theano,import pymc3


text:
#v1.04beta
#17 Oct 2019 by Mengting Ju

######################################################################################
from astropy.io import fits
import numpy as np
import pandas as pd
import pylab as pl
from scipy.integrate import simps
import read_filter
#########################################################################################
#select input parameters
def input_mag_model(targetmag,galtpl,deltal, filtera,specsample):

    resulte=read_filter.filteraa(filtera)
    wavefilter=resulte[0]
    fluxfilter=resulte[1]
    vmin=resulte[2]
    vmax=resulte[3]
    filtereff=resulte[4]

  ####################################################################################
    # define wavelength array,
    #cover the range of 350nm to 1050nm, depend on the spectral resolution wanted.

    sampling=specsample #pixels per spectral resolution element ?1D or 2D/linear or area?
    print('sampling:', sampling)

    delta_lambda=deltal # has to be in unit of A
    print('delta_lambda:', delta_lambda)

    narray=int((10000.0-3500.0)/delta_lambda)
    wavearr=3500.0+delta_lambda*pl.frange(narray-1)
    # select out the array of V band filter
    ii=np.logical_and(wavearr >= vmin, wavearr <= vmax)
    wavetmp2=wavearr[ii]
    x=np.interp(wavetmp2,wavefilter,fluxfilter)
    integratef4=x*wavetmp2
    integconst=simps(integratef4,wavetmp2) # int(lambda*Rlambda*dlambda)

    lambdav=filtereff #A
    ###############################################################
    # define basic target brightness, parameters constantly change
    itarget=targetmag    # in Johnson V mag/arcsec^2 unit
    print('itarget:',itarget)

    itarget_jy=3631.0*10**(-itarget/2.5+3.0)  # target flux in V in mJy/arcsec^2 unit
    itarget_nm=itarget_jy*3.0/(lambdav/100.0)**2 #target flux in V in 10^(-12)erg/s/cm^2/A (/arcsec^2 ?)

    galtpl=galtpl
    tplfile='../obs/SFgal_tpl/'+galtpl
    print('tplfile:',tplfile)

    sfgal=fits.open(tplfile)
    wavegal=sfgal[1].data['wave'] # A
    galflux2=sfgal[1].data['flux']
    galflux1=np.interp(wavearr,wavegal,galflux2)

    #;normalize the galaxy spectrum to the V band magnitude specified.
    ii=np.logical_and(wavegal >= vmin, wavegal <= vmax)
    wavetmp=wavegal[ii]
    fluxtmp=galflux2[ii]
    x=np.interp(wavetmp,wavefilter,fluxfilter)
    vfluxtmp=x*wavetmp*fluxtmp #bandpass*lambda*F_gal_lambda
    galintegrate=simps(vfluxtmp,wavetmp)
    galnorm=itarget_nm*integconst/galintegrate
    galflux=galnorm*galflux1   # the unit should now be in 10^(-12)erg/s/A/cm^2 (/arcsec^2 ?)

##################################################################
    return wavearr, galflux, integconst

##############################################################

max_stars_repo_name: nanamomoko/snpp

max_issues_repo_name: nanamomoko/snpp

max_forks_repo_name: nanamomoko/snpp

reason: import numpy,from scipy,from astropy