text:
import torch
import torch.nn.functional as F
import numpy as np

device = torch.device("cpu")

class ImageToWordModel():
    def __init__(self, encoder, decoder, word_map):
        """
        :param encoder: encoder model
        :param decoder: decoder model
        :param word_map: word map
        """

        self.encoder = encoder
        self.decoder = decoder
        self.word_map = word_map
        self.rev_word_map = {v: k for k, v in word_map.items()}  # ix2word

    def predict(self, img, beam_size=3):
        """
        Reads an image and captions it with beam search.
        :param img: PIL image
        :param beam_size: number of sequences to consider at each decode-step
        :return: caption, weights for visualization
        """

        k = beam_size
        vocab_size = len(self.word_map)

        img = img.resize((256, 256))
        img = np.asarray(img)

    #     img = imresize(img, (256, 256))
        img = img.transpose(2, 0, 1)
        img = img / 255.
        img = torch.FloatTensor(img).to(device)[:3]
        mean = torch.Tensor([0.485, 0.456, 0.406])
        std = torch.Tensor([0.229, 0.224, 0.225])
        image = (img - mean[:,None,None]) / std[:,None,None]

        # Encode
        image = image.unsqueeze(0)  # (1, 3, 256, 256)
        encoder_out = self.encoder(image)  # (1, enc_image_size, enc_image_size, encoder_dim)
        enc_image_size = encoder_out.size(1)
        encoder_dim = encoder_out.size(3)

        # Flatten encoding
        encoder_out = encoder_out.view(1, -1, encoder_dim)  # (1, num_pixels, encoder_dim)
        num_pixels = encoder_out.size(1)

        # We'll treat the problem as having a batch size of k
        encoder_out = encoder_out.expand(k, num_pixels, encoder_dim)  # (k, num_pixels, encoder_dim)

        # Tensor to store top k previous words at each step; now they're just <start>
        k_prev_words = torch.LongTensor([[self.word_map['<start>']]] * k).to(device)  # (k, 1)

        # Tensor to store top k sequences; now they're just <start>
        seqs = k_prev_words  # (k, 1)

        # Tensor to store top k sequences' scores; now they're just 0
        top_k_scores = torch.zeros(k, 1).to(device)  # (k, 1)

        # Tensor to store top k sequences' alphas; now they're just 1s
        seqs_alpha = torch.ones(k, 1, enc_image_size, enc_image_size).to(device)  # (k, 1, enc_image_size, enc_image_size)

        # Lists to store completed sequences, their alphas and scores
        complete_seqs = list()
        complete_seqs_alpha = list()
        complete_seqs_scores = list()

        # Start decoding
        step = 1
        h, c = self.decoder.init_hidden_state(encoder_out)

        # s is a number less than or equal to k, because sequences are removed from this process once they hit <end>
        while True:

            embeddings = self.decoder.embedding(k_prev_words).squeeze(1)  # (s, embed_dim)

            awe, alpha = self.decoder.attention(encoder_out, h)  # (s, encoder_dim), (s, num_pixels)

            alpha = alpha.view(-1, enc_image_size, enc_image_size)  # (s, enc_image_size, enc_image_size)

            gate = self.decoder.sigmoid(self.decoder.f_beta(h))  # gating scalar, (s, encoder_dim)
            awe = gate * awe

            h, c = self.decoder.decode_step(torch.cat([embeddings, awe], dim=1), (h, c))  # (s, decoder_dim)

            scores = self.decoder.fc(h)  # (s, vocab_size)
            scores = F.log_softmax(scores, dim=1)

            # Add
            scores = top_k_scores.expand_as(scores) + scores  # (s, vocab_size)

            # For the first step, all k points will have the same scores (since same k previous words, h, c)
            if step == 1:
                top_k_scores, top_k_words = scores[0].topk(k, 0, True, True)  # (s)
            else:
                # Unroll and find top scores, and their unrolled indices
                top_k_scores, top_k_words = scores.view(-1).topk(k, 0, True, True)  # (s)

            # Convert unrolled indices to actual indices of scores
            prev_word_inds = (top_k_words / vocab_size).long()  # (s)
            next_word_inds = top_k_words % vocab_size  # (s)

            # Add new words to sequences, alphas
            seqs = torch.cat([seqs[prev_word_inds], next_word_inds.unsqueeze(1)], dim=1)  # (s, step+1)
            seqs_alpha = torch.cat([seqs_alpha[prev_word_inds], alpha[prev_word_inds].unsqueeze(1)],
                                   dim=1)  # (s, step+1, enc_image_size, enc_image_size)

            # Which sequences are incomplete (didn't reach <end>)?
            incomplete_inds = [ind for ind, next_word in enumerate(next_word_inds) if
                               next_word != self.word_map['<end>']]
            complete_inds = list(set(range(len(next_word_inds))) - set(incomplete_inds))

            # Set aside complete sequences
            if len(complete_inds) > 0:
                complete_seqs.extend(seqs[complete_inds].tolist())
                complete_seqs_alpha.extend(seqs_alpha[complete_inds].tolist())
                complete_seqs_scores.extend(top_k_scores[complete_inds])
            k -= len(complete_inds)  # reduce beam length accordingly

            # Proceed with incomplete sequences
            if k == 0:
                break
            seqs = seqs[incomplete_inds]
            seqs_alpha = seqs_alpha[incomplete_inds]
            h = h[prev_word_inds[incomplete_inds]]
            c = c[prev_word_inds[incomplete_inds]]
            encoder_out = encoder_out[prev_word_inds[incomplete_inds]]
            top_k_scores = top_k_scores[incomplete_inds].unsqueeze(1)
            k_prev_words = next_word_inds[incomplete_inds].unsqueeze(1)

            # Break if things have been going on too long
            if step > 50:
                break
            step += 1

        i = complete_seqs_scores.index(max(complete_seqs_scores))
        seq = complete_seqs[i]
        alphas = complete_seqs_alpha[i]

        return seq, alphas


text:
from typing import Iterator, Dict, Any, Optional
from datetime import datetime
from io import StringIO, TextIOBase
from emannotationschemas import get_schema
from emannotationschemas.models import split_annotation_schema
from multiwrapper import multiprocessing_utils as mu
from dynamicannotationdb.annotation_client import DynamicAnnotationClient
from dynamicannotationdb.materialization_client import DynamicMaterializationClient
from dynamicannotationdb.key_utils import build_table_id, build_segmentation_table_id
import pandas as pd
from collections import OrderedDict
import time

def _process_dataframe_worker(args):
    ind, dataframe, schema, schema_name, data_mapping = args
    annotations = []
    segmentations = []

    insert_time = datetime.now()
    flat_annotation_schema, flat_segmentation_schema = split_annotation_schema(schema)
    for index, row in dataframe.iterrows():
        ann = dict(row)

        ordered_anno_data = OrderedDict.fromkeys(data_mapping["anno_model"])
        ordered_seg_data = OrderedDict.fromkeys(data_mapping["seg_model"])

        annotation_data, segmentation_data = map(
            lambda keys: {x: ann[x] for x in keys},
            [data_mapping["position_keys"], data_mapping["segmentation_keys"]],
        )

        position_data = {
            key: f"POINTZ({val[0]} {val[1]} {val[2]})"
            for key, val in annotation_data.items()
            if "position" in key
        }

        ordered_seg_data.update(
            {
                "id": index,
                "annotation_id": index,
            }
        )

        ordered_anno_data.update(
            {
                "id": index,
                "created": insert_time,
                "deleted": None,
                "superceded_id": None,
                "valid": True,
                "size": None,
            }
        )

        ordered_anno_data.update(position_data)
        ordered_seg_data.update(segmentation_data)

        annotations.append(ordered_anno_data)
        segmentations.append(ordered_seg_data)

    return ind, annotations, segmentations

def common_column_set(columns_a, columns_b):
    columns_set_a = set(columns_a)
    columns_set_b = set(columns_b)
    if columns_set_a & columns_set_b:
        return columns_set_a & columns_set_b
    else:
        return "There are no common elements"

def process_dataframe(
    schema,
    schema_name: str,
    dataframe,
    data_mapping: dict,
    chunksize: int,
    n_threads: int = 16,
):

    timing = []
    multi_args = []

    schema = schema(context={"postgis": True})

    for i_start in range(0, len(dataframe), chunksize):
        multi_args.append(
            [
                i_start,
                dataframe[i_start : i_start + chunksize],
                schema,
                schema_name,
                data_mapping,
            ]
        )

    multi_start = time.time()
    print("START")

    results = mu.multiprocess_func(
        _process_dataframe_worker, multi_args, n_threads=n_threads, debug=n_threads == 1
    )

    multi_end = time.time()
    print(f"MULTI PROCESSING TIME: {multi_end-multi_start}")
    return results

def test_sqlalchemy_orm_bulk_insert(
    sql_uri,
    aligned_volume_name,
    table_name,
    pcg_table_name,
    schema_name,
    dataframe,
    chunksize,
):
    data_mapping = {}

    client = DynamicAnnotationClient(aligned_volume_name, sql_uri)
    mat_client = DynamicMaterializationClient(aligned_volume_name, sql_uri)
    schema = get_schema(schema_name)

    AnnotationModel = client.get_annotation_model(aligned_volume, table_name)
    SegmentationModel = mat_client.get_segmentation_model(
        aligned_volume, table_name, pcg_table_name
    )

    anno_cols = AnnotationModel.__table__.columns.keys()
    seg_cols = SegmentationModel.__table__.columns.keys()

    if isinstance(dataframe, pd.DataFrame):
        data_cols = dataframe.columns.tolist()
        anno_matching = common_column_set(anno_cols, data_cols)
        seg_matching = common_column_set(seg_cols, data_cols)
        data_mapping["position_keys"] = anno_matching
        data_mapping["segmentation_keys"] = seg_matching

    data_mapping.update({"anno_model": anno_cols, "seg_model": seg_cols})
    table_id = build_table_id(aligned_volume, table_name)

    results = process_dataframe(schema, schema_name, dataframe, data_mapping, chunksize)

    client_engine = client.engine
    insert_start = time.time()
    for data in results:
        copy_string_iterator(
            client_engine, str(AnnotationModel.__table__), anno_cols, data[1]
        )

        copy_string_iterator(
            client_engine, str(SegmentationModel.__table__), seg_cols, data[2]
        )
    insert_end = time.time()
    print(f"INSERT TIME {insert_end-insert_start}")

class StringIteratorIO(TextIOBase):
    """https://stackoverflow.com/a/12604375/2221667"""

    def __init__(self, iter: Iterator[str]):
        self._iter = iter
        self._buffer = ""

    def readable(self) -> bool:
        return True

    def _read_single_line(self, n: Optional[int] = None) -> str:
        while not self._buffer:
            try:
                self._buffer = next(self._iter)
            except StopIteration:
                break
        ret = self._buffer[:n]
        self._buffer = self._buffer[len(ret) :]
        return ret

    def read(self, n: Optional[int] = None) -> str:
        line = []
        if n is None or n < 0:
            while True:
                m = self._read_single_line()
                if not m:
                    break
                line.append(m)
        else:
            while n > 0:
                m = self._read_single_line(n)
                if not m:
                    break
                n -= len(m)
                line.append(m)
        return "".join(line)

def clean_csv_value(value: Optional[Any]) -> str:
    if value is None:
        return r"\N"
    return str(value).replace("\n", "\\n")

def copy_string_iterator(
    engine,
    table_id: str,
    columns,
    annotations: Iterator[Dict[str, Any]],
    size: int = 8192,
) -> None:
    """Inspired by https://hakibenita.com/fast-load-data-python-postgresql"""
    connection = engine.connect().connection
    with connection.cursor() as cursor:

        data_iterator = StringIteratorIO(
            (
                ",".join(map(clean_csv_value, (data.values()))) + "\n"
                for data in annotations
            )
        )

        cursor.copy_from(data_iterator, table_id, sep=",", size=size, columns=columns)

    connection.commit()

if __name__ == "__main__":

    #####   GENERATE DATA
    import pandas as pd
    import numpy as np

    df = pd.DataFrame()

    n_samples = 1_000  # number of synapses to make

    pre_coords = np.random.randint(0, 1000, size=(n_samples, 3))
    ctr_coords = np.random.randint(0, 1000, size=(n_samples, 3))
    post_coords = np.random.randint(0, 1000, size=(n_samples, 3))

    df["pre_pt_position"] = list(pre_coords.tolist())
    df["ctr_pt_position"] = list(ctr_coords.tolist())
    df["post_pt_position"] = list(post_coords.tolist())

    pre_root_id = np.repeat(np.arange(100), 10)
    post_root_id = np.repeat(np.arange(100), 10)
    df["pre_pt_root_id"] = pre_root_id
    df["post_pt_root_id"] = post_root_id
    df["pre_pt_supervoxel_id"] = np.random.randint(1, 100000, size=(n_samples, 1))
    df["post_pt_supervoxel_id"] = np.random.randint(1, 100000, size=(n_samples, 1))
    df.to_csv("synapses_data.csv", index=False)

    synapse_df = pd.read_csv("synapses_data.csv")

    from ast import literal_eval

    # CONVERT csv strings to lists
    synapse_df["pre_pt_position"] = synapse_df.pre_pt_position.apply(literal_eval)
    synapse_df["ctr_pt_position"] = synapse_df.ctr_pt_position.apply(literal_eval)
    synapse_df["post_pt_position"] = synapse_df.post_pt_position.apply(literal_eval)

    sql_uri = "postgres://postgres:annodb@localhost:5432/minnie"

    # table params and metadata
    aligned_volume = "minnie"
    table_name = "synapse_test"
    schema_name = "synapse"
    description = "This is an example description for this table"
    user_id = "foo@bar.com"
    # segmentation table params
    pcg_table_name = "pcg_synapse"
    # initialize dynamic annotation clients
    anno_client = DynamicAnnotationClient(aligned_volume, sql_uri)
    mat_client = DynamicMaterializationClient(aligned_volume, sql_uri)

    # generate table_id for annotation and segmentation tables
    annotation_table_id = build_table_id(aligned_volume, table_name)
    segmentation_table_id = build_segmentation_table_id(
        aligned_volume, table_name, pcg_table_name
    )
    # check if tables exist if not make them
    if annotation_table_id not in anno_client._get_existing_table_ids():
        anno_table = anno_client.create_table(
            table_name, schema_name, description, user_id
        )
    tables = anno_client._get_existing_table_ids()

    # create segmentation table
    seg_table = mat_client.create_and_attach_seg_table(table_name, pcg_table_name)

    tables = anno_client._get_existing_table_ids()
    print(tables)

    test_sqlalchemy_orm_bulk_insert(
        sql_uri,
        aligned_volume,
        table_name,
        pcg_table_name,
        schema_name,
        synapse_df,
        100,
    )

ext: py

max_stars_repo_name: fcollman/AnnotationEngine

max_issues_repo_name: fcollman/AnnotationEngine

max_forks_repo_name: fcollman/AnnotationEngine

reason: import numpy


text:
#!/usr/bin/env python3
"""
Deep Neural Network Class
"""
import numpy as np

class DeepNeuralNetwork:
    """
    Defines a a deep neural network performing binary classification
    """

    def __init__(self, nx, layers):
        """
        Constructor method
        ------------------

        nx: it's the number of input features to the neuron
        layers: it's a list representing the number of nodes in each layer of
            the network
        L: The number of layers in the neural network.
        cache: A dictionary to hold all intermediary values of the network.
            Upon instantiation, it should be set to an empty dictionary
        weights: A dictionary to hold all weights and biased of the network.
        """
        if type(nx) is not int:
            raise TypeError('nx must be an integer')
        if nx < 1:
            raise ValueError('nx must be a positive integer')
        if type(layers) is not list or len(layers) == 0:
            raise TypeError('layers must be a list of positive integers')
        self.nx = nx
        self.layers = layers
        self.L = len(layers)
        self.cache = {}
        self.weights = {}
        lys = layers
        randn = np.random.randn
        for i in range(len(lys)):
            if (type(lys[i]) is not int) or (lys[i] <= 0):
                raise TypeError('layers must be a list of positive integers')
            if i == 0:
                self.weights['W1'] = randn(lys[i], nx) * np.sqrt(2 / nx)
            else:
                k = "W{}".format(i + 1)
                h = i - 1
                self.weights[k] = randn(lys[i], lys[h]) * np.sqrt(2 / lys[h])
            self.weights["b{}".format(i + 1)] = np.zeros((lys[i], 1))


text:
# modify this for your own path
SCRIPT_PATH = '/Users/jingyaxun/Documents/research/tennis_prediction/tennis_match_prediction/build_datasets/sackmann'
TOUR = 'atp'
COUNT = False
START_YEAR = 2000
ONLY_PBP = 1
DATE = '11_26'

import sys
sys.path.insert(0,SCRIPT_PATH)
import tennisGameProbability,tennisMatchProbability,tennisSetProbability,tennisTiebreakProbability
from tennisMatchProbability import matchProb
from helper_functions import *
from data_functions import *
import pandas as pd
pd.options.mode.chained_assignment = None
import numpy as np
import re
import math
import copy

if __name__=='__main__':
	print 'main'
	atp_year_list = []
	for i in xrange(2001,2019):
	    atp_year_list.append(pd.read_csv("../my_data/matches/"+TOUR+"_matches_{0}.csv".format(i)))
	df = pd.concat(atp_year_list, ignore_index = True)

	# these may be changes specific to atp dataframe; normalize_name() is specific to atp/wta...
	df = df.rename(columns={'winner_name':'w_name','loser_name':'l_name','tourney_id':'tny_id',\
	                        'tourney_name':'tny_name','tourney_date':'tny_date'})
	df['w_name'] = [normalize_name(x,tour=TOUR) for x in df['w_name']]
	df['l_name'] = [normalize_name(x,tour=TOUR) for x in df['l_name']]
	df['tny_name'] = ['Davis Cup' if 'Davis Cup' in s else s for s in df['tny_name']]
	df['tny_name'] = [s.replace('Australian Chps.','Australian Open').replace('Australian Open-2',\
	            'Australian Open').replace('U.S. National Chps.','US Open') for s in df['tny_name']]

	ret_strings = ('ABN','DEF','In Progress','RET','W/O',' RET',' W/O','nan','walkover')
	abd_strings = ('abandoned','ABN','ABD','DEF','def','unfinished','Walkover')
	atp_all_matches = format_match_df(df,ret_strings=ret_strings,abd_strings=abd_strings)

	# generate tourney stats from one year behind START_DATE for stats_52
	# get elo with constant and dynamic K
	start_ind = atp_all_matches[atp_all_matches['match_year']>=START_YEAR-1].index[0]
	atp_all_matches = generate_elo(atp_all_matches,0)
	atp_all_matches = generate_elo(atp_all_matches,1)
	atp_all_matches = generate_52_stats(atp_all_matches,start_ind)
	atp_all_matches = generate_52_adj_stats(atp_all_matches,start_ind)
	atp_all_matches = generate_tny_stats(atp_all_matches,start_ind)
	#print 'adj stats: ', atp_all_matches[atp_all_matches['match_year']==2014][['w_52_s_adj','w_52_r_adj']]
	#print 'now: ', atp_all_matches[['match_year','match_month','w_name','l_name','w_52_s_adj','l_52_s_adj','l_52_svpt']].loc[137969]

	# Combine all the matches that have pbp (point by point) information into one dataframe
	# and clean up columns in preparation for merging with all_atp_matches
	pbp_matches_archive = pd.read_csv("../my_data/pbp/pbp_matches_atp_main_archive.csv")
	pbp_matches_archive_old = pd.read_csv("../my_data/pbp/pbp_matches_atp_main_archive_old.csv")
	pbp_matches_current = pd.read_csv("../my_data/pbp/pbp_matches_atp_main_current.csv")
	pbp_matches = pd.concat([pbp_matches_archive_old.loc[:932],pbp_matches_archive,pbp_matches_current])
	pbp_matches.winner = pbp_matches.winner - 1
	pbp_matches = pbp_matches.reset_index(); del pbp_matches['index']
	pbp_matches = format_pbp_df(pbp_matches,tour=TOUR)

	# dictionary with each key as 'w_name'+'l_name'+'match_year'+'score' to connect pbp
	# strings to atp_all_matches (I removed parentheses terms from tb scores)
	duplicates = ['Janko Tipsarevic Kei Nishikori 2011 6-4 6-4','Robin Soderling Michael Berrer 2011 6-3 7-6',
	        'Juan Martin Kevin Anderson 2011 6-4 6-4','Philipp Kohlschreiber Mikhail Youzhny 2011 6-4 6-2',
	        'Philipp Kohlschreiber Olivier Rochus 2012 6-1 6-4','Viktor Troicki Radek Stepanek 2012 2-6 6-4 6-3',
	        'Gilles Simon Grigor Dimitrov 2012 6-3 6-3','Alexandr Dolgopolov Gilles Simon 2012 6-3 6-4',
	        'Fabio Fognini Tommy Haas 2013 6-2 6-4','Richard Gasquet Florian Mayer 2013 6-3 7-6',
	        'Novak Djokovic Rafael Nadal 2013 6-3 6-4','Tomas Berdych Gael Monfils 2015 6-1 6-4',
	        'Novak Djokovic Rafael Nadal 2015 6-3 6-3']
	collision_d = dict(zip(duplicates,[0]*len(duplicates)))

	# connects the two dataframes on match keys and reformats columns fro w/l to p0/p1
	cols = ['_name','_elo','_sf_elo','_elo_538','_sf_elo_538','_52_swon','_52_svpt','_52_rwon',\
        '_52_rpt','_sf_52_swon','_sf_52_svpt','_sf_52_rwon','_sf_52_rpt','_52_s_adj','_52_r_adj']
	df = connect_df(match_df=atp_all_matches,pbp_df=pbp_matches,col_d=collision_d,player_cols=cols,\
	                start_year=START_YEAR)

	df['elo_diff'] = [df['p0_elo'][i] - df['p1_elo'][i] for i in xrange(len(df))]
	df['sf_elo_diff'] = [df['p0_sf_elo'][i] - df['p1_sf_elo'][i] for i in xrange(len(df))]
	df['elo_diff_538'] = [df['p0_elo_538'][i] - df['p1_elo_538'][i] for i in xrange(len(df))]
	df['sf_elo_diff_538'] = [df['p0_sf_elo_538'][i] - df['p1_sf_elo_538'][i] for i in xrange(len(df))]

	# generate win probabilities from logit of elo/s_elo 538 differences, trained on 2011-2013 data
	df = generate_logit_probs(df,cols=['elo_diff_538','sf_elo_diff_538'],col_name='logit_elo_538_prob')
	# df = generate_logit_probs(df,cols=['elo_diff','sf_elo_diff'],col_name='logit_elo_prob')
	# df = generate_logit_probs(df,cols=['elo_diff'],col_name='logit_elo_diff_prob')
	# df = generate_logit_probs(df,cols=['elo_diff_538'],col_name='logit_elo_diff_538_prob')

	# dataframe with only matches that have pbp
	if ONLY_PBP:
	    df = df[df['pbp']!='None']
	else:
	    df = df[df['winner']!='None']
	df = df.reset_index(drop=True)

	cols = ['52_s_adj','52_r_adj']
	test_df = generate_JS_stats(df,cols)

	# keep relevant columns
	df = df[['tny_id','tny_name','surface','tny_date','match_year','match_month',
	         u'p0_name', u'p1_name', u'p0_elo',
	         u'p1_elo', u'p0_sf_elo', u'p1_sf_elo', u'p0_elo_538', u'p1_elo_538',
	         u'p0_sf_elo_538', u'p1_sf_elo_538', u'p0_52_swon',u'p0_52_svpt',
	         u'p1_52_swon', u'p1_52_svpt', u'p0_52_rwon', u'p0_52_rpt',
	         u'p1_52_rwon', u'p1_52_rpt',
	         u'elo_diff', u'sf_elo_diff',
	         u'elo_diff_538', u'sf_elo_diff_538',
	         u'p0_s_pct', u'p0_r_pct', u'p1_s_pct', u'p1_r_pct',
	         u'p0_s_pct_JS', u'p1_s_pct_JS', u'p0_r_pct_JS', u'p1_r_pct_JS',
	         u'p0_sf_52_swon', u'p0_sf_52_svpt',u'p1_sf_52_swon', u'p1_sf_52_svpt',
	         u'p0_sf_52_rwon', u'p0_sf_52_rpt', u'p1_sf_52_rwon', u'p1_sf_52_rpt',
	         u'p0_sf_s_pct', u'p0_sf_r_pct', u'p1_sf_s_pct', u'p1_sf_r_pct',
	         u'p0_sf_s_pct_JS', u'p1_sf_s_pct_JS', u'p0_sf_r_pct_JS', u'p1_sf_r_pct_JS',
	         u'p0_52_s_adj',u'p0_52_r_adj',u'p1_52_s_adj',u'p1_52_r_adj',
	         u'p0_52_s_adj_JS',u'p0_52_r_adj_JS',u'p1_52_s_adj_JS',u'p1_52_r_adj_JS',
	         u'avg_52_s', u'avg_52_r', u'sf_avg_52_s', u'sf_avg_52_r',
	         'tny_stats','best_of','score','pbp',
	         'logit_elo_538_prob', #'logit_elo_prob','logit_elo_diff_prob','logit_elo_diff_538_prob',
	         'winner']]

	# binary indicator for whether player 0 won
	df['winner'] = [1-winner for winner in df['winner']]

	# generate serving probabilities for Klaassen-Magnus model
	df['match_id'] = range(len(df))
	df['tny_stats'] = [df['avg_52_s'][i] if df['tny_stats'][i]==0 else df['tny_stats'][i] for i in xrange(len(df))]
	df['p0_s_kls'] = df['tny_stats']+(df['p0_s_pct']-df['avg_52_s']) - (df['p1_r_pct']-df['avg_52_r'])
	df['p1_s_kls'] = df['tny_stats']+(df['p1_s_pct']-df['avg_52_s']) - (df['p0_r_pct']-df['avg_52_r'])
	df['p0_s_kls_JS'] = df['tny_stats']+(df['p0_s_pct_JS']-df['avg_52_s']) - (df['p1_r_pct_JS']-df['avg_52_r'])
	df['p1_s_kls_JS'] = df['tny_stats']+(df['p1_s_pct_JS']-df['avg_52_s']) - (df['p0_r_pct_JS']-df['avg_52_r'])
	df['p0_sf_s_kls'] = df['tny_stats']+(df['p0_sf_s_pct']-df['sf_avg_52_s']) - (df['p1_sf_r_pct']-df['sf_avg_52_r'])
	df['p1_sf_s_kls'] = df['tny_stats']+(df['p1_sf_s_pct']-df['sf_avg_52_s']) - (df['p0_sf_r_pct']-df['sf_avg_52_r'])
	df['p0_sf_s_kls_JS'] = df['tny_stats']+(df['p0_sf_s_pct_JS']-df['sf_avg_52_s']) - (df['p1_sf_r_pct_JS']-df['sf_avg_52_r'])
	df['p1_sf_s_kls_JS'] = df['tny_stats']+(df['p1_sf_s_pct_JS']-df['sf_avg_52_s']) - (df['p0_sf_r_pct_JS']-df['sf_avg_52_r'])
	df['p0_s_kls_adj'] = df['tny_stats']+(df['p0_52_s_adj']) - (df['p1_52_r_adj'])
	df['p1_s_kls_adj'] = df['tny_stats']+(df['p1_52_s_adj']) - (df['p0_52_r_adj'])
	df['p0_s_kls_adj_JS'] = df['tny_stats']+(df['p0_52_s_adj_JS']) - (df['p1_52_r_adj_JS'])
	df['p1_s_kls_adj_JS'] = df['tny_stats']+(df['p1_52_s_adj_JS']) - (df['p0_52_r_adj_JS'])

	# generate match probabilities and z-scores for Klaassen method, with and w/o JS estimators
	df['match_prob_kls'] = [matchProb(row['p0_s_kls'],1-row['p1_s_kls']) for i,row in df.iterrows()]
	df['match_prob_kls_JS'] = [matchProb(row['p0_s_kls_JS'],1-row['p1_s_kls_JS']) for i,row in df.iterrows()]
	df['match_prob_sf_kls'] = [matchProb(row['p0_sf_s_kls'],1-row['p1_sf_s_kls']) for i,row in df.iterrows()]
	df['match_prob_sf_kls_JS'] = [matchProb(row['p0_sf_s_kls_JS'],1-row['p1_sf_s_kls_JS']) for i,row in df.iterrows()]
	df['match_prob_adj_kls'] = [matchProb(row['p0_s_kls_adj'],1-row['p1_s_kls_adj']) for i,row in df.iterrows()]
	df['match_prob_adj_kls_JS'] = [matchProb(row['p0_s_kls_adj_JS'],1-row['p1_s_kls_adj_JS']) for i,row in df.iterrows()]

	# generate win probabilities from elo differences
	df['elo_prob'] = [(1+10**(diff/-400.))**-1 for diff in df['elo_diff']]
	df['elo_prob_538'] = [(1+10**(diff/-400.))**-1 for diff in df['elo_diff_538']]
	df['sf_elo_prob'] = [(1+10**(diff/-400.))**-1 for diff in df['sf_elo_diff']]
	df['sf_elo_prob_538'] = [(1+10**(diff/-400.))**-1 for diff in df['sf_elo_diff_538']]

	# elo-induced serve percentages
	df = generate_elo_induced_s(df, 'elo',start_ind=0)
	df = generate_elo_induced_s(df, 'logit_elo_538',start_ind=0)

	# depending on ONLY_PBP, this will have point-by-point matches, or all
	# tour-level matches from START_DATE to present
	name = 'elo_pbp_with_surface_'+DATE if ONLY_PBP else 'elo_atp_matches_all_'+DATE
	substr = '_dynamic_rating_tny_level_wrong'
	print name + '.csv saved to my_data'
	df.to_csv('../my_data/'+name+substr+'.csv')


text:
import numpy as np
from bico.geometry.squared_euclidean import SquaredEuclideanDistance
from bico.nearest_neighbor.base import NearestNeighbor, NearestNeighborResult
from nearpy import Engine
from nearpy.filters import DistanceThresholdFilter
from nearpy.hashes import RandomBinaryProjectionTree

class RandomBinaryTreeNN(NearestNeighbor):
    """ Nearest neighbor implementation by using random binary trees from nearpy package """
    def __init__(self, dimension: int, number_projections: int, threshold: float):
        """
        :param dimension:
            Number of dimensions of input points
        :param number_projections:
            Number of random projections used for finding nearest neighbors.
            Trade-off: More projections result in a smaller number of false positives in candidate set
        :param threshold:
            Distance threshold for definition nearest: all points within this specific distance
        """
        self.rbpt = RandomBinaryProjectionTree('rbpt', number_projections, 1)
        self.sqdist = SquaredEuclideanDistance()
        self.ann_engine = Engine(dimension, lshashes=[self.rbpt], distance=self.sqdist,
                                 vector_filters=[DistanceThresholdFilter(threshold)])

    def insert_candidate(self, point: np.ndarray, metadata):
        self.ann_engine.store_vector(point, data=metadata)

    def get_candidates(self, point: np.ndarray):
        return [NearestNeighborResult(res[0], res[1], res[2])
                for res in self.ann_engine.neighbours(point)]

max_stars_repo_name: gallmerci/bico

max_issues_repo_name: gallmerci/bico

max_forks_repo_name: gallmerci/bico


text:
# -*- coding: utf-8 -*-
"""
@Author : Horizon
@Date   : 2021-03-20 00:30:23
"""

# This code is used to test the time cost to load a video and transform it into PILImage.

# Methods to be tested
import cv2
from PIL import Image
import skvideo.io
import kornia
import skimage
import imageio
import torchvision
from vidgear.gears import CamGear
import av

# Other library
import torch
from torchvision import transforms

import numpy as np

import os
import time

#from rich.console import Console
#from rich.table import Table

def cv2_loadvideo(video_path):

    cap = cv2.VideoCapture(video_path)

    assert cap.isOpened()

    frames_tensor = []

    while(True):

        ret, frame = cap.read()

        # read over
        if not ret:
            break

        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        #frame = cv2.resize(frame, (224, 224))
        #frame = transform(frame).unsqueeze(0)

        frames_tensor.append(frame)

    # note: please use list to append different frame and concate them finally
    #       or else it will take muck longer time if they are concated after one frame has been just loaded
    frames_tensor = np.asarray_chkfinite(frames_tensor, dtype=np.uint8)
    frames_tensor = kornia.image_to_tensor(frames_tensor, keepdim=False).div(255.0)
    #frames_tensor = torch.cat(frames_tensor, 0)

    #print(frames_tensor.shape)

    return

def cv2_to_PIL_loadvideo(video_path):

    cap = cv2.VideoCapture(video_path)

    assert cap.isOpened()

    frames_tensor = []

    transform = transforms.Compose([
                                    #transforms.ToPILImage(),
                                    #transforms.Resize((224, 224)),
                                    transforms.ToTensor(),
                                    ])

    while(True):

        ret, frame = cap.read()

        # read over
        if not ret:
            break

        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame = Image.fromarray(frame)
        frame = transform(frame).unsqueeze(0)

        frames_tensor.append(frame)

    # note: please use list to append different frame and concate them finally
    #       or else it will take muck longer time if they are concated after one frame has been just loaded
    frames_tensor = torch.cat(frames_tensor, 0)

    #print(frames_tensor.shape)

    return

def sk_video_loadvideo(video_path):

    videodata = skvideo.io.vreader(video_path)

    frames_tensor = []

    for frame in videodata:

        #frame = transform(frame).unsqueeze(0)

        frames_tensor.append(frame)

    # note: please use list to append different frame and concate them finally
    #       or else it will take muck longer time if they are concated after one frame has been just loaded
    frames_tensor = np.asarray_chkfinite(frames_tensor, dtype=np.uint8)
    frames_tensor = kornia.image_to_tensor(frames_tensor, keepdim=False).div(255.0)
    #frames_tensor = torch.cat(frames_tensor, 0)

    #print(frames_tensor.shape)

    return

def imageio_loadvideo(video_path):

    reader = imageio.get_reader(video_path)

    frames_tensor = []

    for frame in reader:

        frame = skimage.img_as_float(frame).astype(np.float32)

        frames_tensor.append(frame)

    # note: please use list to append different frame and concate them finally
    #       or else it will take muck longer time if they are concated after one frame has been just loaded
    frames_tensor = np.asarray_chkfinite(frames_tensor, dtype=np.uint8)
    frames_tensor = kornia.image_to_tensor(frames_tensor, keepdim=False).div(255.0)

    #print(frames_tensor.shape)

    return

def vidgear_loadvideo(video_path):

    stream = CamGear(source=video_path).start()

    frames_tensor = []

    while True:

        frame = stream.read()

        if frame is None:
            break

        frames_tensor.append(frame)

    # note: please use list to append different frame and concate them finally
    #       or else it will take muck longer time if they are concated after one frame has been just loaded
    frames_tensor = np.asarray_chkfinite(frames_tensor, dtype=np.uint8)
    frames_tensor = kornia.image_to_tensor(frames_tensor, keepdim=False).div(255.0)

    #print(frames_tensor.shape)

    stream.stop()

    return

def pyav_loadvideo(video_path):

    container = av.open(video_path)

    frames_tensor = []

    # !!! This is the only difference.
    container.streams.video[0].thread_type = 'AUTO'

    for packet in container.demux():
        for frame in packet.decode():

            frame = frame.to_image()  # PIL/Pillow image
            frame = np.asarray(frame)  # numpy array
            # Do something!
            frames_tensor.append(frame)

    # note: please use list to append different frame and concate them finally
    #       or else it will take muck longer time if they are concated after one frame has been just loaded
    frames_tensor = np.asarray_chkfinite(frames_tensor, dtype=np.uint8)
    frames_tensor = kornia.image_to_tensor(frames_tensor, keepdim=False).div(255.0)

    #print(frames_tensor.shape)
    container.close()

    return

def load_frames(video_path):

    video_dir = video_path.split('.')[0]

    frame_paths = os.listdir(video_dir)

    frames_tensor = []

    for frame_path in frame_paths:
        frame = cv2.imread(os.path.join(video_dir, frame_path))

        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        #frame = cv2.resize(frame, (224, 224))

        frames_tensor.append(frame)

    # note: please use list to append different frame and concate them finally
    #       or else it will take muck longer time if they are concated after one frame has been just loaded
    frames_tensor = np.asarray_chkfinite(frames_tensor, dtype=np.uint8)
    frames_tensor = kornia.image_to_tensor(frames_tensor, keepdim=False).div(255.0)

    #print(frames_tensor.shape)

    return

def torchio_loadvideo(video_path):

    frames_tensor, _, _ = torchvision.io.read_video(video_path, pts_unit='sec')

    frames_tensor = frames_tensor.div(255.0).permute((0, 3, 1, 2))

    #print(frames_tensor.shape, frames_tensor.max(), frames_tensor.min())

    return

def test(video_path, method):

    start_time = time.time()

    method(video_path)

    end_time = time.time()

    return end_time - start_time

def task(method):

    time_short = test("test_videos/short_video.avi", method)
    time_long  = test("test_videos/long_video.avi" , method)
    time_small = test("test_videos/small_video.avi", method)
    time_big   = test("test_videos/big_video.avi"  , method)

    return time_short, time_long, time_small, time_big

if(__name__ == "__main__"):

    '''

    console = Console()

    table = Table(show_header=True, header_style="bold magenta")
    table.add_column("method"                , justify="center")
    table.add_column("initial datatype"      , justify="center")
    table.add_column("intermediate  datatype", justify="center")
    table.add_column("final datatype"        , justify="center")
    table.add_column("short video cost /s"   , justify="center")
    table.add_column("long video cost /s"    , justify="center")
    table.add_column("small video cost /s"   , justify="center")
    table.add_column("big video cost /s"     , justify="center")

    '''

    #1.opencv → Tensor
    time_short, time_long, time_small, time_big = task(cv2_loadvideo)
    print("opencv: ndarray → Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.addrow("opencv", "ndarray", "-", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #2.opencv → PIL → Tensor
    time_short, time_long, time_small, time_big = task(cv2_to_PIL_loadvideo)
    print("opencv: ndarray → PILImage → Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.add_row("opencv", "ndarray", "PILImage", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #3.scikit-video → Tensor
    time_short, time_long, time_small, time_big = task(sk_video_loadvideo)
    print("scikit-video: ndarray → Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.add_row("scikit-video", "ndarray", "-", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #4.imageio → Tensor
    time_short, time_long, time_small, time_big = task(imageio_loadvideo)
    print("imageio: imageio.core.util.Array → ndarray → Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.add_row("imageio", "imageio.core.util.Array", "ndarray", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #5.vidgear → Tensor
    time_short, time_long, time_small, time_big = task(vidgear_loadvideo)
    print("vidgear: ndarray → Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.add_row("vidgear", "ndarray", "-", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #6.pyav → Tensor
    time_short, time_long, time_small, time_big = task(pyav_loadvideo)
    print("pyav: av.video.frame.VideoFrame → PILImage → ndarray → Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.add_row("pyav", "av.video.frame.VideoFrame", "PILImage → ndarray", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #7.frames → Tensor
    time_short, time_long, time_small, time_big = task(load_frames)
    print("frames: ndarray → Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.add_row("frames", "ndarray", "-", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #8.torchvision.io → Tensor
    time_short, time_long, time_small, time_big = task(torchio_loadvideo)
    print("torchvision.io: Tensor")
    print("{:2.2f} {:2.2f} {:2.2f} {:2.2f}".format(time_short, time_long, time_small, time_big))
    print()
    #table.add_row("torchvision.io", "Tensor", "-", "Tensor",
    #                '{:.2f}'.format(time_short), '{:.2f}'.format(time_long), '{:.2f}'.format(time_small), '{:.2f}'.format(time_big))

    #console.print(table)


text:
#!/usr/bin/env python
from __future__ import print_function
from snpy import CSPtemp
import os,sys
from numpy import *
from pygplot import *
import pickle
import scipy.interpolate as inter

base = os.path.dirname(CSPtemp.__file__)
tck_file = "bs_tck.pickle"

CSPtemp.use_gloes=1
temp = CSPtemp.template()

dm15_low = 0.6
dm15_high = 2.0
t_low = -10.0
t_high = 70

bands = ['u','B','g','V','r','i','Y','J','H']
dm15s = arange(31)/30.0*(dm15_high-dm15_low) + dm15_low
ts = arange(81)/80.0*(t_high-t_low) + t_low

data0 = {}
list0 = {}
SNe = {}
for band in bands:
   data0[band],list0[band] = CSPtemp.dm15tempc.get_data(band)
   list0[band] = array(list0[band])
   SNe[band] = list({}.fromkeys(list0[band]).keys())
   SNe[band].sort()

tck = {}.fromkeys(bands)
for b in bands:
   tck[b] = []
   tck['e_'+b] = []

for j in range(50):
   print("iteration ", j)
   dm15_mat = []
   t_mat = []
   z_mat = {}
   ez_mat = {}
   for band in bands:
      z_mat[band] = []
      ez_mat[band] = []
   for i in range(len(dm15s)):
      sys.stdout.write('%.2f ' % dm15s[i])
      sys.stdout.flush()
      temp.mktemplate(dm15s[i])
      dm15_mat.append(ts*0 + dm15s[i])
      t_mat.append(ts)
      for band in bands:
         N = len(SNe[band])
         randints = random.randint(0, N, size=(N,))
         data1 = []
         sn_list = [SNe[band][id] for id in randints]
         masks = [list0[band] == sn for sn in sn_list]
         data1 = concatenate([compress(m, data0[band], axis=1) for m in masks],
               axis=1)
         list1 = concatenate([list0[band][m] for m in masks])
         list1 = list1.tolist()
         CSPtemp.dm15tempc.set_data(band, data1, list1)

         z,ez,mask = temp.eval(band, ts, mag=0)
         z_mat[band].append(z)
         ez_mat[band].append(ez)

   x = ravel(array(t_mat))
   y = ravel(array(dm15_mat))
   smooths = {}
   for band in bands:
      smooths[band] = 0.1
      z = ravel(array(z_mat[band]))
      ez = ravel(array(ez_mat[band]))
      print("fitting band ",band)
      print(inter.__file__)
      try:
         tck[band].append(inter.bisplrep(x, y, z, w=1.0/ez, s=smooths[band]*len(x)))
      except:
         continue
      tck["e_"+band].append(inter.bisplrep(x, y, ez, kx=1, ky=1, s=0.0*len(x)))
f = open(tck_file, 'w')
pickle.dump(tck, f)
f.close()

max_stars_repo_name: emirkmo/snpy

max_issues_repo_name: emirkmo/snpy

max_forks_repo_name: emirkmo/snpy

reason: from numpy,import scipy


text:
import os
import numpy as np
import cv2

class DataLoader:

    def __init__(self, shuffle=False):
        self.datadir = os.listdir('./dataset')
        print(len(self.datadir))
        self.labels = []
        self.imgs = np.empty((len(self.datadir), 500, 500, 3))
        self.shuffle = shuffle

    def create(self, ratio1=0.7, ratio2=0.9):
        print('create')
        for idx, img in enumerate(self.datadir):
            temp_label = self.datadir[idx].split('_')[0]
            self.labels.append(int(temp_label))

            img = cv2.imread('./dataset/'+img, cv2.IMREAD_COLOR)
            self.imgs[idx] = img

        data = self.imgs
        labels = self.labels

        if self.shuffle:
            shuffle = np.arange(len(self.datadir))
            np.random.shuffle(shuffle)

            data = self.imgs[shuffle]
            labels = self.labels[shuffle]

        split1 = int(ratio1*data.shape[0])
        split2 = int(ratio2*data.shape[0])

        [train_data, validation_data, test_data] = np.split(data, [split1, split2])
        [train_label, validation_label, test_label] = np.split(labels, [split1, split2])
        # print(test_label)

        np.savez('./results/train', data=train_data, label=train_label)
        np.savez('./results/validation', data=validation_data, label=validation_label)
        np.savez('./results/test', data=test_data, label=test_label)

    def load(self):
        print('load')
        train_set = np.load('./results/train.npz')
        validation_set = np.load('./results/validation.npz')
        test_set = np.load('./results/test.npz')

        return train_set, validation_set, test_set

if __name__ == "__main__":
    dl =DataLoader()
    # dl.create()
    dl.load()

max_stars_repo_name: dhleekr/test

max_issues_repo_name: dhleekr/test

max_forks_repo_name: dhleekr/test


text:
# python3 extractor.py
import os
import cv2
import numpy as np
from scipy import stats
from pathlib import Path

def main():
    num_imgs = 50000
    # The output dir for storing the extracted imgs
    output_dir = "extracted"
    os.makedirs(output_dir, exist_ok=True)

    # The location of the root directory to extract videos from
    root_dir = "data"

    num_total_frames = 0  # accumulated total frames
    record_total_frames = []  # list of total frames
    video_files = []  # list of video files
    for video_dir in Path(root_dir).iterdir():
        for video_file_path in video_dir.iterdir():
            # If the file is a hidden file
            if video_file_path.stem[0] == ".":
                continue

            # cv2.VideoCapture requires a string argument
            video_file = str(video_file_path)
            video_files.append(video_file)

            captured_video = cv2.VideoCapture(video_file)

            # Type convert: float -> int
            num_frames = int(captured_video.get(cv2.CAP_PROP_FRAME_COUNT))  # total frames

            record_total_frames.append(num_frames)
            num_total_frames += num_frames

    pr_dist = [total_frames / num_total_frames for total_frames in record_total_frames]  # probability distribution

    # Construct specific distribution for discrete random variables
    rnd_gen = stats.rv_discrete(
        values=(np.arange(len(record_total_frames)), pr_dist)
    )
    # Count number of occurrences of each value in array of non-negative ints
    counters = np.bincount(rnd_gen.rvs(size=num_imgs))

    # Add zeros for the tailing missing values of 'counters'
    if len(video_files) != len(counters):
        for i in range(len(video_files) - len(counters)):
            counters = np.append(counters, 0)

    for video_file, num_frames in zip(video_files, counters):
        captured_video = cv2.VideoCapture(video_file)

        # Type convert: float -> int
        num_frames = int(captured_video.get(cv2.CAP_PROP_FRAME_COUNT))  # total frames

        # The No. of frames to read
        frame_nos = np.random.choice(num_frames, num_frames, replace=False)

        for frame_no in frame_nos:
            # Setting the next frame to read
            captured_video.set(cv2.CAP_PROP_POS_FRAMES, frame_no)

            # Extraction
            did_read_frame, img = captured_video.read()

            assert did_read_frame, "Read frame failed."

            # Output
            img_file = str(os.path.join(
                output_dir,
                Path(video_file).stem + "_" + str(frame_no) + ".jpg"
            ))
            cv2.imwrite(img_file, img)

if __name__ == "__main__":
    main()

max_stars_repo_name: hu-minghao/Anji-data

max_issues_repo_name: hu-minghao/Anji-data

max_forks_repo_name: hu-minghao/Anji-data

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
"""Copy of 03-Numpy Operations.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1JopIS7SFuJqtnvO3U6GgPxHS-clp-Smx

#Introduction to NumPy

![alt text](https://upload.wikimedia.org/wikipedia/commons/thumb/1/1a/NumPy_logo.svg/1200px-NumPy_logo.svg.png)

NumPy (or Numpy) is a Linear Algebra Library for Python. NumPy is a Python package. It stands for 'Numerical Python'. It is a library consisting of multidimensional array objects and a collection of routines for processing of array.

Numpy is also incredibly fast, as it has bindings to C libraries.

## **About iPython Notebooks**

iPython Notebooks are interactive coding environments embedded in a webpage. You will be using iPython notebooks in this class. You only need to write code between the ### START CODE HERE ### and ### END CODE HERE ### comments. After writing your code, you can run the cell by either pressing "SHIFT"+"ENTER" or by clicking on "Run Cell" (denoted by a play symbol) in the left bar of the cell.

**In this notebook you will learn -**
* Numpy arrays
* Numpy Selecting and Indexing
* Numpy Operations

## Using NumPy

To import NumPy in Colaboratory simply type the following:
"""

import numpy as np

"""# NumPy Operations

## Arithmetic

You can easily perform array with array arithmetic, or scalar with array arithmetic. Let's see some examples:
"""

import numpy as np
arr = np.arange(0,10)
arr

"""###Addition of arrays"""

arr + arr

"""###Multiplication of arrays"""

arr * arr

"""###Subtraction of arrays"""

arr - arr

"""###Division of arrays"""

# Warning on division by zero, but not an error!
# Just replaced with nan
arr/arr

# Also warning, but not an error instead infinity
1/arr

"""###Exponent"""

arr**3

"""## Universal Array Functions

Numpy comes with many universal array functions, which are essentially just mathematical operations you can use to perform the operation across the array. Let's see some common ones:
"""

#Taking Square Roots
np.sqrt(arr)

#Calcualting exponential (e^)
np.exp(arr)

np.max(arr) #same as arr.max()

np.sin(arr)

np.log(arr)

"""# Great Job!"""


text:
import numpy as np
import os
import tensorflow as tf
from tensorflow.keras.callbacks import Callback
from tensorflow.keras.preprocessing import image_dataset_from_directory
from tensorflow.keras import models
from picsellia.client import Client
from picsellia.pxl_exceptions import AuthenticationError
from sklearn.metrics import classification_report, confusion_matrix

if 'api_token' not in os.environ:
    raise AuthenticationError("You must set an api_token to run this image")

api_token = os.environ["api_token"]

if "host" not in os.environ:
    host = "https://app.picsellia.com/sdk/v2/"
else:
    host = os.environ["host"]

if "experiment_id" in os.environ:
    experiment_id = os.environ['experiment_id']
    project_token = os.environ['project_token']
    experiment = Client.Experiment(api_token=api_token, project_token=project_token, host=host, interactive=False)
    experiment.id = experiment_id
    exp = experiment.checkout(id=experiment_id, with_file=True)
else:
    if "experiment_name" in os.environ and "project_token" in os.environ:
        project_token = os.environ['project_token']
        experiment_name = os.environ['experiment_name']
        experiment = Client.Experiment(api_token=api_token, project_token=project_token, host=host, interactive=False)
        exp = experiment.checkout(experiment_name, with_file=True)
    else:
        raise AuthenticationError("You must either set the experiment id or the project token + experiment_name")

exp.dl_annotations()
exp.dl_pictures()
exp.train_test_split()
train_split = {
    'x': exp.categories,
    'y': exp.train_repartition,
    'image_list': exp.train_list_id
}
exp.log('train-split', train_split, 'bar', replace=True)

test_split = {
    'x': exp.categories,
    'y': exp.test_repartition,
    'image_list': exp.eval_list_id
}
exp.log('test-split', test_split, 'bar', replace=True)

labels_index = [e for e in range(1, len(exp.categories)+1)]
labelmap = dict(zip(labels_index, exp.categories))
exp.log('labelmap', labelmap, 'labelmap', replace=True)

splits = ["train", "validation"]
for split in splits:
    if not split in os.listdir("images"):
        os.mkdir(f"images/{split}")
    for category in exp.categories:
        if not category in os.listdir(f"images/{split}"):
            os.mkdir(f"images/{split}/{category}")

train_dir = "images/train"
validation_dir = "images/validation"

for image in exp.dict_annotations["annotations"]:
    label = image["annotations"][0]["label"]
    image_id = image["internal_picture_id"]
    filename = image["external_picture_url"]
    if image_id in exp.train_list_id:
        os.rename(f'images/{filename}', f"{train_dir}/{label}/{filename}")
    elif image_id in exp.eval_list_id:
        os.rename(f'images/{filename}', f"{validation_dir}/{label}/{filename}")

parameters = exp.get_data("parameters")

BATCH_SIZE = parameters["batch_size"]
IMG_SIZE = (parameters["image_size"], parameters["image_size"])

train_dataset = image_dataset_from_directory(train_dir,
                                             shuffle=True,
                                             batch_size=BATCH_SIZE,
                                             image_size=IMG_SIZE)
validation_dataset = image_dataset_from_directory(validation_dir,
                                                  shuffle=True,
                                                  batch_size=BATCH_SIZE,
                                                  image_size=IMG_SIZE)

class_names = train_dataset.class_names
val_batches = tf.data.experimental.cardinality(validation_dataset)
test_dataset = validation_dataset.take(val_batches // 5)
validation_dataset = validation_dataset.skip(val_batches // 5)
print('Number of validation batches: %d' % tf.data.experimental.cardinality(validation_dataset))
print('Number of test batches: %d' % tf.data.experimental.cardinality(test_dataset))

AUTOTUNE = tf.data.AUTOTUNE

train_dataset = train_dataset.prefetch(buffer_size=AUTOTUNE)
validation_dataset = validation_dataset.prefetch(buffer_size=AUTOTUNE)
test_dataset = test_dataset.prefetch(buffer_size=AUTOTUNE)

data_augmentation = tf.keras.Sequential([
  tf.keras.layers.experimental.preprocessing.RandomFlip('horizontal'),
  tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),
])

preprocess_input = tf.keras.applications.mobilenet_v2.preprocess_input
rescale = tf.keras.layers.experimental.preprocessing.Rescaling(1./127.5, offset= -1)

# Create the base model from the pre-trained model MobileNet V2
IMG_SHAPE = IMG_SIZE + (3,)

class PicselliaLogger(Callback):

    def on_epoch_end(self, epoch, logs={}):
        for log_name, log_value in logs.items():
            exp.log(log_name, [float(log_value)], 'line')

callback_list = [PicselliaLogger()]

fine_tune = parameters["fine_tune"]
if not fine_tune:
    base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,
                                                include_top=False,
                                                weights=os.path.join(exp.base_dir, "model.h5"))

    base_model.trainable = False
    base_model.summary()

    image_batch, label_batch = next(iter(train_dataset))
    feature_batch = base_model(image_batch)
    print(feature_batch.shape)

    global_average_layer = tf.keras.layers.GlobalAveragePooling2D()
    feature_batch_average = global_average_layer(feature_batch)
    print(feature_batch_average.shape)

    prediction_layer = tf.keras.layers.Dense(1)
    prediction_batch = prediction_layer(feature_batch_average)
    print(prediction_batch.shape)

    inputs = tf.keras.Input(shape=(parameters["image_size"], parameters["image_size"], 3))
    x = data_augmentation(inputs)
    x = preprocess_input(x)
    x = base_model(x, training=False)
    x = global_average_layer(x)
    x = tf.keras.layers.Dropout(0.2)(x)
    outputs = prediction_layer(x)
    model = tf.keras.Model(inputs, outputs)

    base_learning_rate = parameters["learning_rate"]
    model.compile(optimizer=tf.keras.optimizers.Adam(lr=base_learning_rate),
                loss=tf.keras.losses.CategoricalCrossentropy(),
                metrics=['accuracy'])

    model.summary()

    initial_epochs = parameters["initial_epochs"]

    loss0, accuracy0 = model.evaluate(validation_dataset)
    print("initial loss: {:.2f}".format(loss0))
    print("initial accuracy: {:.2f}".format(accuracy0))
    history = model.fit(train_dataset,
                        epochs=initial_epochs,
                        callbacks=callback_list,
                        validation_data=validation_dataset)

    base_model.trainable = True
    # Let's take a look to see how many layers are in the base model
    print("Number of layers in the base model: ", len(base_model.layers))

    # Fine-tune from this layer onwards
    fine_tune_at = 100

    # Freeze all the layers before the `fine_tune_at` layer
    for layer in base_model.layers[:fine_tune_at]:
        layer.trainable =  False

    model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
                optimizer = tf.keras.optimizers.RMSprop(lr=base_learning_rate/10),
                metrics=['accuracy'])
    model.summary()

    fine_tune_epochs = parameters["fine_tune_epochs"]
    total_epochs =  initial_epochs + fine_tune_epochs

    history_fine = model.fit(train_dataset,
                            epochs=total_epochs,
                            initial_epoch=history.epoch[-1],
                            callbacks=callback_list,
                            validation_data=validation_dataset)

else:
    model = models.load_model(os.path.join(exp.base_dir, "model.h5"))
    history_fine = model.fit(train_dataset,
                            epochs=parameters["fine_tune_epochs"],
                            initial_epoch=0,
                            callbacks=callback_list,
                            validation_data=validation_dataset)

models_dir = os.path.join(exp.base_dir, 'models')
tf.saved_model.save(model, os.path.join(models_dir, 'saved_model'))
model.save(os.path.join(models_dir, 'keras_model/model.h5'))
exp.store("keras_model", os.path.join(models_dir, 'keras_model/model.h5'))
exp.store("model-latest", os.path.join(models_dir, 'saved_model'), zip=True)
parameters["fine_tune"] = True
exp.log("parameters", parameters, "table", replace=True)
exp.update(status="success")

y_true = []
y_pred = []
for btc in range(len(validation_dataset)):
    image_batch, label_batch = validation_dataset.as_numpy_iterator().next()
    predictions = model.predict_on_batch(image_batch).flatten()

    # Apply a sigmoid since our model returns logits
    predictions = tf.nn.sigmoid(predictions)
    predictions = tf.where(predictions < 0.5, 0, 1)
    y_true.extend(label_batch)
    y_pred.extend(predictions.numpy())

print('Confusion Matrix')
print('----------------')
print(confusion_matrix(y_true, y_pred))
print()
print('Classification Report')
print('---------------------')
print(classification_report(y_true, y_pred, target_names=exp.categories))
cm = {
    'categories': exp.categories,
    'values': confusion_matrix(y_true, y_pred).tolist()
}
exp.log('confusion-matrix', cm, 'heatmap')

max_stars_repo_name: PicselliaTeam/picsellia-notebooks

max_issues_repo_name: PicselliaTeam/picsellia-notebooks

max_forks_repo_name: PicselliaTeam/picsellia-notebooks


text:
import os
import numpy as np

os.environ['KERAS_BACKEND'] = 'tensorflow'
from keras import backend as K

K.tensorflow_backend._get_available_gpus()

import keras
from keras.models import Sequential
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras.datasets import cifar10
from keras import regularizers
from keras.callbacks import LearningRateScheduler
import numpy as np
from keras.optimizers import RMSprop, SGD
from keras.datasets import cifar10
from keras.utils import to_categorical

from architectures.target_model import CNNModel
from utils import get_keras_dataset

class CNNCifar10Model(CNNModel):
    def __init__(self, num_classes: int = 10, custom_optimizer=None, input_shape=(32, 32, 3)):
        super().__init__(num_classes)

        self.input_shape = input_shape
        x_train, y_train, x_test, y_test = get_keras_dataset(cifar10.load_data(), input_shape=self.input_shape)
        self.x_train = x_train
        self.x_test = x_test
        self.y_train = y_train
        self.y_test = y_test

        weight_decay = 0.0002
        self.model = Sequential()
        self.model.add(Conv2D(32, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay),
                              input_shape=input_shape))
        self.model.add(Activation('elu'))
        self.model.add(BatchNormalization())
        self.model.add(Conv2D(32, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
        self.model.add(Activation('elu'))
        self.model.add(BatchNormalization())
        self.model.add(MaxPooling2D(pool_size=(2, 2)))
        self.model.add(Dropout(0.2))

        self.model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
        self.model.add(Activation('elu'))
        self.model.add(BatchNormalization())
        self.model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
        self.model.add(Activation('elu'))
        self.model.add(BatchNormalization())
        self.model.add(MaxPooling2D(pool_size=(2, 2)))
        self.model.add(Dropout(0.3))

        self.model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
        self.model.add(Activation('elu'))
        self.model.add(BatchNormalization())
        self.model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
        self.model.add(Activation('elu'))
        self.model.add(BatchNormalization())
        self.model.add(MaxPooling2D(pool_size=(2, 2)))
        self.model.add(Dropout(0.4))

        self.model.add(Flatten())
        self.model.add(Dense(10, activation='softmax'))

        optimizer = custom_optimizer or SGD(lr=0.001, momentum=0.9)
        self.model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy"])


text:
"""
The ``hierarchical_risk_parity`` module implements the HRP portfolio from Marcos Lopez de Prado.
Call ``hrp_portfolio(returns)`` to generate portfolio weights.
"""
import numpy as np
import pandas as pd
import scipy.cluster.hierarchy as sch
import scipy.spatial.distance as ssd

# This code has been reproduced(with modification) from the paper:
# López de Prado, M. (2016). Building Diversified Portfolios that Outperform Out of Sample.
# The Journal of Portfolio Management, 42(4), 59–69. https: // doi.org / 10.3905 / jpm.2016.42.4.059

def _get_cluster_var(cov, cluster_items):
    # Compute variance per cluster
    cov_slice = cov.loc[cluster_items, cluster_items]
    weights = 1 / np.diag(cov_slice)  # Inverse variance weights
    weights /= weights.sum()
    w = weights.reshape(-1, 1)
    cluster_var = np.dot(np.dot(w.T, cov_slice), w)[0, 0]
    return cluster_var

def _get_quasi_diag(link):
    # Sort clustered items by distance
    link = link.astype(int)
    sort_ix = pd.Series([link[-1, 0], link[-1, 1]])
    num_items = link[-1, -1]  # number of original items
    while sort_ix.max() >= num_items:
        sort_ix.index = range(0, sort_ix.shape[0] * 2, 2)  # make space
        df0 = sort_ix[sort_ix >= num_items]  # find clusters
        i = df0.index
        j = df0.values - num_items
        sort_ix[i] = link[j, 0]  # item 1
        df0 = pd.Series(link[j, 1], index=i + 1)
        sort_ix = sort_ix.append(df0)  # item 2
        sort_ix = sort_ix.sort_index()  # re-sort
        sort_ix.index = range(sort_ix.shape[0])  # re-index

    return sort_ix.tolist()

def _raw_hrp_allocation(cov, ordered_tickers):
    # Compute HRP allocation
    w = pd.Series(1, index=ordered_tickers)
    cluster_items = [ordered_tickers]  # initialize all items in one cluster

    while len(cluster_items) > 0:
        cluster_items = [
            i[j:k]
            for i in cluster_items
            for j, k in ((0, len(i) // 2), (len(i) // 2, len(i)))
            if len(i) > 1
        ]  # bi-section
        # For each pair, optimise locally.
        for i in range(0, len(cluster_items), 2):
            first_cluster = cluster_items[i]
            second_cluster = cluster_items[i + 1]
            # Form the inverse variance portfolio for this pair
            first_variance = _get_cluster_var(cov, first_cluster)
            second_variance = _get_cluster_var(cov, second_cluster)
            alpha = 1 - first_variance / (first_variance + second_variance)
            w[first_cluster] *= alpha  # weight 1
            w[second_cluster] *= 1 - alpha  # weight 2
    return w

def hrp_portfolio(returns):
    """
    Construct a hierarchical risk parity portfolio

    :param returns: asset historical returns
    :type returns: pd.DataFrame
    :return: weights for the HRP portfolio
    :rtype: dict
    :raises TypeError: if ``returns`` is not a dataframe
    """
    if not isinstance(returns, pd.DataFrame):
        raise TypeError("returns are not a dataframe")
    corr, cov = returns.corr(), returns.cov()

    # Compute distance matrix, with ClusterWarning fix as
    # per https://stackoverflow.com/questions/18952587/
    dist = ssd.squareform(((1 - corr) / 2) ** 0.5)

    link = sch.linkage(dist, "single")
    sort_ix = _get_quasi_diag(link)
    ordered_tickers = corr.index[sort_ix].tolist()
    hrp = _raw_hrp_allocation(cov, ordered_tickers)
    return dict(hrp.sort_index())

max_stars_repo_name: arminv/PyPortfolioOpt

max_issues_repo_name: noisyoscillator/PyPortfolioOpt

max_forks_repo_name: noisyoscillator/PyPortfolioOpt

reason: import numpy,import scipy


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 12 09:54:06 2020

@author: yokoyama
"""

from copy import deepcopy
from numpy.matlib import repmat
from numpy.random import randn, rand
import numpy as np

class AR_Kalman:
    def __init__(self, x, P, UC, flimits):
        self.x       = x
        self.P       = P
        self.UC      = UC
        self.flimits = flimits

##############################################################################
    def est_kalman(self):
        x           = self.x
        P           = self.P
        UC          = self.UC
        flimits     = self.flimits

        Nt, Nch     = x.shape

        self.Nch    = Nch

        Total_Epoch = int(Nt-P)

        Kb0         = np.eye(Nch*(Nch*P+1))
        mu_beta0    = np.zeros(Nch*(Nch*P+1))
        sigma0      = np.eye(Nch)

        S           = np.zeros((Nch, Nch, Total_Epoch))
        Changes     = np.nan * np.ones(Total_Epoch)
        loglike     = np.nan * np.ones(Total_Epoch)

        beta        = np.zeros((Nt-P, Nch*Nch, P))
        residual    = np.zeros((Nt-P, Nch))
        #%%
        cnt      = 0
        Q1       = UC * np.eye(Kb0.shape[0])
        Q2       = np.eye(Nch)

        for i in range(P, Nt, 1):
            #%%
            print('Epoch: (%d / %d), index: %d'%(cnt+1, Total_Epoch, i))
            #########################################################################
            AR_Kalman.make_features(self, x, i)
            Dx      = AR_Kalman.make_Dx(self)#(x_train, T, Nosc, 2*P)
            self.Dx = Dx

            #########################################################################
            tmp_y = (Dx @ mu_beta0).reshape((1,Nch), order='C')
            if i == P:
                y_hat = deepcopy(tmp_y)
            else:
                y_hat = np.concatenate((y_hat, deepcopy(tmp_y)), axis=0)

            #### Update step : Update prior distribution (update model parameter)
            mu_beta, Kb, change_ratio, sigma, L, Q2 = AR_Kalman.update_coeff(self, mu_beta0, Kb0, sigma0, UC, Q2)
            Kb           = Kb + Q1

            mu_beta0     = deepcopy(mu_beta)
            Kb0          = deepcopy(Kb)
            sigma0       = deepcopy(sigma)

            S[:,:,cnt]   = deepcopy(sigma)
            Changes[cnt] = deepcopy(change_ratio)
            loglike[cnt] = L

            tmp_beta = mu_beta.reshape((Nch, Nch*P+1))
            # tmp_beta = mu_beta.reshape((Nch, Nch*P))
            for p in range(P):
                idx = np.arange(0, Nch, 1) + p*Nch
                beta[cnt, :, p]  = tmp_beta[:,idx].reshape((Nch*Nch))

            residual[cnt, :] = tmp_beta[:,-1]

            cnt += 1

        PDC = AR_Kalman.calc_connectivity(beta, Nch, Nt, P, flimits)

        self.PDC      = PDC
        self.beta     = beta
        self.residual = residual
        self.Changes  = Changes
        self.loglike  = loglike
        self.y_hat    = y_hat
        self.S        = S
        self.Kb0      = Kb0

        # return beta, OMEGA, Changes, L, y_hat, sigma0, Kb0
##############################################################################
    def calc_connectivity(beta, Nch, Nt, P, flimits):
        frqs        = np.arange(flimits[0], flimits[-1]+1, 1)

        Nf           = len(frqs)
        Nt, Ndim, Np = beta.shape

        PDC          = np.zeros((Nch, Nch, Nt))
        for t in range(Nt):
            Aij        = np.zeros((Nch, Nch, Nf), dtype=complex)
            for f in range(Nf):
                Af_tmp = np.zeros((Nch, Nch, Np), dtype=complex)
                for p in range(Np):
                    Af_tmp[:,:,p] = beta[t, :, p].reshape(Nch, Nch) * np.exp(1j * 2 * np.pi * frqs[f] * (p+1))

                Aij[:,:,f] = np.eye(Nch) - np.sum(Af_tmp, axis=2)

            f_width    = flimits[1] - flimits[0]

            term1      = abs(Aij)**2
            term2      = np.sum(abs(Aij)**2, axis=1)
            term2      = term2[np.newaxis,:,:]

            pdc        = 1/f_width * np.sum(term1/term2, axis=2)

            PDC[:,:,t] = pdc.T

        return PDC
##############################################################################
    def make_features(self, x, idx):
        Nch = self.Nch
        P   = self.P
        i   = idx
        x_train = np.flipud(x[i-P:i,:]).reshape(-1)
        x_train = np.concatenate((x_train, np.ones(1)), axis=0)
        #########################################################################
        y_train = x[i,:]

        self.x_train = x_train
        self.y_train = y_train
##############################################################################

    def make_Dx(self):#(X, T, N, P):
        X    = self.x_train
        N    = self.Nch

        Dx   = np.kron(np.eye(N), X)

        return Dx

##############################################################################
    def update_coeff(self, mu, P, S, UC, Q2):#(X, Y, Dx, mu_beta, Kb, sigma0, T, N):
        Y  = self.y_train
        H  = self.Dx
        N  = self.Nch

        def mylogdet(S):
            L       = np.linalg.cholesky(S)
            logdetS = 2*np.sum(np.log(np.diag(L)))

            return logdetS

        def mydet(S):
            L       = np.linalg.cholesky(S)
            detS = np.linalg.det(L)**2

            return detS

        def hoteling_T2(error, S):
            S_inv = inv_use_cholensky(S)
            T2    = error.T @ S_inv @ error

            return T2

        def inv_use_cholensky(M):
            L     = np.linalg.cholesky(M)
            L_inv = np.linalg.inv(L)
            M_inv = np.dot(L_inv.T, L_inv)

            return M_inv

        ############# Estimate posterior distribution #############################
        err     = (Y.reshape(-1) - np.dot(H, mu).reshape(-1))
        Q2      = (1 - UC) * Q2 + UC * err[:, np.newaxis] @ err[:, np.newaxis].T

        S_new   = Q2 + H @ P @ H.T
        S_new_inv = inv_use_cholensky(S_new)
        #### update Kalman gain
        K       = P @ H.T @ S_new_inv # Kalman Gain
        #### update mean
        mu_new  = mu + K  @ (Y.reshape(-1) - np.dot(H, mu).reshape(-1))
        #### update covariance
        P_new   = P - K @ S_new @ K.T
        ###########################################################################
        Ndim    = Y.shape[0]
        loglike = 0.5 * (-Ndim * np.log(2*np.pi) - mylogdet(S) - (Y.reshape(-1) - np.dot(H, mu).reshape(-1))  @ inv_use_cholensky(S) @ (Y.reshape(-1) - np.dot(H, mu).reshape(-1)) )
        change_ratio = hoteling_T2(Y - np.dot(H, mu), S)
        return mu_new, P_new, change_ratio, S_new, loglike, Q2

max_stars_repo_name: myGit-YokoyamaHiroshi/ChangeDetectSim

max_issues_repo_name: myGit-YokoyamaHiroshi/ChangeDetectSim

max_forks_repo_name: myGit-YokoyamaHiroshi/ChangeDetectSim

reason: import numpy,from numpy


text:
import os

from pycocotools.coco import COCO
from demo.predictor import COCODemo
import torch
from torchvision.transforms import transforms as T
from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.modeling.detector import build_detection_model
from maskrcnn_benchmark.structures.image_list import to_image_list
import matplotlib.pylab as pylab
import matplotlib.pyplot as plt
import numpy as np
import cv2
import time

# set image dpi
plt.rcParams['figure.dpi'] = 300

# define the address of the image files
dataset_dir = './datasets/kiktech/skyeye_data'
annotations_dir = '{}/annotations'.format(dataset_dir)
test_images_dir = '{}/train'.format(dataset_dir)
test_ann_file = '{}/kiktech_train.json'.format(annotations_dir)
coco = COCO(test_ann_file)

config_file = './configs/retinanet/retinanet_R-50-FPN_1x_quick.yaml'
cfg.merge_from_file(config_file)
coco_demo = COCODemo(
    cfg,
    # TODO: add confidence threshold
)
device = torch.device(cfg.MODEL.DEVICE)

def build_transforms(cfg):
    """
    copy from predictor
    """
    normalize_transform = T.Normalize(
        mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD
    )
    min_size = cfg.INPUT.MIN_SIZE_TEST
    max_size = cfg.INPUT.MAX_SIZE_TEST
    transform = T.Compose(
        [
            T.ToPILImage(),
            T.ToTensor(),
            normalize_transform,
        ]
    )
    return transform

def calculate_time(images, transforms):
    """
    :param images:images info list
    :param transforms: the transform function
    :return: The time it takes to load each image
    """
    model = build_detection_model(cfg)
    model.eval()
    model.to(device)
    images_paths = [_["file_name"] for _ in images]
    start_time = time.time()
    for img_path in images_paths:
        img_path = test_images_dir + '/{}'.format(img_path)
        I = cv2.imread(img_path)
        I = transforms(I)
        I = to_image_list(I, 32)
        I = I.to(device)
        with torch.no_grad():
            model(I)
    end_time = time.time()
    total_time = end_time - start_time
    per_time = total_time / len(images)
    return per_time

def compute_label_color(labels):
    """
    copy from predictor
    """
    palette = np.array([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])
    labels = np.array(labels)
    colors = labels[:, None] * palette
    colors = (colors % 255).astype('uint8')
    return colors

def evaluation_in_coco(f_image_path):
    """
    output the ground truth
    :arg f_image_path: image path
    """
    I = cv2.imread(f_image_path)
    plt.axis('off')
    plt.imshow(I)
    # plt.show()
    annIds = coco.getAnnIds(imgIds=img['id'])
    anns = coco.loadAnns(annIds)
    labels = [anns[k]['category_id'] for k in range(len(anns))]

    for j in range(len(labels)):
        if labels[j] == -1:
            labels[j] = 0

    colors = compute_label_color(labels).tolist()
    for n in range(len(anns)):
        x, y, w, h = anns[n]['bbox']
        x, y, w, h = int(x), int(y), int(w), int(h)
        if anns[n]['category_id'] != -1:
            text = coco.loadCats(anns[n]['category_id'])[0]['name']
        else:
            text = 'ignore'
        cv2.putText(I, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, .5, (255, 255, 255), 1)
        cv2.rectangle(I, (x, y), (x + w, y + h), tuple(colors[n]), 1)
    coco.showAnns(anns)
    plt.imshow(I)
    plt.savefig('test.png', dpi=100)
    plt.show()

def evaluation_in_model(f_image_path):
    """
    output the prediction results
    :arg f_image_path: image path
    """
    I = cv2.imread(f_image_path)
    composite, total_time = coco_demo.run_on_opencv_image(I)
    print(total_time)
    cv2.imwrite('model.png', I)
    cv2.imshow("predict detections", composite)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    categories = coco.loadCats(coco.getCatIds())
    print(categories)
    # for n in range(len(categories)):
    #     print(categories[n]['name'])
    catIds = coco.getCatIds(catNms=['mask_person_top'])
    # print(len(coco.loadCats(coco.getCatIds())))
    imgIds = coco.getImgIds()
    all_images = coco.loadImgs(imgIds)
    # print("the time it takes to load each image is {}.".format(calculate_time(all_images, build_transforms(cfg))))
    img = coco.loadImgs(imgIds[np.random.randint(0, len(imgIds))])[0]
    image_path = test_images_dir + '/{}'.format(img['file_name'])

    evaluation_in_coco(image_path)
    evaluation_in_model(image_path)

max_stars_repo_name: Lyears/maskrcnn-benchmark

max_issues_repo_name: Lyears/maskrcnn-benchmark

max_forks_repo_name: Lyears/maskrcnn-benchmark


text:
import sys
sys.path.append('.')
from typing import List

import cv2
import numpy as np
from PyQt5.QtCore import pyqtSlot
from PyQt5.QtGui import QImage, QPixmap, QFontDatabase
from PyQt5.QtWidgets import QApplication, QMainWindow, QLabel, QAction, QStyle, QGroupBox, QDialog, QHBoxLayout
from qtpy import uic, QtCore

from pedrec.configs.app_config import AppConfig
from pedrec.models.human import Human
from pedrec.ui.models.pedrec_ui_config import PedRecUIConfig
from pedrec.ui.pedrec_worker import PedRecWorker
from pedrec.utils.skeleton_helper_3d import get_human_size_from_skeleton_3d

# Load icons
import pedrec.ui.qt_icon_resources

class PedRecApp(QMainWindow):
    def __init__(self, app: QApplication, worker: PedRecWorker, app_cfg: AppConfig):
        super().__init__()
        app.setStyle('Breeze')
        QFontDatabase().addApplicationFont("./emojione-android.ttf")
        print(sys.path)
        uic.loadUi("pedrec/ui/mainwindow.ui", self)
        self.cfg_path = "pedrec/ui/.ui_cfg.pkl"
        self.cfg = PedRecUIConfig()
        self.cfg.load(self.cfg_path)
        self.app_cfg = app_cfg
        self.play = True
        self.playback_action = None
        self.play_icon = self.style().standardIcon(QStyle.SP_MediaPlay)
        self.pause_icon = self.style().standardIcon(QStyle.SP_MediaPause)
        self.selected_human_uid: int = None
        self.humans: List[Human] = None
        self.worker = worker
        self.frame_nr_label = QLabel("Frame: No data")
        self.fps_label = QLabel("FPS: No data")
        self.actions_label = QLabel("Actions: No data")
        self.human_score_label = QLabel("-")
        self.human_size_label = QLabel("-")
        self.actions_bar_chart_view.initialize_actions_chart(self.app_cfg.inference.action_list)

        self.init_menu(app)
        self.init_status_bar()
        self.init_img_view(app_cfg.inference.img_size)
        self.init_buttons()
        self.__clear_ehpi()
        # self.init_metadata_view()
        self.show()
        self.start_worker(app)

    def init_menu(self, app):
        menu_bar = self.menuBar()
        # File menu
        file_menu = menu_bar.addMenu('&File')
        exit_action = QAction(self.style().standardIcon(QStyle.SP_DialogCancelButton), '&Exit', self)
        exit_action.setShortcut('Esc')
        exit_action.setStatusTip('Exit application')
        exit_action.triggered.connect(app.quit)
        file_menu.addAction(exit_action)
        # Settings menu
        # settings_menu = menu_bar.addMenu('&Settings')
        # show_object_bbs_toggle = QAction('&Show objects', self, checkable=True, checked=True)
        # show_object_bbs_toggle.chec
        # show_object_bbs_toggle.triggered(true)
        # show_object_bbs_toggle.setShortcut('o')
        # show_object_bbs_toggle.setStatusTip('Show object bounding boxes')
        # show_object_bbs_toggle.triggered.connect(self.toggle_show_objects)
        # settings_menu.addAction(show_object_bbs_toggle)

        # Control actions
        self.playback_action = QAction(self.pause_icon, '', self)
        self.playback_action.setShortcut('Space')
        self.playback_action.setStatusTip('Pause')
        self.playback_action.triggered.connect(self.toggle_worker_playback)
        menu_bar.addAction(self.playback_action)

    #######################################################################
    ############################ Buttons ##################################
    #######################################################################

    def init_buttons(self):
        # pose_2d
        self.action_toggle_pose_2d.setChecked(self.cfg.show_pose_2d)
        self.action_toggle_pose_2d.triggered.connect(
            lambda status: self.toggle_button(f'{self.cfg.show_pose_2d=}'.split('=')[0].split('.')[-1],
                                              status))

        # object_bb
        self.action_toggle_object_bb.setChecked(self.cfg.show_object_bb)
        self.action_toggle_object_bb.triggered.connect(
            lambda status: self.toggle_button(f'{self.cfg.show_object_bb=}'.split('=')[0].split('.')[-1],
                                              status))

        # human_bb
        self.action_toggle_human_bb.setChecked(self.cfg.show_human_bb)
        self.action_toggle_human_bb.triggered.connect(
            lambda status: self.toggle_button(f'{self.cfg.show_human_bb=}'.split('=')[0].split('.')[-1],
                                              status))

        # head_orientation_2d
        self.action_toggle_head_orientation_2d.setChecked(self.cfg.show_head_orientation_2d)
        self.action_toggle_head_orientation_2d.triggered.connect(
            lambda status: self.toggle_button(f'{self.cfg.show_head_orientation_2d=}'.split('=')[0].split('.')[-1],
                                              status))

        # body_orientation_2d
        self.action_toggle_body_orientation_2d.setChecked(self.cfg.show_body_orientation_2d)
        self.action_toggle_body_orientation_2d.triggered.connect(
            lambda status: self.toggle_button(f'{self.cfg.show_body_orientation_2d=}'.split('=')[0].split('.')[-1],
                                              status))

        # actions
        self.action_toggle_actions.setChecked(self.cfg.show_actions)
        self.action_toggle_actions.triggered.connect(
            lambda status: self.toggle_button(f'{self.cfg.show_actions=}'.split('=')[0].split('.')[-1],
                                              status))

        # sees_car
        self.action_toggle_sees_car.setChecked(self.cfg.show_sees_car_flag)
        self.action_toggle_sees_car.triggered.connect(
            lambda status: self.toggle_button(f'{self.cfg.show_sees_car_flag=}'.split('=')[0].split('.')[-1],
                                              status))

    # Button Actions

    def toggle_button(self, property_name: str, active: bool):
        print(f"{property_name}: {active}")
        setattr(self.cfg, property_name, active)
        self.cfg.save(self.cfg_path)

    def toggle_worker_playback(self):
        if self.play:
            self.playback_action.setStatusTip('Play')
            self.playback_action.setIcon(self.play_icon)
            self.worker.pause()
        else:
            self.playback_action.setStatusTip('Pause')
            self.playback_action.setIcon(self.pause_icon)
            self.worker.resume()
        self.play = not self.play

    def init_status_bar(self):
        self.statusBar().addWidget(self.frame_nr_label)
        self.statusBar().addWidget(self.fps_label)
        self.statusBar().addWidget(self.human_score_label)
        self.statusBar().addWidget(self.human_size_label)
        self.statusBar().addWidget(self.actions_label)

    def init_img_view(self, img_size):
        self.dialog = QDialog()
        self.hlayout = QHBoxLayout()
        self.img_view.img_size = img_size
        self.img_view.cfg = self.cfg
        self.img_view.human_selected.connect(self.set_selected_human_uid)

        # show img in seperat window
        # self.grid.removeWidget(self.groupBox_2)
        # self.hlayout.addWidget(self.img_view)
        # self.hlayout.setContentsMargins(0,0,0,0)
        # self.dialog.setLayout(self.hlayout)
        # # self.dialog.setWindowFlags(QtCore.Qt.Window | QtCore.Qt.FramelessWindowHint)
        # self.dialog.show()

    # def init_img_ephi(self):
    #     self.img_ehpi.img_size = ImageSize(200, 100)

    def __update_ehpi(self, ehpi: np.ndarray):
        if ehpi is None:
            return
        # ehpi = np.transpose(np.copy(ehpi_normalized), (1, 0, 2))
        # ehpi *= 255
        # ehpi = ehpi.astype(np.uint8)
        ehpi = cv2.resize(ehpi, (640, 320), interpolation=cv2.INTER_NEAREST)
        # ehpi = cv2.cvtColor(ehpi, cv2.COLOR_BGR2RGB)
        h, w, ch = ehpi.shape
        bytes_per_line = ch * w
        qt_img = QImage(ehpi.data, w, h, bytes_per_line, QImage.Format_RGB888)
        self.img_ehpi.setPixmap(QPixmap.fromImage(qt_img))

    def __clear_ehpi(self):
        ehpi = np.zeros((320, 640, 3), dtype=np.uint8)
        h, w, ch = ehpi.shape
        bytes_per_line = ch * w
        qt_img = QImage(ehpi.data, w, h, bytes_per_line, QImage.Format_RGB888)
        self.img_ehpi.setPixmap(QPixmap.fromImage(qt_img))

    # Event Handler
    @pyqtSlot(int, np.ndarray, list, np.ndarray, int)
    def update_worker_data(self, frame_nr: int, img: np.ndarray, humans: List[Human], object_bbs: np.ndarray, fps: int):
        # frame nr label:
        self.frame_nr_label.setText(f"Frame: {frame_nr:05}")
        self.humans = humans

        # img_view
        h, w, ch = img.shape
        bytes_per_line = ch * w
        qt_img = QImage(img.data, w, h, bytes_per_line, QImage.Format_RGB888)
        self.img_view.setPixmap(QPixmap.fromImage(qt_img))
        self.img_view.set_humans(humans)
        self.img_view.set_object_bbs(object_bbs)

        # Skeleton 2.5d
        self.update_selected_human()

        # fps_label
        self.fps_label.setText(f"FPS: {fps:03}")

    @pyqtSlot(int)
    def set_selected_human_uid(self, selected_human_uid: int):
        self.selected_human_uid = selected_human_uid
        self.update_selected_human()
        self.img_view.repaint()

    def __clear_selected_human(self):
        self.__clear_ehpi()
        self.body_orientation_view.clear()
        self.human_score_label.setText("Score: -%")
        self.human_size_label.setText("Size: -mm")
        self.actions_label.setText("Actions: -")
        self.actions_bar_chart_view.clear_data()
        # self.human_score_label.setText("-")

    def update_selected_human(self):
        self.skeleton_view.clear()
        if self.selected_human_uid == -1:
            self.__clear_selected_human()
            return

        selected_human = next((human for human in self.humans if human.uid == self.selected_human_uid), None)
        if selected_human is None:
            self.__clear_selected_human()
            return
        self.__update_ehpi(selected_human.ehpi)
        self.__update_metadata(selected_human)
        self.__update_skeleton_3d(selected_human)
        self.__update_orientation(selected_human)
        self.actions_bar_chart_view.set_actions(selected_human.action_probabilities)

    def __update_metadata(self, selected_human: Human):
        self.human_score_label.setText(f"Score: {int(selected_human.score * 100):03}%")
        self.actions_label.setText(f"Actions: {', '.join([a.name for a in selected_human.actions])}")
        size = get_human_size_from_skeleton_3d(selected_human.skeleton_3d)

    def __update_skeleton_3d(self, selected_human: Human):
        skeleton = selected_human.skeleton_3d.copy()
        self.skeleton_view.add_skeleton(skeleton)

    def __update_orientation(self, selected_human: Human):
        orientation = selected_human.orientation.copy()
        self.body_orientation_view.set_orientation(orientation[0, 0], orientation[0, 1], orientation[1, 0],
                                                   orientation[1, 1])

    def start_worker(self, app):
        self.worker.data_updated.connect(self.update_worker_data)
        self.worker.start()
        app.aboutToQuit.connect(self.worker.stop)

max_stars_repo_name: noboevbo/PedRec

max_issues_repo_name: noboevbo/PedRec

max_forks_repo_name: noboevbo/PedRec


text:
import decimal
import math
import re
from typing import Any, Union
from .fuzzyfield import FuzzyField
from .errors import DomainError, FieldTypeError, MalformedFieldError
from .tools import NA_VALUES

try:
    import numpy
    CAN_CAST_TO_INT = str, numpy.integer
except ImportError:
    CAN_CAST_TO_INT = str

class Float(FuzzyField):
    """Convert a string representing a number, an int, or other numeric types
    (e.g. `numpy.float64`) to float.

    :param default:
        Default value. Unlike in all other FuzzyFields, if omitted it is NaN
        instead of None.
    :param min_value:
        Minimum allowable value. Omit for no minimum.
    :param max_value:
        Maximum allowable value. Omit for no maximum.
    :param bool allow_min:
        If True, test that value >= min_value, otherwise value > min_value
    :param bool allow_max:
        If True, test that value <= max_value, otherwise value < max_value
    :param bool allow_zero:
        If False, test that value != 0
    :param dict kwargs:
        parameters to be passed to :class:`FuzzyField`
    """
    min_value: Union[int, float]
    max_value: Union[int, float]
    allow_min: bool
    allow_max: bool
    allow_zero: bool

    def __init__(self, *, min_value: Union[int, float] = -math.inf,
                 max_value: Union[int, float] = math.inf,
                 allow_min: bool = True, allow_max: bool = True,
                 allow_zero: bool = True, default: Any = math.nan, **kwargs):
        super().__init__(default=default, **kwargs)

        assert min_value <= max_value
        self.min_value = min_value
        self.max_value = max_value
        self.allow_min = allow_min
        self.allow_max = allow_max
        self.allow_zero = allow_zero

    def validate(self, value: Any) -> Union[float, int, decimal.Decimal, None]:
        """Convert a number or a string representation of a number to a
        validated number.

        :param value:
            string representing a number, possibly with thousands separator
            or in accounting negative format, e.g. (5,000.200) ==> -5000.2,
            or any number-like object
        :rtype:
            as returned by :meth:`Numeric.num_converter`
        :raises DomainError:
            Number out of allowed range
        :raises MalformedFieldError, FieldTypeError:
            Not a number
        """
        if isinstance(value, str):
            # Preprocess strings before passing them to self._num_converter
            # Remove thousands separator
            value = value.replace(',', '')

            # Convert accounting-style negative numbers, e.g. '(1000)'
            if value.startswith('(') and value.endswith(')'):
                value = '-' + value[1:-1]

            # Convert negative numbers formatted by Excel in some cases
            # '- 1000 -'
            elif value.startswith('- ') and value.endswith(' -'):
                value = '-' + value[2:-2]

        value = self._num_converter(value)
        if value is None:
            return None

        # Decimal has problems comparing to float/int
        valuef = float(value)

        if ((not self.allow_zero and valuef == 0)
                or (self.allow_min and valuef < self.min_value)
                or (not self.allow_min and valuef <= self.min_value)
                or (self.allow_max and valuef > self.max_value)
                or (not self.allow_max and valuef >= self.max_value)):
            raise DomainError(self.name, value, choices=self.domain_str)

        return value

    @property
    def domain_str(self) -> str:
        """String representation of the allowed domain, e.g. "]-1, 1] non-zero"
        """
        lbracket = '[' if self.allow_min else ']'
        rbracket = ']' if self.allow_max else '['
        msg = f'{lbracket}{self.min_value}, {self.max_value}{rbracket}'
        if not self.allow_zero:
            msg += ' non-zero'
        return msg

    def _num_converter(self, value: Any) -> float:
        """Convert string, int, or other to float
        """
        try:
            return float(value)
        except TypeError:
            raise FieldTypeError(self.name, value, "number")
        except ValueError:
            raise MalformedFieldError(self.name, value, "number")

    @property
    def sphinxdoc(self) -> str:
        return f"Any number in the domain {self.domain_str}"

class Decimal(Float):
    """Convert a number or a string representation of a number to
    :class:`~decimal.Decimal`, which is much much slower and heavier than float
    but avoids converting 3.1 to 3.0999999.
    """
    def __init__(self, *, default: Any = decimal.Decimal('nan'), **kwargs):
        super().__init__(default=default, **kwargs)

    def _num_converter(self, value: Any) -> decimal.Decimal:
        """Convert string, float, or int to decimal.Decimal
        """
        # Performance shortcut
        if isinstance(value, decimal.Decimal):
            return value

        orig_value = value
        # Remove leading zeros after comma, as they confuse Decimal
        # e.g.  Decimal('0.0000000000') -> Decimal("0E-10")
        # Do not accidentally drop leading zeros in the exponent.
        if isinstance(value, str):
            value = value.upper()
            if 'E' in value:
                # Scientific notation
                mantissa, _, exponent = value.partition('E')
                if '.' in mantissa:
                    mantissa = re.sub(r'0*$', '', mantissa)
                value = f'{mantissa}E{exponent}'
            elif '.' in value:
                # Not scientific notation
                value = re.sub(r'0*$', '', value)

        try:
            return decimal.Decimal(value)
        except (TypeError, ValueError):
            raise FieldTypeError(self.name, orig_value, "number")
        except decimal.InvalidOperation:
            raise MalformedFieldError(self.name, orig_value, "number")

class Integer(Float):
    """Whole number.

    Valid values are:

    - anything that is parsed by the `int` constructor.
    - floats with strictly trailing zeros (e.g. 1.0000)
    - scientific format as long as there are no digits below 10^0 (1.23e2)

    .. note::
       inf and -inf are valid inputs, but in these cases
       the output will be of type float. To disable them you can use

       - ``min_value=-math.inf, allow_min=False``
       - ``max_value=math.inf, allow_max=False``

       NaN is treated as an empty cell, so it is accepted if required=False;
       in that case the validation will return whatever is set for default,
       which is math.nan unless overridden, which makes it a third case where
       the output value won't be int but float.

    :raises MalformedFieldError:
        if the number can't be cast to int without losing precision
    """
    def _num_converter(self, value: Any) -> Union[int, float]:
        """Convert value to int
        """
        # Quick exit
        if isinstance(value, int):
            return value

        # Attempt quick conversion. This won't work in case of the more
        # sophisticated cases we want to cover, e.g. '1.0e1'.
        # DO NOT blindly convert to int if value is a float, as int(3.5) = 3!
        # Not passing by float also prevents precision loss issues, e.g.
        # int('9999999999999999') != float('9999999999999999')
        if isinstance(value, CAN_CAST_TO_INT):
            try:
                return int(value)
            except ValueError:
                pass

        # float, np.float32/64, or string representation of a float
        try:
            valued = decimal.Decimal(value)
        except (TypeError, ValueError):
            raise FieldTypeError(self.name, value, "integer")
        except decimal.InvalidOperation:
            raise MalformedFieldError(self.name, value, "integer")

        # This should be already catered for by :meth:`FuzzyField.preprocess`
        assert not math.isnan(valued)
        if math.isinf(valued):
            return float(valued)

        valuei = int(valued)
        if valuei != valued:
            # Mantissa after the dot is not zero
            raise MalformedFieldError(self.name, value, "integer")
        return valuei

    @property
    def sphinxdoc(self) -> str:
        return f"Any whole number in the domain {self.domain_str}"

class Percentage(Float):
    """Percentage, e.g. 5% or .05

    .. warning::
       There's nothing stopping somebody from writing "35" where it should have
       been either "35%" or "0.35". If this field receives "35", it will
       return 3500.0. You should use the min_value and max_value parameters of
       :class:`Float` to prevent this kind of incidents. Still, nothing will
       ever protect you from a "1", which will be converted to 1.00 but the
       author of the input may have wanted to say 0.01.
    """
    def _num_converter(self, value: Any) -> Union[float, None]:
        """Convert string, int, or other to float
        """
        try:
            if isinstance(value, str) and value[-1] == '%':
                value = value[:-1].strip()
                if value in NA_VALUES:
                    return None
                return float(value) / 100
            return float(value)
        except TypeError:
            raise FieldTypeError(self.name, value, "percentage")
        except ValueError:
            raise MalformedFieldError(self.name, value, "percentage")

    @property
    def sphinxdoc(self) -> str:
        return f"Percentage, e.g. 5% or 0.05, in the domain {self.domain_str}"

max_stars_repo_name: crusaderky/validators

max_issues_repo_name: crusaderky/validators

max_forks_repo_name: crusaderky/validators


text:
# -*- coding: utf-8 -*-
import sys
import os
import numpy as np
import scipy as sp
from pylayers.util import geomutil as geo
from pylayers.util import pyutil as pyu
import re
"""

.. currentmodule:: pylayers.gis.lireifc2

.. autosummary::

"""

_filename = 'P-0007-BEAULIEU_BAT11D-x-2-E1.ifc'
filename = pyu.getlong(_filename,os.path.join('struc','bat11'))

fd  = open(filename,'r')
lig = fd.readlines()
dic1 = {}
tentry = []
for li in lig:
    li = li.replace(';\r\n','')
    if li.find('=')!=-1:
        t = li.split('=')
        Id = t[0]
        v  = t[1]
        entry = t[1].split('(')[0]
        dic1[Id] = li.split(entry)[1]
        if entry not in tentry:
            tentry.append(entry)

#
#
#
for k in dic1:
    lv  = dic1[k]
    while lv.find('#')!=-1:
        m = re.search('(#[0-9]*)',lv)
        cle = m.group(0)
        lv  = lv.replace(m.group(0),dic1[cle])
    dic1[k]=lv

dico = {}
for entry in tentry:
    cle = entry.replace('IFC','')
    dico[cle]={}
    for li in lig:
        li = li.replace(';\r\n','')
        if li.find('=')!=-1:
            t = li.split('=')
            v  = t[1]
            Id = t[0]
            if v.find(entry)!=-1:
                dico[cle][Id] = dic1[Id].replace('(','').replace(')','').split(',')

#
# Reshaping
#
for k in dico['CARTESIANPOINT']:
    p = map(float,dico['CARTESIANPOINT'][k])
    dico['CARTESIANPOINT'][k] = p

max_stars_repo_name: usmanwardag/pylayers

max_issues_repo_name: usmanwardag/pylayers

max_forks_repo_name: usmanwardag/pylayers