text:
import argparse
from torch.utils.data import RandomSampler, DataLoader
import numpy as np
import time
import pylab as plt
import os
import itertools
import pprint
import pandas as pd
import tqdm
import torchvision
import torch
from haven import haven_examples as he
from haven import haven_utils as hu
import sys
import os
import pprint

path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, path)

# from haven.haven_jobs import slurm_manager as sm

# from haven import haven_chk as hc
# from haven import haven_results as hr
print()

try:
    import job_configs
except Exception:
    pass
# from haven_utils import file_utils

# Job status
# ===========
def get_job(job_id):
    """Get job information."""
    command = "scontrol show job %s" % job_id
    job_info = ""
    while True:
        try:
            job_info = hu.subprocess_call(command)
            job_info = job_info.replace("\n", "")
            job_info = {v.split("=")[0]: v.split("=")[1] for v in job_info.split(" ") if "=" in v}
        except Exception:
            print("scontrol time out and retry now")
            time.sleep(1)
            continue
        break
    return job_info

def get_jobs(user_name):
    # account_id = hu.subprocess_call('eai account get').split('\n')[-2].split(' ')[0]
    """ get the first 3 jobs"""
    command = "squeue --user=%s" % user_name
    while True:
        try:
            job_list = hu.subprocess_call(command)
            job_list = job_list.split("\n")
            job_list = [v.lstrip().split(" ")[0] for v in job_list[1:]]
            result = []
            for job_id in job_list:
                result.append(get_job(job_id))
        except Exception:
            print("scontrol time out and retry now")
            time.sleep(1)
            continue
        break
    return result

# Job kill
# ===========
def kill_job(job_id):
    """Kill a job job until it is dead."""
    kill_command = "scancel %s" % job_id
    while True:
        try:
            hu.subprocess_call(kill_command)  # no return message after scancel
        except Exception:
            print("scancel time out and retry now")
            time.sleep(1)
            continue
        break
    return

def get_existing_slurm_job_commands(exp_list, savedir_base):
    existing_job_commands = []
    for exp_dict in exp_list:
        exp_id = hu.hash_dict(exp_dict)
        savedir = os.path.join(savedir_base, exp_id)
        file_name = os.path.join(savedir, "job_dict.json")
        if not os.path.exists(file_name):
            continue
        job_dict = hu.load_json(file_name)
        job_id = job_dict["job_id"]
        job_status = hu.subprocess_call("scontrol show job %s" % job_id).split("JobState=")[1].split(" ")[0]
        if job_status == "RUNNING" or job_status == "PENDING":
            existing_job_commands += [job_dict["command"]]

    return existing_job_commands

def get_job_spec():
    # read slurm setting
    lines = "#! /bin/bash \n"
    lines += "#SBATCH --account=%s \n" % job_configs.ACCOUNT_ID
    for key in list(job_configs.JOB_CONFIG.keys()):
        lines += "#SBATCH --%s=%s \n" % (key, job_configs.JOB_CONFIG[key])
    return lines

def submit_job(command, savedir):
    # read slurm setting
    lines = "#! /bin/bash \n"
    # if job_config is not None:
    #     lines += "#SBATCH --account=%s \n" % job_configs.ACCOUNT_ID
    #     for key in list(job_config.keys()):
    #         lines += "#SBATCH --%s=%s \n" % (key, job_config[key])
    lines += "#SBATCH --account=%s \n" % job_configs.ACCOUNT_ID
    for key in list(job_configs.JOB_CONFIG.keys()):
        lines += "#SBATCH --%s=%s \n" % (key, job_configs.JOB_CONFIG[key])
    path_log = os.path.join(savedir, "logs.txt")
    path_err = os.path.join(savedir, "err.txt")
    lines += "#SBATCH --output=%s \n" % path_log
    lines += "#SBATCH --error=%s \n" % path_err

    lines += command

    file_name = os.path.join(savedir, "bash.sh")
    hu.save_txt(file_name, lines)
    # launch the exp
    submit_command = "sbatch %s" % file_name
    while True:
        try:
            job_id = hu.subprocess_call(submit_command).split()[-1]
        except Exception:
            print("slurm time out and retry now")
            time.sleep(1)
            continue
        break

    # save the command and job id in job_dict.json
    job_dict = {"command": command, "job_id": job_id}
    hu.save_json(os.path.join(savedir, "job_dict.json"), job_dict)

    # delete the bash.sh
    os.remove(file_name)

    return job_id

# 1. define the training and validation function

def trainval(exp_dict, savedir, args):
    """
    exp_dict: dictionary defining the hyperparameters of the experiment
    savedir: the directory where the experiment will be saved
    args: arguments passed through the command line
    """
    # 2. Create data loader and model
    train_loader = he.get_loader(
        name=exp_dict["dataset"], split="train", datadir=os.path.dirname(savedir), exp_dict=exp_dict
    )
    model = he.get_model(name=exp_dict["model"], exp_dict=exp_dict)

    # 3. load checkpoint
    chk_dict = hw.get_checkpoint(savedir)

    # 4. Add main loop
    for epoch in tqdm.tqdm(range(chk_dict["epoch"], 10), desc="Running Experiment"):
        # 5. train for one epoch
        train_dict = model.train_on_loader(train_loader, epoch=epoch)

        # 6. get and save metrics
        score_dict = {"epoch": epoch, "acc": train_dict["train_acc"], "loss": train_dict["train_loss"]}
        chk_dict["score_list"] += [score_dict]

        images = model.vis_on_loader(train_loader)

    hw.save_checkpoint(savedir, score_list=chk_dict["score_list"], images=[images])
    print("Experiment done\n")

if __name__ == "__main__":
    # task 1 - submit example job
    """
    Run echo 35 and forward the logs to <savedir>/logs.txt
    """
    # command = 'echo 35'
    # savedir = '/home/xhdeng/shared/results/test_slurm/example_get_jobs'
    # job_id = submit_job(command, savedir)

    # # task 2 - get job info as dict
    # """
    # Get job info as dict and save it as json in the directory specified below
    # """
    # hu.save_json('/home/xhdeng/shared/results/test_slurm/example/job_info.json', job_info)

    # task 3 - kill job
    # """
    # Kill job then gets its info as dict and save it as json in the directory specified below
    # it should say something like CANCELLED
    # """
    # kill_job(job_id)
    # job_info = get_job(job_id)
    # hu.save_json('/home/xhdeng/shared/results/test_slurm/example_kill_job/job_info_dead.json', job_info)

    # task 4 - get all jobs from an account as a list
    # """
    # Get all jobs from an account as a list and save it in directory below
    # """
    # job_info_list = get_jobs(job_configs.USER_NAME)
    # hu.save_json('/home/xhdeng/shared/results/test_slurm/example_get_jobs/job_info_list.json', job_info_list)

    # # task 5 - run 10 jobs using threads
    # """
    # Use thee parallel threads from Haven and run these jobs in parallel
    # """
    # pr = hu.Parallel()

    # for i in range(1,20):
    #   command = 'echo %d' % i
    #   savedir = '/home/xhdeng/shared/results/test_slurm/example_%d' % i

    #   pr.add(submit_job, command, savedir)

    # pr.run()
    # pr.close()

    # # task 6 with menu - run mnist experiments on these 5 learning rates
    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument("-ei", "--exp_id")
    args = parser.parse_args()
    exp_list = []
    for lr in [1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, "bug"]:
        exp_list += [{"lr": lr, "dataset": "mnist", "model": "linear"}]

    if args.exp_id is None:
        # run jobs

        print("\nTotal Experiments:", len(exp_list))
        prompt = (
            "\nMenu:\n"
            "  0)'ipdb' run ipdb for an interactive session; or\n"
            "  1)'reset' to reset the experiments; or\n"
            "  2)'run' to run the remaining experiments and retry the failed ones; or\n"
            "  3)'status' to view the job status; or\n"
            "  4)'kill' to kill the jobs.\n"
            "Type option: "
        )

        option = input(prompt)
        if option == "run":
            # only run if job has failed or never ran before

            for exp_dict in exp_list:
                exp_id = hu.hash_dict(exp_dict)
                command = "python test_slurm.py -ei %s" % exp_id

        elif option == "reset":
            # ressset each experiment (delete the checkpoint and reset)
            command = "python test_slurm.py -ei %s" % exp_id
            for exp_dict in exp_list:
                exp_id = hu.hash_dict(exp_dict)
                command = "python test_slurm.py -ei %s" % exp_id

        elif option == "status":
            # get job status of each exp
            for exp_dict in exp_list:
                exp_id = hu.hash_dict(exp_dict)
        elif option == "kill":
            # make sure all jobs for the exps are dead
            for exp_dict in exp_list:
                exp_id = hu.hash_dict(exp_dict)

    else:
        for exp_dict in exp_list:
            exp_id = hu.hash_dict(exp_dict)
            savedir = "/home/xhdeng/shared/results/test_slurm/%s" % exp_id
            if exp_id is not None and exp_id == args.exp_id:
                trainval(exp_dict, savedir, args={})

max_stars_repo_name: mariatippler/haven-ai

max_issues_repo_name: mariatippler/haven-ai

max_forks_repo_name: mariatippler/haven-ai


text:
from typing import List, Union, Tuple, Dict, Optional
import numpy as np
from pyitlib import discrete_random_variable as drv

from categoryeval.representation import make_context_by_term_matrix

class DPScorer:
    """
    computes divergence-from-prototype, a measure of how close a learned representation is to a category prototype.
    """

    def __init__(self,
                 types_eval: List[str],  # the words that make up the category to be evaluated
                 tokens: List[str],  # the corpus
                 vocab: List[str],  # the vocabulary of the corpus
                 ) -> None:

        self.types_eval = types_eval

        # make p for each name - p is a theoretical probability distribution over x-words (next-words)
        # rows index contexts, and columns index next-words
        self.ct_mat, y_words_ = make_context_by_term_matrix(tokens, vocab, context_size=1)
        self.total_frequency = self.ct_mat.sum().sum().item()
        self.x_words = vocab
        self.y_words = [self.tuple2str(yw) for yw in y_words_]  # convert tuple to str

    def score(self,
              qs: np.ndarray,
              return_mean: bool = True,
              metric: str = 'js',
              prototype_is_unigram_distribution: bool = False,
              ) -> Union[float, List[float]]:
        """
        measure bits divergence of a set of next-word predictions from prototype next-word probability distribution,
        where the prototype is the category to which all probes belong.
        dp = divergence-from-prototype

        notes:
            computation of divergence checks for NaNs. this is done in 2 ways, slow or fast, depending on dtype:
            if dtype is float64, fast check is performed (float32 results in slow check)
            the slow check is much slower and should be avoided.
        """
        assert np.ndim(qs) == 2
        assert np.sum(qs[0]).round(1).item() == 1.0, np.sum(qs[0]).round(1).item()

        if qs.dtype != np.float64:
            raise TypeError('To aovid slow NaN check, cast input to float64.')

        if metric == 'ce':
            raise NotImplementedError
        elif metric == 'xe':
            fn = drv.entropy_cross_pmf
        elif metric == 're':
            raise NotImplementedError
        elif metric == 'js':
            fn = drv.divergence_jensenshannon_pmf
        else:
            raise AttributeError('Invalid arg to "metric".')

        # determine prototype representation
        if prototype_is_unigram_distribution:
            p = self._make_p(is_unconditional=True)
        else:
            p = self._make_p(is_unconditional=False)

        if qs.shape[1] != len(p):
            raise ValueError(f'Shape of input has {qs.shape[1]} columns '
                             f'but {len(p)} are required')

        # do the computation: compare each word's predicted and expected next-word probability distribution
        res = [fn(p, q) for q in qs]

        if return_mean:
            return np.mean(res).item()
        else:
            return res

    @staticmethod
    def tuple2str(yw):
        """convert a context (one or more y-words) which is an instance of a tuple to a string"""
        return '_'.join(yw)

    def _make_p(self,
                is_unconditional: bool = False,
                e=0.00000000001,  # probabilities cannot be zero -otherwise cross-entropy is inf
                ) -> np.ndarray:
        """
        make the true next-word probability distribution (by convention called "p"),
        which is defined as each word representing an iid sample from the distribution.
        """

        if is_unconditional:  # "unconditional" is a category whose members are all words in vocab
            return self._make_unconditional_p()

        # get slice of ct matrix
        row_ids = [self.y_words.index(w) for w in self.types_eval]
        assert row_ids
        sliced_ct_mat = self.ct_mat.tocsc()[row_ids, :]

        # make p, the true probability distribution over x-words given some category.
        # assumes there is a single distribution generating all probes
        res = []
        for col_id, xw in enumerate(self.x_words):
            if xw in self.types_eval:  # a test-word is not allowed to be next-word of a test-word
                f = e
            else:
                xw_frequency = sliced_ct_mat[:, col_id].sum()
                if xw_frequency > 0.0:
                    f = xw_frequency
                else:
                    f = e
            res.append(f)
        return np.array(res) / np.sum(res)

    def _make_unconditional_p(self
                              ) -> np.ndarray:
        """
        make theoretical next-word distribution for the "average" word;
        that is, what is the best next-word distribution given any word from the vocabulary,
         not just members of a category?
        """
        res = self.ct_mat.tocsr().sum(axis=0).A1 / self.total_frequency  # A1 converts matrix to 1D array
        return res

ext: py

max_stars_repo_name: phueb/CategoryEval

max_issues_repo_name: phueb/CategoryEval

max_forks_repo_name: phueb/CategoryEval

reason: import numpy


text:
"""
pyLDAvis Prepare
===============
Main transformation functions for preparing LDAdata to the visualization's data structures
"""

from __future__ import absolute_import
from past.builtins import basestring
from collections import namedtuple
import json
import logging
from joblib import Parallel, delayed, cpu_count
import numpy as np
import pandas as pd
from scipy.stats import entropy
from scipy.spatial.distance import pdist, squareform
from .utils import NumPyEncoder
try:
    from sklearn.manifold import MDS, TSNE
    sklearn_present = True
except ImportError:
    sklearn_present = False

def __num_dist_rows__(array, ndigits=2):
    return array.shape[0] - int((pd.DataFrame(array).sum(axis=1) < 0.999).sum())

class ValidationError(ValueError):
    pass

def _input_check(topic_term_dists, doc_topic_dists, doc_lengths, vocab, term_frequency):
    ttds = topic_term_dists.shape
    dtds = doc_topic_dists.shape
    errors = []

    def err(msg):
        errors.append(msg)

    if dtds[1] != ttds[0]:
        err_msg = ('Number of rows of topic_term_dists does not match number of columns of '
                   'doc_topic_dists; both should be equal to the number of topics in the model.')
        err(err_msg)

    if len(doc_lengths) != dtds[0]:
        err_msg = ('Length of doc_lengths not equal to the number of rows in doc_topic_dists;'
                   'both should be equal to the number of documents in the data.')
        err(err_msg)

    W = len(vocab)
    if ttds[1] != W:
        err_msg = ('Number of terms in vocabulary does not match the number of columns of '
                   'topic_term_dists (where each row of topic_term_dists is a probability '
                   'distribution of terms for a given topic)')
        err(err_msg)
    if len(term_frequency) != W:
        err_msg = ('Length of term_frequency not equal to the number of terms in the '
                   'number of terms in the vocabulary (len of vocab)')
        err(err_msg)

    if __num_dist_rows__(topic_term_dists) != ttds[0]:
        err('Not all rows (distributions) in topic_term_dists sum to 1.')

    if __num_dist_rows__(doc_topic_dists) != dtds[0]:
        err('Not all rows (distributions) in doc_topic_dists sum to 1.')

    if len(errors) > 0:
        return errors

def _input_validate(*args):
    res = _input_check(*args)
    if res:
        raise ValidationError('\n' + '\n'.join([' * ' + s for s in res]))

def _jensen_shannon(_P, _Q):
    _M = 0.5 * (_P + _Q)
    return 0.5 * (entropy(_P, _M) + entropy(_Q, _M))

def _pcoa(pair_dists, n_components=2):
    """Principal Coordinate Analysis,
    aka Classical Multidimensional Scaling
    """
    # code referenced from skbio.stats.ordination.pcoa
    # https://github.com/biocore/scikit-bio/blob/0.5.0/skbio/stats/ordination/_principal_coordinate_analysis.py

    # pairwise distance matrix is assumed symmetric
    pair_dists = np.asarray(pair_dists, np.float64)

    # perform SVD on double centred distance matrix
    n = pair_dists.shape[0]
    H = np.eye(n) - np.ones((n, n)) / n
    B = - H.dot(pair_dists ** 2).dot(H) / 2
    eigvals, eigvecs = np.linalg.eig(B)

    # Take first n_components of eigenvalues and eigenvectors
    # sorted in decreasing order
    ix = eigvals.argsort()[::-1][:n_components]
    eigvals = eigvals[ix]
    eigvecs = eigvecs[:, ix]

    # replace any remaining negative eigenvalues and associated eigenvectors with zeroes
    # at least 1 eigenvalue must be zero
    eigvals[np.isclose(eigvals, 0)] = 0
    if np.any(eigvals < 0):
        ix_neg = eigvals < 0
        eigvals[ix_neg] = np.zeros(eigvals[ix_neg].shape)
        eigvecs[:, ix_neg] = np.zeros(eigvecs[:, ix_neg].shape)

    return np.sqrt(eigvals) * eigvecs

def js_PCoA(distributions):
    """Dimension reduction via Jensen-Shannon Divergence & Principal Coordinate Analysis
    (aka Classical Multidimensional Scaling)

    Parameters
    ----------
    distributions : array-like, shape (`n_dists`, `k`)
        Matrix of distributions probabilities.

    Returns
    -------
    pcoa : array, shape (`n_dists`, 2)
    """
    dist_matrix = squareform(pdist(distributions, metric=_jensen_shannon))
    return _pcoa(dist_matrix)

def js_MMDS(distributions, **kwargs):
    """Dimension reduction via Jensen-Shannon Divergence & Metric Multidimensional Scaling

    Parameters
    ----------
    distributions : array-like, shape (`n_dists`, `k`)
        Matrix of distributions probabilities.

    **kwargs : Keyword argument to be passed to `sklearn.manifold.MDS()`

    Returns
    -------
    mmds : array, shape (`n_dists`, 2)
    """
    dist_matrix = squareform(pdist(distributions, metric=_jensen_shannon))
    model = MDS(n_components=2, random_state=0, dissimilarity='precomputed', **kwargs)
    return model.fit_transform(dist_matrix)

def js_TSNE(distributions, **kwargs):
    """Dimension reduction via Jensen-Shannon Divergence & t-distributed Stochastic Neighbor Embedding

    Parameters
    ----------
    distributions : array-like, shape (`n_dists`, `k`)
        Matrix of distributions probabilities.

    **kwargs : Keyword argument to be passed to `sklearn.manifold.TSNE()`

    Returns
    -------
    tsne : array, shape (`n_dists`, 2)
    """
    dist_matrix = squareform(pdist(distributions, metric=_jensen_shannon))
    model = TSNE(n_components=2, random_state=0, metric='precomputed', **kwargs)
    return model.fit_transform(dist_matrix)

def _df_with_names(data, index_name, columns_name):
    if type(data) == pd.DataFrame:
        # we want our index to be numbered
        df = pd.DataFrame(data.values)
    else:
        df = pd.DataFrame(data)
    df.index.name = index_name
    df.columns.name = columns_name
    return df

def _series_with_name(data, name):
    if type(data) == pd.Series:
        data.name = name
        # ensures a numeric index
        return data.reset_index()[name]
    else:
        return pd.Series(data, name=name)

def _topic_coordinates(mds, topic_term_dists, topic_proportion, start_index=1):
    K = topic_term_dists.shape[0]
    mds_res = mds(topic_term_dists)
    assert mds_res.shape == (K, 2)
    mds_df = pd.DataFrame({'x': mds_res[:, 0], 'y': mds_res[:, 1],
                           'topics': range(start_index, K + start_index),
                           'cluster': 1, 'Freq': topic_proportion * 100})
    # note: cluster (should?) be deprecated soon. See: https://github.com/cpsievert/LDAvis/issues/26
    return mds_df

def _chunks(l, n):
    """ Yield successive n-sized chunks from l.
    """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def _job_chunks(l, n_jobs):
    n_chunks = n_jobs
    if n_jobs < 0:
        # so, have n chunks if we are using all n cores/cpus
        n_chunks = cpu_count() + 1 - n_jobs

    return _chunks(l, n_chunks)

def _find_relevance(log_ttd, log_lift, R, lambda_):
    relevance = lambda_ * log_ttd + (1 - lambda_) * log_lift
    return relevance.T.apply(lambda topic: topic.nlargest(R).index)

def _find_relevance_chunks(log_ttd, log_lift, R, lambda_seq):
    return pd.concat([_find_relevance(log_ttd, log_lift, R, l) for l in lambda_seq])

def _topic_info(topic_term_dists, topic_proportion, term_frequency, term_topic_freq,
                vocab, lambda_step, R, n_jobs, start_index=1):
    # marginal distribution over terms (width of blue bars)
    term_proportion = term_frequency / term_frequency.sum()

    # compute the distinctiveness and saliency of the terms:
    # this determines the R terms that are displayed when no topic is selected
    tt_sum = topic_term_dists.sum()
    topic_given_term = pd.eval("topic_term_dists / tt_sum")
    log_1 = np.log(pd.eval("(topic_given_term.T / topic_proportion)"))
    kernel = pd.eval("topic_given_term * log_1.T")
    distinctiveness = kernel.sum()
    saliency = term_proportion * distinctiveness
    # Order the terms for the "default" view by decreasing saliency:
    default_term_info = pd.DataFrame({
        'saliency': saliency,
        'Term': vocab,
        'Freq': term_frequency,
        'Total': term_frequency,
        'Category': 'Default'})
    default_term_info = default_term_info.sort_values(
        by='saliency', ascending=False).head(R).drop('saliency', 1)
    # Rounding Freq and Total to integer values to match LDAvis code:
    default_term_info['Freq'] = np.floor(default_term_info['Freq'])
    default_term_info['Total'] = np.floor(default_term_info['Total'])
    ranks = np.arange(R, 0, -1)
    default_term_info['logprob'] = default_term_info['loglift'] = ranks
    default_term_info = default_term_info.reindex(columns=[
        "Term", "Freq", "Total", "Category", "logprob", "loglift"
    ])

    # compute relevance and top terms for each topic
    log_lift = np.log(pd.eval("topic_term_dists / term_proportion")).astype("float64")
    log_ttd = np.log(topic_term_dists).astype("float64")
    lambda_seq = np.arange(0, 1 + lambda_step, lambda_step)

    def topic_top_term_df(tup):
        new_topic_id, (original_topic_id, topic_terms) = tup
        term_ix = topic_terms.unique()
        df = pd.DataFrame({'Term': vocab[term_ix],
                           'Freq': term_topic_freq.loc[original_topic_id, term_ix],
                           'Total': term_frequency[term_ix],
                           'Category': 'Topic%d' % new_topic_id,
                           'logprob': log_ttd.loc[original_topic_id, term_ix].round(4),
                           'loglift': log_lift.loc[original_topic_id, term_ix].round(4),
                         })
        return df.reindex(columns=[
            "Term", "Freq", "Total", "Category", "logprob", "loglift"
        ])

    top_terms = pd.concat(Parallel(n_jobs=n_jobs)
                          (delayed(_find_relevance_chunks)(log_ttd, log_lift, R, ls)
                          for ls in _job_chunks(lambda_seq, n_jobs)))
    topic_dfs = map(topic_top_term_df, enumerate(top_terms.T.iterrows(), start_index))
    return pd.concat([default_term_info] + list(topic_dfs))

def _token_table(topic_info, term_topic_freq, vocab, term_frequency, start_index=1):
    # last, to compute the areas of the circles when a term is highlighted
    # we must gather all unique terms that could show up (for every combination
    # of topic and value of lambda) and compute its distribution over topics.

    # term-topic frequency table of unique terms across all topics and all values of lambda
    term_ix = topic_info.index.unique()
    term_ix = np.sort(term_ix)

    top_topic_terms_freq = term_topic_freq[term_ix]
    # use the new ordering for the topics
    K = len(term_topic_freq)
    top_topic_terms_freq.index = range(start_index, K + start_index)
    top_topic_terms_freq.index.name = 'Topic'

    # we filter to Freq >= 0.5 to avoid sending too much data to the browser
    token_table = pd.DataFrame({'Freq': top_topic_terms_freq.unstack()})\
        .reset_index().set_index('term').query('Freq >= 0.5')

    token_table['Freq'] = token_table['Freq'].round()
    token_table['Term'] = vocab[token_table.index.values].values
    # Normalize token frequencies:
    token_table['Freq'] = token_table.Freq / term_frequency[token_table.index]
    return token_table.sort_values(by=['Term', 'Topic'])

def prepare(topic_term_dists, doc_topic_dists, doc_lengths, vocab, term_frequency,
            R=30, lambda_step=0.01, mds=js_PCoA, n_jobs=-1,
            plot_opts=None, sort_topics=True, start_index=1):
    """Transforms the topic model distributions and related corpus data into
    the data structures needed for the visualization.

    Parameters
    ----------
    topic_term_dists : array-like, shape (`n_topics`, `n_terms`)
        Matrix of topic-term probabilities. Where `n_terms` is `len(vocab)`.
    doc_topic_dists : array-like, shape (`n_docs`, `n_topics`)
        Matrix of document-topic probabilities.
    doc_lengths : array-like, shape `n_docs`
        The length of each document, i.e. the number of words in each document.
        The order of the numbers should be consistent with the ordering of the
        docs in `doc_topic_dists`.
    vocab : array-like, shape `n_terms`
        List of all the words in the corpus used to train the model.
    term_frequency : array-like, shape `n_terms`
        The count of each particular term over the entire corpus. The ordering
        of these counts should correspond with `vocab` and `topic_term_dists`.
    R : int
        The number of terms to display in the barcharts of the visualization.
        Default is 30. Recommended to be roughly between 10 and 50.
    lambda_step : float, between 0 and 1
        Determines the interstep distance in the grid of lambda values over
        which to iterate when computing relevance.
        Default is 0.01. Recommended to be between 0.01 and 0.1.
    mds : function or a string representation of function
        A function that takes `topic_term_dists` as an input and outputs a
        `n_topics` by `2`  distance matrix. The output approximates the distance
        between topics. See :func:`js_PCoA` for details on the default function.
        A string representation currently accepts `pcoa` (or upper case variant),
        `mmds` (or upper case variant) and `tsne` (or upper case variant),
        if `sklearn` package is installed for the latter two.
    n_jobs : int
        The number of cores to be used to do the computations. The regular
        joblib conventions are followed so `-1`, which is the default, will
        use all cores.
    plot_opts : dict, with keys 'xlab' and `ylab`
        Dictionary of plotting options, right now only used for the axis labels.
    sort_topics : sort topics by topic proportion (percentage of tokens covered). Set to false to
        to keep original topic order.
    start_index: how to number topics for prepared data. Defaults to one-based indexing.
        Set to 0 for zero-based indexing.

    Returns
    -------
    prepared_data : PreparedData
        A named tuple containing all the data structures required to create
        the visualization. To be passed on to functions like :func:`display`.
        This named tuple can be represented as json or a python dictionary.
        There is a helper function 'sorted_terms' that can be used to get
        the terms of a topic using lambda to rank their relevance.

    Notes
    -----
    This implements the method of `Sievert, C. and Shirley, K. (2014):
    LDAvis: A Method for Visualizing and Interpreting Topics, ACL Workshop on
    Interactive Language Learning, Visualization, and Interfaces.`

    http://nlp.stanford.edu/events/illvi2014/papers/sievert-illvi2014.pdf

    See Also
    --------
    :func:`save_json`: save json representation of a figure to file
    :func:`save_html` : save html representation of a figure to file
    :func:`show` : launch a local server and show a figure in a browser
    :func:`display` : embed figure within the IPython notebook
    :func:`enable_notebook` : automatically embed visualizations in IPython notebook
   """
    if plot_opts is None:
        plot_opts = {'xlab': 'PC1', 'ylab': 'PC2'}

    # parse mds
    if isinstance(mds, basestring):
        mds = mds.lower()
        if mds == 'pcoa':
            mds = js_PCoA
        elif mds in ('mmds', 'tsne'):
            if sklearn_present:
                mds_opts = {'mmds': js_MMDS, 'tsne': js_TSNE}
                mds = mds_opts[mds]
            else:
                logging.warning('sklearn not present, switch to PCoA')
                mds = js_PCoA
        else:
            logging.warning('Unknown mds `%s`, switch to PCoA' % mds)
            mds = js_PCoA

    # Conceptually, the items in `topic_term_dists` end up as individual rows in the
    # DataFrame, but we can speed up ingestion by treating them as columns and
    # transposing at the end. (This is especially true when the number of terms far
    # exceeds the number of topics.)
    topic_term_dist_cols = [
        pd.Series(topic_term_dist, dtype="float64")
        for topic_term_dist in topic_term_dists
    ]
    topic_term_dists = pd.concat(topic_term_dist_cols, axis=1).T

    topic_term_dists = _df_with_names(topic_term_dists, 'topic', 'term')
    doc_topic_dists = _df_with_names(doc_topic_dists, 'doc', 'topic')
    term_frequency = _series_with_name(term_frequency, 'term_frequency')
    doc_lengths = _series_with_name(doc_lengths, 'doc_length')
    vocab = _series_with_name(vocab, 'vocab')
    _input_validate(topic_term_dists, doc_topic_dists, doc_lengths, vocab, term_frequency)
    R = min(R, len(vocab))

    topic_freq = doc_topic_dists.mul(doc_lengths, axis="index").sum()
    # topic_freq       = np.dot(doc_topic_dists.T, doc_lengths)
    if (sort_topics):
        topic_proportion = (topic_freq / topic_freq.sum()).sort_values(ascending=False)
    else:
        topic_proportion = (topic_freq / topic_freq.sum())

    topic_order = topic_proportion.index
    # reorder all data based on new ordering of topics
    topic_freq = topic_freq[topic_order]
    topic_term_dists = topic_term_dists.iloc[topic_order]
    # Unused: doc_topic_dists = doc_topic_dists[topic_order]

    # token counts for each term-topic combination (widths of red bars)
    term_topic_freq = (topic_term_dists.T * topic_freq).T
    # Quick fix for red bar width bug.  We calculate the
    # term frequencies internally, using the topic term distributions and the
    # topic frequencies, rather than using the user-supplied term frequencies.
    # For a detailed discussion, see: https://github.com/cpsievert/LDAvis/pull/41
    term_frequency = np.sum(term_topic_freq, axis=0)

    topic_info = _topic_info(topic_term_dists, topic_proportion,
                             term_frequency, term_topic_freq, vocab, lambda_step, R,
                             n_jobs, start_index)
    token_table = _token_table(topic_info, term_topic_freq, vocab, term_frequency, start_index)
    topic_coordinates = _topic_coordinates(mds, topic_term_dists, topic_proportion, start_index)
    client_topic_order = [x + start_index for x in topic_order]

    return PreparedData(topic_coordinates, topic_info,
                        token_table, R, lambda_step, plot_opts, client_topic_order)

class PreparedData(namedtuple('PreparedData', ['topic_coordinates', 'topic_info', 'token_table',
                                               'R', 'lambda_step', 'plot_opts', 'topic_order'])):

    def sorted_terms(self, topic=1, _lambda=1):
        """Returns a dataframe using _lambda to calculate term relevance of a given topic."""
        tdf = pd.DataFrame(self.topic_info[self.topic_info.Category == 'Topic' + str(topic)])
        if _lambda < 0 or _lambda > 1:
            _lambda = 1
        stdf = tdf.assign(relevance=_lambda * tdf['logprob'] + (1 - _lambda) * tdf['loglift'])
        return stdf.sort_values('relevance', ascending=False)

    def to_dict(self):
        return {'mdsDat': self.topic_coordinates.to_dict(orient='list'),
                'tinfo': self.topic_info.to_dict(orient='list'),
                'token.table': self.token_table.to_dict(orient='list'),
                'R': self.R,
                'lambda.step': self.lambda_step,
                'plot.opts': self.plot_opts,
                'topic.order': self.topic_order}

    def to_json(self):
        return json.dumps(self.to_dict(), cls=NumPyEncoder)

max_stars_repo_name: vituslehner/pyLDAvis

max_issues_repo_name: ivofurman/pyLDAvis

max_forks_repo_name: ivofurman/pyLDAvis

reason: import numpy,from scipy


text:
# Copyright 2019, 2020 DeepMind Technologies Limited and Adam Gleave
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""CLI script to compare one source model onto a target model."""

import copy
import functools
import logging
import os
import pickle
from typing import Any, Dict, Iterable, Mapping, Type

import numpy as np
import ray
import sacred

from evaluating_rewards import datasets, serialize
from evaluating_rewards.distances import common_config, npec
from evaluating_rewards.rewards import base, comparisons
from evaluating_rewards.scripts import regress_utils, script_utils
from evaluating_rewards.scripts.distances import common

npec_distance_ex = sacred.Experiment("npec_distance")
logger = logging.getLogger("evaluating_rewards.scripts.distances.npec")

common.make_config(npec_distance_ex)
common.make_transitions_configs(npec_distance_ex)

ZERO_CFG = (serialize.ZERO_REWARD, "dummy")

@npec_distance_ex.config
def default_config():
    """Default configuration values."""
    # Parallelization
    ray_kwargs = {}
    num_cpus = 2

    # Aggregation
    n_seeds = 3
    normalize = True  # divide by distance from Zero reward, an upper bound on the distance

    # Model to train and hyperparameters
    model_reward_type = base.MLPRewardModel
    discount = 0.99
    comparison_class = npec.RegressWrappedModel
    comparison_kwargs = {
        "learning_rate": 1e-2,
    }
    total_timesteps = int(5e5)
    batch_size = 4096
    fit_kwargs = {
        "affine_size": 16384,  # number of timesteps to use in pretraining; set to None to disable
    }

    _ = locals()  # quieten flake8 unused variable warning
    del _

@npec_distance_ex.config
def default_kwargs(comparison_class, comparison_kwargs):
    """Sets comparison_kwargs to defaults when not overridden."""
    # TODO(): remove this function when Sacred issue #238 is fixed
    if comparison_class == npec.RegressWrappedModel and "model_wrapper" not in comparison_kwargs:
        comparison_kwargs["model_wrapper"] = npec.equivalence_model_wrapper
    _ = locals()  # quieten flake8 unused variable warning
    del _

@npec_distance_ex.named_config
def alternating_maximization():
    """Use less flexible (but sometimes more accurate) RegressEquivalentLeastSq.

    Uses least-squares loss and affine + potential shaping wrapping.
    """
    comparison_class = npec.RegressEquivalentLeastSqModel
    _ = locals()  # quieten flake8 unused variable warning
    del _

@npec_distance_ex.named_config
def affine_only():
    """Equivalence class consists of just affine transformations."""
    comparison_kwargs = {  # noqa: F841  pylint:disable=unused-variable
        "model_wrapper": functools.partial(npec.equivalence_model_wrapper, potential=False),
    }

@npec_distance_ex.named_config
def no_rescale():
    """Equivalence class are shifts plus potential shaping (no scaling)."""
    comparison_kwargs = {  # noqa: F841  pylint:disable=unused-variable
        "model_wrapper": functools.partial(
            npec.equivalence_model_wrapper,
            affine_kwargs=dict(scale=False),
        ),
    }

@npec_distance_ex.named_config
def shaping_only():
    """Equivalence class consists of just potential shaping."""
    comparison_kwargs = {
        "model_wrapper": functools.partial(npec.equivalence_model_wrapper, affine=False),
    }
    fit_kwargs = {"affine_size": None}
    _ = locals()  # quieten flake8 unused variable warning
    del _

@npec_distance_ex.named_config
def ellp_loss():
    """Use mean (x-y)^p loss, default to p=0.5 (sparsity inducing)"""
    p = 0.5
    # Note if p specified at CLI, it will take priority over p above here
    # (Sacred configuration scope magic).
    comparison_kwargs = {
        "loss_fn": functools.partial(comparisons.ellp_norm_loss, p=p),
    }
    _ = locals()  # quieten flake8 unused variable warning
    del _

# TODO(): add a sparsify named config combining ellp_loss, no_rescale
# and Zero target. (Sacred does not currently support combining named configs
# but they're intending to add it.)

FAST_CONFIG = dict(
    n_seeds=1,
    # Disable studentt_ci and sample_mean_sd since they need >1 seed (this test is already slow)
    aggregate_kinds=("bootstrap",),
    fit_kwargs={"affine_size": 512},
    ray_kwargs={
        # CI build only has 1 core per test
        "num_cpus": 1,
    },
    num_cpus=1,
    batch_size=512,
    total_timesteps=2048,
)

@npec_distance_ex.named_config
def fast():
    """Small number of epochs, finish quickly, intended for tests / debugging."""
    locals().update(**FAST_CONFIG)

@npec_distance_ex.named_config
def test():
    """Eliminate parallelism in VecEnv to reduce load on CI."""
    locals().update(**FAST_CONFIG)
    visitations_factory_kwargs = {  # noqa: F841  pylint:disable=unused-variable
        "env_name": "evaluating_rewards/PointMassLine-v0",
        "parallel": False,
        "policy_type": "random",
        "policy_path": "dummy",
    }

@npec_distance_ex.named_config
def high_precision():
    """Increase number of timesteps to increase change of convergence."""
    total_timesteps = int(1e6)  # noqa: F841  pylint:disable=unused-variable

script_utils.add_logging_config(npec_distance_ex, "npec")

@ray.remote
def npec_worker(
    seed: int,
    # Dataset
    env_name: str,
    discount: float,
    visitations_factory,
    visitations_factory_kwargs: Dict[str, Any],
    # Models to compare
    source_reward_cfg: common_config.RewardCfg,
    target_reward_cfg: common_config.RewardCfg,
    # Model parameters
    comparison_class: Type[comparisons.RegressModel],
    comparison_kwargs: Dict[str, Any],
    total_timesteps: int,
    batch_size: int,
    fit_kwargs: Dict[str, Any],
    # Logging
    log_dir: str,
) -> comparisons.FitStats:
    """Performs a single NPEC comparison by fitting a model.

    Args:
        seed: Seed used for visitation factory and model initialization.
        env_name: the name of the environment to compare rewards for.
        discount: discount to use for reward models (mostly for shaping).
        visitations_factory: factory to sample transitions from during training.
        visitations_factory_kwargs: keyword arguments for the visitations factory.
        source_reward_cfg: specifies the serialized source reward.
        target_reward_cfg: specifies the serialized target reward to fit the source onto.
        comparison_class: how to fit the source onto target.
        comparison_kwargs: keyword arguments customizing `comparison_class`.
        total_timesteps: the total number of timesteps to train for.
        batch_size: the number of timesteps in each training batch.
        fit_kwargs: extra arguments to pass to the `fit` method of `comparison_class`.
        log_dir: directory to save data to.

    Returns:
        Statistics for training, including the final loss aka estimated NPEC distance.
    """
    # Configure logging, since Ray children do not by default inherit logging configs.
    script_utils.configure_logging()

    with visitations_factory(seed=seed, **visitations_factory_kwargs) as dataset_generator:

        def make_source(venv):
            kind, path = source_reward_cfg
            return serialize.load_reward(kind, path, venv, discount)

        def make_trainer(model, model_scope, target):
            del model_scope
            return comparison_class(model, target, **comparison_kwargs)

        def do_training(target, trainer, callback):
            del target
            return trainer.fit(
                dataset_generator,
                total_timesteps=total_timesteps,
                batch_size=batch_size,
                callback=callback,
                **fit_kwargs,
            )

        target_reward_type, target_reward_path = target_reward_cfg
        return regress_utils.regress(
            seed=seed,
            env_name=env_name,
            discount=discount,
            make_source=make_source,
            source_init=False,
            make_trainer=make_trainer,
            do_training=do_training,
            target_reward_type=target_reward_type,
            target_reward_path=target_reward_path,
            log_dir=log_dir,
            checkpoint_interval=0,  # disable checkpoints
        )

@npec_distance_ex.capture
def compute_npec(  # pylint:disable=unused-argument
    num_cpus: int,
    seed: int,
    # Dataset
    env_name: str,
    discount: float,
    visitations_factory: datasets.TransitionsFactory,
    visitations_factory_kwargs: Dict[str, Any],
    # Models to compare
    source_reward_cfg: common_config.RewardCfg,
    target_reward_cfg: common_config.RewardCfg,
    # Model parameters
    comparison_class: Type[comparisons.RegressModel],
    comparison_kwargs: Dict[str, Any],
    total_timesteps: int,
    batch_size: int,
    fit_kwargs: Dict[str, Any],
    log_dir: str,
) -> ray.ObjectRef:
    """Compute NPEC comparison between reward models in the background.

    Thin wrapper around `npec_worker`; arguments are the same as `npec_worker`.
    This sets up hierarchical log directory structure (sanitizing paths using
    `sanitize_path`) and copies arguments to make them pickleable. It also is
    declared as a Sacred capture function, which `npec_worker` cannot be since
    Sacred experiments are not picklable.
    """
    # deepcopy to workaround sacred issue GH#499
    visitations_factory_kwargs = copy.deepcopy(visitations_factory_kwargs)
    comparison_kwargs = copy.deepcopy(comparison_kwargs)
    fit_kwargs = copy.deepcopy(fit_kwargs)
    log_dir = os.path.join(
        log_dir,
        script_utils.sanitize_path(source_reward_cfg),
        script_utils.sanitize_path(target_reward_cfg),
        str(seed),
    )

    params = locals()
    del params["num_cpus"]
    return npec_worker.options(num_cpus=num_cpus).remote(**params)

@npec_distance_ex.capture
def compute_vals(
    ray_kwargs: Mapping[str, Any],
    n_seeds: int,
    aggregate_fns: Mapping[str, common.AggregateFn],
    log_dir: str,
    x_reward_cfgs: Iterable[common_config.RewardCfg],
    y_reward_cfgs: Iterable[common_config.RewardCfg],
    normalize: bool,
) -> common_config.AggregatedDistanceReturn:
    """Entry-point into script to regress source onto target reward model."""
    if normalize:
        y_reward_cfgs = list(y_reward_cfgs) + [ZERO_CFG]

    ray.init(**ray_kwargs)

    try:
        keys = []
        refs = []
        for target in x_reward_cfgs:
            for source in y_reward_cfgs:
                for seed in range(n_seeds):
                    obj_ref = compute_npec(  # pylint:disable=no-value-for-parameter
                        seed=seed, source_reward_cfg=source, target_reward_cfg=target
                    )
                    keys.append((target, source))
                    refs.append(obj_ref)
        values = ray.get(refs)
    finally:
        ray.shutdown()

    stats = {}
    for k, v in zip(keys, values):
        stats.setdefault(k, []).append(v)

    logger.info("Saving raw statistics")
    with open(os.path.join(log_dir, "stats.pkl"), "wb") as f:
        pickle.dump(stats, f)

    dissimilarities = {k: [v["loss"][-1]["singleton"] for v in s] for k, s in stats.items()}
    if normalize:
        mean = {k: np.mean(v) for k, v in dissimilarities.items()}
        normalized = {}
        for k, v in dissimilarities.items():
            target, source = k
            if source == ZERO_CFG:
                continue
            zero_mean = mean[(target, ZERO_CFG)]
            normalized[k] = [x / zero_mean for x in v]
        dissimilarities = normalized

    return common.aggregate_seeds(aggregate_fns, dissimilarities)

common.make_main(npec_distance_ex, compute_vals)

if __name__ == "__main__":
    script_utils.experiment_main(npec_distance_ex, "npec_distance")

max_issues_repo_name: HumanCompatibleAI/evaluating-rewards


text:
"""Some small edits to json output.

* Float decimals are truncated to three digits
* [x, y, z] vectors are displayed on one line
* Converts numpy arrays to lists and defined objects to dictionaries
* Removes internal parent references in objects
* Atoms and bonds are on one line each (looks more like other chem formats)
"""
from itertools import takewhile
import re
import json

try:
    import numpy as np
    HAS_NUMPY = True
except ImportError:
    HAS_NUMPY = False

load = json.load
loads = json.loads

def compress(obj):
    """Output json without whitespace + object handling."""
    return json.dumps(obj, sort_keys=True, separators=(",", ":"),
                      cls=CustomEncoder)

def dumps(obj):
    """Output json with formatting edits + object handling."""
    return json.dumps(obj, indent=4, sort_keys=True,
                      cls=CustomEncoder)

class CustomEncoder(json.JSONEncoder):
    """Override JSON encoder to make MOF JSON look like CIF/MOL/PDB files."""

    def default(self, obj):
        """Fire when an unserializable object is hit."""
        if hasattr(obj, '__dict__'):
            return self._round_floats(obj.__dict__.copy())
        elif HAS_NUMPY and isinstance(obj, np.ndarray):
            return self._round_floats(obj.copy().tolist())
        else:
            try:
                return self._round_floats(np.asscalar(obj))
            except Exception:
                raise TypeError(f"Object of type {type(obj)} with value of "
                                f"{repr(obj)} is not JSON serializable.")

    def encode(self, obj):
        """Fire for every object."""
        s = super(CustomEncoder, self).encode(self._round_floats(obj))
        # If uncompressed, postprocess for formatting
        if len(s.splitlines()) > 1:
            s = self.postprocess(s)
        return s

    def postprocess(self, json_string):
        """Display float lists as one line in json. Useful for vectors."""
        # As a general rule, all three-element float lists are on one line
        json_string = re.sub(r"\[\s*([-+]?\d*\.?\d*),\s*([-+]?\d*\.?\d*),"
                             r"\s*([-+]?\d*\.?\d*)\s*\]", r"[ \1, \2, \3 ]",
                             json_string)

        # This further compresses atoms and bonds to be on one line
        is_compressing = False
        compressed = []
        spaces = 0
        for row in json_string.split("\n"):
            if is_compressing:
                if row.strip() == "{":
                    compressed.append(row.rstrip())
                elif row.rstrip() == " " * spaces + "],":
                    compressed.append(row.rstrip())
                    is_compressing = False
                else:
                    compressed[-1] += " " + row.strip()
            else:
                compressed.append(row.rstrip())
                if any(a in row for a in ["atoms", "bonds"]):
                    # Fix to handle issues that arise with empty lists
                    if "[]" in row:
                        continue
                    spaces = sum(1 for _ in takewhile(str.isspace, row))
                    is_compressing = True
        return "\n".join(compressed)

    def _round_floats(self, o):
        """Round off floats to clean up outputted JSON.

        From https://stackoverflow.com/a/53798633/1309324
        """
        if isinstance(o, float):
            return round(o, 5)
        if isinstance(o, dict):
            return {k: self._round_floats(v) for k, v in o.items()}
        if isinstance(o, (list, tuple)):
            return [self._round_floats(x) for x in o]
        return o

max_stars_repo_name: dcyang/imolecule

max_issues_repo_name: dcyang/imolecule

max_forks_repo_name: dcyang/imolecule


text:
import cv2
import numpy as np

#ARCHIVO PARA PROBAR MIS HAARC

kernel=np.ones((3,3))
cam = cv2.VideoCapture(0)#("http://192.168.0.4:4747/mjpegfeed")

face_csc = cv2.CascadeClassifier('haarcascade/cascade.xml')

def BuscarPersonas():
	BuscarRostros()

def BuscarRostros():
		for (x, y, w, h) in faces:						#Obtenemos los puntos para el ROI
			roi=gray[y:y+h,x:x+w]										#Area de interés
			blur=cv2.GaussianBlur(roi,(3,3),1)						#Desenfoque
			dilate=cv2.dilate(blur, kernel, iterations=5) #7			#Dilatado
			erode=cv2.erode(dilate,kernel,iterations=5)	#7		#Erosión
			canny=cv2.Canny(erode,150,200)
			contours, hierarchy = cv2.findContours(canny,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
			cv2.drawContours(img[y:y+h,x:x+w],contours,-1,(0,255,0),3)
		#cv2.rectangle(img, (x,y), (x+w, y+h), (255,0,0), 5)
		#Imprimir el texto
			cv2.putText(img, "Rostro detectado", (x+w+ 20, y + 20), cv2.FONT_HERSHEY_COMPLEX, .7,(0,255, 0), 2)
			if faces is ():
				print("No veo rostros")

while(True):
	ret, img = cam.read()

	gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
	faces = face_csc.detectMultiScale(img, 1.4,3)

	BuscarPersonas()

	cv2.imshow('img', img)
	#cv2.imshow('cuerpos',roi)

	key = cv2.waitKey(1)
	if key == 27:
		break

#cam.release()
cv2.destroyAllWindows()

max_stars_repo_name: RodrigoBucrin/VisionJarocha-TopicosIA

max_issues_repo_name: RodrigoBucrin/VisionJarocha-TopicosIA

max_forks_repo_name: RodrigoBucrin/VisionJarocha-TopicosIA


text:
# -*- coding: utf-8 -*-
from os import path, listdir, mkdir

import numpy as np
from skimage.color import label2rgb

np.random.seed(1)
import random

random.seed(1)
import timeit
import cv2
import os
from multiprocessing import Pool
import lightgbm as lgb
from train_classifier import get_inputs

pred_folder = path.join('predictions')

test_pred_folder = path.join(pred_folder, 'masks/ensemble')

lgbm_models_folder = 'lgbm_models'

test_out_folders = ['lgbm_test_sub1']
color_out_folders = ['color_test_sub1']

DATA_THREADS = 8

num_split_iters = 1
folds_count = 30

pixels_threshold = 76

sep_count = 3
sep_thresholds = [0.6, 0.7, 0.8]

best_thrs = [0.3]
step_size=20

def process_images(step):
    gbm_models = []

    for it in range(num_split_iters):
        for it2 in range(folds_count):
            gbm_models.append(
                lgb.Booster(model_file=path.join(lgbm_models_folder, 'gbm_model_{0}_{1}.txt'.format(it, it2))))

    paramss = []
    files = list(reversed(sorted(listdir(test_pred_folder))))
    for filename in files[step:step + step_size]:
        if path.isfile(path.join(test_pred_folder, filename)) and '.png' in filename:
            paramss.append((filename, test_pred_folder, None))
    inputs = []
    inputs2 = []
    labels = []
    labels2 = []
    separated_regions = []
    results = [get_inputs(param[0], param[1]) for param in paramss]
    for i in range(len(results)):
        inp, lbl, inp2, lbl2, sep_regs = results[i]
        inputs.append(inp)
        inputs2.append(inp2)
        labels.append(lbl)
        labels2.append(lbl2)
        separated_regions.append(sep_regs)
    for sub_id in range(1):
        bst_k = np.zeros((sep_count + 1))
        removed = 0
        replaced = 0
        total_cnt = 0
        im_idx = 0

        empty_cnt = 0

        for filename in files[step:step + step_size]:
            if path.isfile(path.join(test_pred_folder, filename)) and '.png' in filename:
                img_id = filename

                inp = inputs[im_idx]
                pred = np.zeros((inp.shape[0]))
                pred2 = [np.zeros((inp2.shape[0])) for inp2 in inputs2[im_idx]]

                for m in gbm_models:
                    if pred.shape[0] > 0:
                        pred += m.predict(inp)
                    for k in range(len(inputs2[im_idx])):
                        if pred2[k].shape[0] > 0:
                            pred2[k] += m.predict(inputs2[im_idx][k])
                if pred.shape[0] > 0:
                    pred /= len(gbm_models)
                for k in range(len(pred2)):
                    if pred2[k].shape[0] > 0:
                        pred2[k] /= len(gbm_models)

                pred_labels = np.zeros_like(labels[im_idx], dtype='uint16')

                clr = 1

                for i in range(pred.shape[0]):
                    max_sep = -1
                    max_pr = pred[i]
                    for k in range(len(separated_regions[im_idx])):
                        if len(separated_regions[im_idx][k][i]) > 0:
                            pred_lvl2 = pred2[k][separated_regions[im_idx][k][i]]
                            if len(pred_lvl2) > 1 and pred_lvl2.mean() > max_pr:
                                max_sep = k
                                max_pr = pred_lvl2.mean()
                                break
                            if len(pred_lvl2) > 1 and pred_lvl2.max() > max_pr:
                                max_sep = k
                                max_pr = pred_lvl2.max()

                    if max_sep >= 0:
                        pred_lvl2 = pred2[max_sep][separated_regions[im_idx][max_sep][i]]
                        replaced += 1
                        for j in separated_regions[im_idx][max_sep][i]:
                            if pred2[max_sep][j] > best_thrs[sub_id]:
                                pred_labels[labels2[im_idx][max_sep] == j + 1] = clr
                                clr += 1
                            else:
                                removed += 1
                    else:
                        if pred[i] > best_thrs[sub_id]:
                            pred_labels[labels[im_idx] == i + 1] = clr
                            clr += 1
                        else:
                            removed += 1
                    bst_k[max_sep + 1] += 1

                clr_labels = label2rgb(pred_labels, bg_label=0)
                clr_labels *= 255
                clr_labels = clr_labels.astype('uint8')
                cv2.imwrite(path.join(pred_folder, color_out_folders[sub_id], img_id), clr_labels, [cv2.IMWRITE_PNG_COMPRESSION, 9])

                total_cnt += pred_labels.max()

                cv2.imwrite(path.join(pred_folder, test_out_folders[sub_id], filename[:-4]+".tif"), pred_labels)
                im_idx += 1

        print('total_cnt', total_cnt, 'removed', removed, 'replaced', replaced, 'empty:', empty_cnt)
        print(bst_k)

if __name__ == '__main__':
    t0 = timeit.default_timer()
    os.makedirs(pred_folder, exist_ok=True)
    for f in test_out_folders:
        if not path.isdir(path.join(pred_folder, f)):
            mkdir(path.join(pred_folder, f))

    for f in color_out_folders:
        if not path.isdir(path.join(pred_folder, f)):
            mkdir(path.join(pred_folder, f))

    steps = [step for step in range(0, 940, step_size)]
    with Pool(processes=DATA_THREADS) as pool:
        results = pool.map(process_images, steps)

    elapsed = timeit.default_timer() - t0
    print('Time: {:.3f} min'.format(elapsed / 60))


text:
#############################################################################
# Import                                                                    #
#############################################################################
import os
import random
import PIL.Image as Image
from tqdm import tqdm

import numpy as np
import scipy.io

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
from torch.autograd import Function
import torch.nn.functional as F

class DotDict(dict):
    """dot.notation access to dictionary attributes"""
    __getattr__ = dict.get
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

#############################################################################
# Hyperparameters                                                           #
#############################################################################
opt = DotDict()

opt.dataset = '3Dchairs'        # [ celebA | 102flowers | 3Dchairs ]
opt.dataPath = './data'

# Input space
opt.nc = 3                    # number of input channels
opt.sizeX = 64                # size of the image
opt.sizeZ = 64                # size of random noise vectors

# Convolution settings
opt.nf = 64                   # base number of filter in G and D
opt.nLayers = 4               # number of conv layers in G and D

# Hardward settings
opt.workers = 4               # workers data for preprocessing
opt.cuda = True               # use CUDA
opt.gpu = 0                   # GPU id

# Optimisation scheme
opt.batchSize = 128           # minibatch size
opt.nIteration = 1000001      # number of training iterations
opt.lrG = 5e-5                # learning rate for G
opt.lrD = 5e-5                # learning rate for D

# Save/Load networks
opt.checkpointDir = '.'       # checkpoints directory
opt.load = 0                  # if > 0, load given checkpoint
opt.checkpointFreq = 1000     # frequency of checkpoints (in number of epochs)

#############################################################################
# Loading Weights                                                           #
#############################################################################
opt.netG = ''
opt.netD = ''
opt.netE = ''
if opt.load > 0:
    opt.netG = '%s/netG_%d.pth' % (opt.checkpointDir, opt.load)
    opt.netD = '%s/netD_%d.pth' % (opt.checkpointDir, opt.load)
    opt.netE = '%s/netE_%d.pth' % (opt.checkpointDir, opt.load)

#############################################################################
# RandomSeed                                                                #
#############################################################################
opt.manualSeed = random.randint(1, 10000)  # fix seed
print("Random Seed: ", opt.manualSeed)
random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)

#############################################################################
# CUDA                                                                      #
#############################################################################
cudnn.benchmark = True
if torch.cuda.is_available() and not opt.cuda:
    print("WARNING: You have a CUDA device, so you should probably run with --cuda")
if opt.cuda:
    torch.cuda.set_device(opt.gpu)

#############################################################################
# Dataloader                                                                #
#############################################################################
class PairCelebADataset(torch.utils.data.Dataset):
    def __init__(self, dataPath, labelFile, transform=transforms.ToTensor()):
        super(PairCelebADataset, self).__init__()
        self.dataPath = dataPath
        with open(labelFile, 'r') as f:
            lines = np.array([p.split() for p in f.readlines()])
        self.files = lines[:,0]
        self.labels = lines[:,1].astype(int)
        self.transform = transform
    def __len__(self):
        return len(self.files)
    def __getitem__(self, idx):
        label = self.labels[idx]
        file1 = self.files[idx]
        file2 = np.random.choice(self.files[self.labels == label])
        img1 = self.transform(Image.open(os.path.join(self.dataPath, file1)))
        img2 = self.transform(Image.open(os.path.join(self.dataPath, file2)))
        return img1, img2, torch.LongTensor(1).fill_(int(label))

class Pair3DchairsDataset(torch.utils.data.Dataset):
    def __init__(self, dataPath, transform=transforms.ToTensor()):
        super(Pair3DchairsDataset, self).__init__()
        self.dataPath = dataPath
        self.folders = np.array(os.listdir(dataPath))
        self.transform = transform
    def __len__(self):
        return len(self.folders)
    def __getitem__(self, idx):
        idA, idB = np.random.choice(os.listdir(os.path.join(self.dataPath, self.folders[idx])),2)
        label = idx
        imgA = Image.open(os.path.join(self.dataPath, self.folders[idx], idA))
        imgB = Image.open(os.path.join(self.dataPath, self.folders[idx], idB))
        imgA = self.transform(imgA)
        imgB = self.transform(imgB)
        return imgA, imgB, torch.LongTensor(1).fill_(int(label))

class Pair102flowersDataset(torch.utils.data.Dataset):
    def __init__(self, dataPath, labelFile, nc, transform=transforms.ToTensor()):
        super(Pair102flowersDataset, self).__init__()
        self.dataPath = dataPath
        self.files = np.sort(os.listdir(dataPath))
        self.labels = scipy.io.loadmat(labelFile)['labels'][0]
        self.transform = transform
        self.nc = nc
    def __len__(self):
        return len(self.files)
    def __getitem__(self, idx):
        label = self.labels[idx]
        fileA = self.files[idx]
        fileB = np.random.choice(self.files[self.labels == label])
        imgA = Image.open(os.path.join(self.dataPath, fileA))
        imgB = Image.open(os.path.join(self.dataPath, fileB))
        imgA = self.transform(imgA)
        imgB = self.transform(imgB)
        if imgA.size(0) == 1:
            imgA = imgA.repeat(self.nc,1,1)
        if imgB.size(0) == 1:
            imgB = imgB.repeat(self.nc,1,1)
        return imgA[:self.nc], imgB[:self.nc], torch.LongTensor(1).fill_(int(label))

#############################################################################
# Datasets                                                                  #
#############################################################################
if opt.dataset == 'celebA':
    dataset = PairCelebADataset(os.path.join(opt.dataPath, "celebA/aligned"),
                                os.path.join(opt.dataPath, "celebA/identity_celebA_train.txt"),
                                transforms.Compose([transforms.CenterCrop(128),
                                                    transforms.Resize(opt.sizeX),
                                                    transforms.ToTensor(),
                                                    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                                ]))
    testset = PairCelebADataset(os.path.join(opt.dataPath, "celebA/aligned"),
                                os.path.join(opt.dataPath, "celebA/identity_celebA_val.txt"),
                                transforms.Compose([transforms.CenterCrop(128),
                                                    transforms.Resize(opt.sizeX),
                                                    transforms.ToTensor(),
                                                    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                                ]))
elif opt.dataset == '3Dchairs':
    dataset = Pair3DchairsDataset(os.path.join(opt.dataPath, "rendered_chairs/train"),
                                  transforms.Compose([transforms.CenterCrop(300),
                                                      transforms.Resize(opt.sizeX),
                                                      transforms.ToTensor(),
                                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                                  ]))
    testset = Pair3DchairsDataset(os.path.join(opt.dataPath, "rendered_chairs/val"),
                                  transforms.Compose([transforms.CenterCrop(300),
                                                      transforms.Resize(opt.sizeX),
                                                      transforms.ToTensor(),
                                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                                  ]))

dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize,
                                         shuffle=True, num_workers=int(opt.workers))

#############################################################################
# weights init                                                              #
#############################################################################
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        if m.weight:
            m.weight.data.normal_(1.0, 0.02)
            m.bias.data.fill_(0)

#############################################################################
# Modules                                                                   #
#############################################################################
class _dcEncoder(nn.Module):
    def __init__(self, nIn=3, nOut=1024, nf=64, nLayer=4, sizeX=64):
        super(_dcEncoder, self).__init__()
        self.mods = nn.Sequential()
        sizeX = sizeX //2
        self.mods.add_module("Conv0_%dx%dx%d" % (nf, sizeX, sizeX), nn.Conv2d(nIn, nf, 4, 2, 1, bias=False))
        self.mods.add_module("BN0", nn.BatchNorm2d(nf))
        self.mods.add_module("ReLU0", nn.ReLU(True))
        for i in range(1,nLayer):
            sizeX = sizeX //2
            self.mods.add_module("Conv%d_%dx%dx%d" % (i, nf*2, sizeX, sizeX), nn.Conv2d(nf, nf*2, 4, 2, 1, bias=False))
            self.mods.add_module("BN%d"% i, nn.BatchNorm2d(nf*2))
            self.mods.add_module("ReLU%d" % i, nn.ReLU(True))
            nf = nf * 2
        self.mods.add_module("FC_%dx1x1" % nOut, nn.Conv2d(nf, nOut, sizeX, bias=False))
        weights_init(self.mods)
    def forward(self, x):
        return self.mods(x)

class _dcDecoder(nn.Module):
    def __init__(self, nIn=1024, nOut=3, nf=512, nLayer=4, sizeX=64):
        super(_dcDecoder, self).__init__()
        sizeX = sizeX // (2**nLayer)
        nf = nf * (2 ** (nLayer - 1))
        self.mods = nn.Sequential()
        self.mods.add_module("FC_%dx%dx%d" % (nf,sizeX,sizeX), nn.ConvTranspose2d(nIn, nf, sizeX, bias=False))
        self.mods.add_module("BN0", nn.BatchNorm2d(nf))
        self.mods.add_module("ReLU0", nn.ReLU(True))
        for i in range(1,nLayer):
            sizeX = sizeX * 2
            self.mods.add_module("ConvTr%d_%dx%dx%d" % (i, nf//2, sizeX, sizeX), nn.ConvTranspose2d(nf, nf//2, 4, 2, 1, bias=False))
            self.mods.add_module("BN%d"% i, nn.BatchNorm2d(nf//2))
            self.mods.add_module("ReLU%d" % i, nn.ReLU(True))
            nf = nf // 2
        self.mods.add_module("ConvTrO_%dx%dx%d" % (nf, sizeX, sizeX), nn.ConvTranspose2d(nf, nOut, 4, 2, 1, bias=False))
        weights_init(self.mods)
    def forward(self, x):
        return self.mods(x)

class _dcDiscriminator(nn.Module):
    def __init__(self, nIn=3, nOut=1024, nf=64, nLayer=4, sizeX=64):
        super(_dcDiscriminator, self).__init__()
        self.mods = nn.Sequential()
        sizeX = sizeX //2
        self.mods.add_module("Conv0_%dx%dx%d" % (nf, sizeX, sizeX), nn.Conv2d(nIn, nf, 4, 2, 1, bias=False))
        self.mods.add_module("LReLU0", nn.LeakyReLU(0.2))
        for i in range(1,nLayer):
            sizeX = sizeX //2
            self.mods.add_module("Conv%d_%dx%dx%d" % (i, nf*2, sizeX, sizeX), nn.Conv2d(nf, nf*2, 4, 2, 1, bias=False))
            self.mods.add_module("BN%d"% i, nn.BatchNorm2d(nf*2))
            self.mods.add_module("LReLU%d" % i, nn.LeakyReLU(0.2))
            nf = nf * 2
        self.mods.add_module("FC_%dx1x1" % nOut, nn.Conv2d(nf, nOut, sizeX, bias=False))
        weights_init(self.mods)
    def forward(self, x):
        return self.mods(x)

netG = _dcDecoder(nIn=opt.sizeZ*2, nOut=opt.nc, nf=opt.nf, nLayer=opt.nLayers, sizeX=opt.sizeX)
netD = _dcDiscriminator(nIn=opt.nc*2, nOut=1, nf=opt.nf, nLayer=opt.nLayers, sizeX=opt.sizeX)
netE = _dcEncoder(nIn=opt.nc, nOut=opt.sizeZ, nf=opt.nf, nLayer=opt.nLayers, sizeX=opt.sizeX)

if  opt.netG != '':
    netG.load_state_dict(torch.load(opt.netG))
if opt.netD != '':
    netD.load_state_dict(torch.load(opt.netD))
if opt.netE != '':
    netE.load_state_dict(torch.load(opt.netE))

print(netG)
print(netD)
print(netE)

discriminationLoss = nn.BCEWithLogitsLoss()

#############################################################################
# Placeholders                                                              #
#############################################################################
x1_real = torch.FloatTensor(opt.batchSize, opt.nc, opt.sizeX, opt.sizeX)
x2_real = torch.FloatTensor(opt.batchSize, opt.nc, opt.sizeX, opt.sizeX)
zView1 = torch.FloatTensor(opt.batchSize, opt.sizeZ, 1, 1).normal_()
zView2 = torch.FloatTensor(opt.batchSize, opt.sizeZ, 1, 1).normal_()
labelPos = torch.FloatTensor(opt.batchSize)
labelNeg = torch.FloatTensor(opt.batchSize)

#############################################################################
# Test data                                                                 #
#############################################################################
batch_test = 16
nViews = 10

testloader = torch.utils.data.DataLoader(testset, batch_size=batch_test, shuffle=True, num_workers=int(opt.workers))

x1_test, x2_test, _ = next(iter(testloader))
zView_test = torch.FloatTensor(nViews, 1, opt.sizeZ, 1, 1).normal_().repeat(1, batch_test, 1, 1, 1)

#############################################################################
# To Cuda                                                                   #
#############################################################################
if opt.cuda:
    print("Convert to Cuda")
    torch.cuda.set_device(opt.gpu)
    netG.cuda()
    netD.cuda()
    netE.cuda()
    discriminationLoss.cuda()
    x1_real = x1_real.cuda()
    x2_real = x2_real.cuda()
    zView1 = zView1.cuda()
    zView2 = zView2.cuda()
    labelPos = labelPos.cuda()
    labelNeg = labelNeg.cuda()
    zView_test = zView_test.cuda()
    x1_test = x1_test.cuda()
    x2_test = x2_test.cuda()

#############################################################################
# Optimizer                                                                 #
#############################################################################
optimizerG = optim.Adam(netG.parameters(), lr=opt.lrG, betas=(0.5, 0.999))
optimizerD = optim.Adam(netD.parameters(), lr=opt.lrD, betas=(0.5, 0.999))
optimizerE = optim.Adam(netE.parameters(), lr=opt.lrG, betas=(0.5, 0.999))

#############################################################################
# Train                                                                     #
#############################################################################
print("Start Training")
iteration = opt.load * len(dataloader)
epoch = opt.load

while iteration <= opt.nIteration:
    log_dNeg = []
    log_dPos = []
    for x1_cpu, x2_cpu, _ in tqdm(dataloader):
        #######################
        # Init iteration      #
        #######################
        netG.train()
        netD.train()
        netE.train()
        x1_real.resize_(x1_cpu.size(0), x1_cpu.size(1), x1_cpu.size(2), x1_cpu.size(3)).copy_(x1_cpu)
        x2_real.resize_(x2_cpu.size(0), x2_cpu.size(1), x2_cpu.size(2), x2_cpu.size(3)).copy_(x2_cpu)
        zView1.resize_(x1_cpu.size(0), opt.sizeZ, 1, 1).normal_()
        zView2.resize_(x1_cpu.size(0), opt.sizeZ, 1, 1).normal_()
        labelPos.resize_(x1_cpu.size(0), 1, 1, 1).fill_(.9)
        labelNeg.resize_(x1_cpu.size(0), 1, 1, 1).fill_(.1)
        #######################
        # Train               #
        #######################
        # Generation Objective
        netG.zero_grad()
        netE.zero_grad()
        zContent = netE(Variable(x1_real))
        if (iteration % 3) == 0:
            x1_generated = Variable(x1_real)
            x2_generated = F.tanh(netG(torch.cat((zContent, Variable(zView2)),1)))
        elif (iteration % 3) == 1:
            x1_generated = F.tanh(netG(torch.cat((zContent, Variable(zView1)),1)))
            x2_generated = F.tanh(netG(torch.cat((zContent, Variable(zView2)),1)))
        elif (iteration % 3) == 2:
            x1_generated = F.tanh(netG(torch.cat((zContent, Variable(zView1)),1)))
            x2_generated = Variable(x1_real)
        dGen = netD(torch.cat((x1_generated, x2_generated),1))
        generationObjective = discriminationLoss(dGen, Variable(labelPos))
        generationObjective.backward()
        # Discriminator objective
        netD.zero_grad()
        dPos = netD(Variable(torch.cat((x1_real,x2_real),1)))
        dNeg = netD(torch.cat((x1_generated.detach(), x2_generated.detach()),1))
        discriminationObjective = discriminationLoss(dNeg, Variable(labelNeg)) + discriminationLoss(dPos, Variable(labelPos))
        discriminationObjective.backward()
        # Update weights
        optimizerG.step()
        optimizerE.step()
        optimizerD.step()
        # Logs
        dPos = dPos.detach()
        dNeg = dNeg.detach()
        dPos.volatile = True
        dNeg.volatile = True
        log_dPos.append(F.sigmoid(dPos).data.mean())
        log_dNeg.append(F.sigmoid(dNeg).data.mean())
        iteration += 1
    epoch = epoch+1
    print(epoch,
          np.array(log_dPos).mean(),
          np.array(log_dNeg).mean(),
          )
    with open('logs.dat', 'ab') as f:
        np.savetxt(f, np.vstack((np.array(log_dPos),
                                 np.array(log_dNeg),
                                 )).T)
    if epoch % opt.checkpointFreq == 0:
        netG.eval()
        netE.eval()
        zContent_test = netE(Variable(x1_test,volatile=True))
        out = torch.FloatTensor(x1_test.size(0), nViews+3, opt.nc, opt.sizeX, opt.sizeX).cuda().zero_()
        out[:,0] = x1_test
        out[:,1] = x2_test
        for t in range(nViews):
            out[:,t+3] = F.tanh(netG(torch.cat((zContent_test, Variable(zView_test[t], volatile=True)),1))).data
        vutils.save_image(out.view(-1,opt.nc,opt.sizeX,opt.sizeX), 'out_%d.png' % (epoch+1), nrow=(nViews+3), normalize=True)
        #torch.save(netG.state_dict(), '%s/netG_%d.pth' % (opt.checkpointDir, epoch))
        #torch.save(netD.state_dict(), '%s/netD_%d.pth' % (opt.checkpointDir, epoch))
        #torch.save(netE.state_dict(), '%s/netE_%d.pth' % (opt.checkpointDir, epoch))

max_stars_repo_name: mickaelChen/gmv

max_issues_repo_name: mickaelChen/gmv

max_forks_repo_name: mickaelChen/gmv

reason: import numpy,import scipy


text:
import heat_ninja_v1
import re
import os
import csv
import time

def t1():
    text = """ if Location_Input[0:2] == "AL" or Location_Input[0:4] == "CM21" or Location_Input[0:4] == "CM22" or\
        Location_Input[0:4] == "CM23" or Location_Input[0:2] == "CR" or Location_Input[0:2] == "EC" or\
        Location_Input[0:2] == "HA" or Location_Input[0:2] == "HP" or Location_Input[0:2] == "KT" or\
        Location_Input[0:2] == "LU" or Location_Input[0:2] == "MK" or Location_Input[0:2] == "NW" or\
        Location_Input[0:2] == "OX" or Location_Input[0:2] == "SE" or Location_Input[0:2] == "SG" or\
        Location_Input[0:2] == "SL" or Location_Input[0:2] == "SM" or Location_Input[0:3] == "SN7" or\
        Location_Input[0:2] == "SW" or Location_Input[0:2] == "TW" or Location_Input[0:2] == "UB" or\
        Location_Input[0:2] == "WC" or Location_Input[0:2] == "WD":
    EPC_Outside_Temp = [5.1, 5.6, 7.4, 9.9, 13.0, 16.0, 17.9, 17.8, 15.2, 11.6, 8.0, 5.1]  # Region 1
    EPC_Solar_Irradiance = [30, 56, 98, 157, 195, 217, 203, 173, 127, 73, 39, 24]  # Thames
elif Location_Input[0:2] == "BN" or Location_Input[0:2] == "BR" or Location_Input[0:2] == "CT" or\
        Location_Input[0:2] == "DA" or Location_Input[0:4] == "GU28" or Location_Input[0:4] == "GU29" or\
        Location_Input[0:2] == "ME" or Location_Input[0:4] == "PO18" or Location_Input[0:4] == "PO19" or\
        Location_Input[0:4] == "PO20" or Location_Input[0:4] == "PO21" or Location_Input[0:4] == "PO22" or\
        Location_Input[0:4] == "RH10" or Location_Input[0:4] == "RH11" or Location_Input[0:4] == "RH12" or\
        Location_Input[0:4] == "RH13" or Location_Input[0:4] == "RH14" or Location_Input[0:4] == "RH15" or\
        Location_Input[0:4] == "RH16" or Location_Input[0:4] == "RH17" or Location_Input[0:4] == "RH18" or\
        Location_Input[0:4] == "RH19" or Location_Input[0:4] == "RH20" or Location_Input[0:2] == "TN":
    EPC_Outside_Temp = [5.0, 5.4, 7.1, 9.5, 12.6, 15.4, 17.4, 17.5, 15.0, 11.7, 8.1, 5.2]  # Region 2
    EPC_Solar_Irradiance = [32, 59, 104, 170, 208, 231, 216, 182, 133, 77, 41, 25]  # South East England
elif Location_Input[0:2] == "BH" or Location_Input[0:2] == "DT" or Location_Input[0:4] == "GU11" or\
        Location_Input[0:4] == "GU12" or Location_Input[0:4] == "GU14" or Location_Input[0:4] == "GU30" or\
        Location_Input[0:4] == "GU31" or Location_Input[0:4] == "GU32" or Location_Input[0:4] == "GU33" or\
        Location_Input[0:4] == "GU34" or Location_Input[0:4] == "GU35" or Location_Input[0:4] == "GU46" or\
        Location_Input[0:4] == "GU51" or Location_Input[0:4] == "GU52" or Location_Input[0:2] == "PO" or\
        Location_Input[0:4] == "RG21" or Location_Input[0:4] == "RG22" or Location_Input[0:4] == "RG23" or\
        Location_Input[0:4] == "RG24" or Location_Input[0:4] == "RG25" or Location_Input[0:4] == "RG26" or\
        Location_Input[0:4] == "RG27" or Location_Input[0:4] == "RG28" or Location_Input[0:4] == "RG29" or\
        Location_Input[0:2] == "SO" or Location_Input[0:3] == "SP6" or Location_Input[0:3] == "SP7" or\
        Location_Input[0:3] == "SP8" or Location_Input[0:3] == "SP9" or Location_Input[0:4] == "SP10" or\
        Location_Input[0:4] == "SP11":
    EPC_Outside_Temp = [5.4, 5.7, 7.3, 9.6, 12.6, 15.4, 17.3, 17.3, 15.0, 11.8, 8.4, 5.5]  # Region 3
    EPC_Solar_Irradiance = [35, 62, 109, 172, 209, 235, 217, 185, 138, 80, 44, 27]  # Southern England
elif Location_Input[0:2] == "EX" or Location_Input[0:2] == "PL" or Location_Input[0:2] == "TQ" or\
        Location_Input[0:2] == "TR":
    EPC_Outside_Temp = [6.1, 6.4, 7.5, 9.3, 11.9, 14.5, 16.2, 16.3, 14.6, 11.8, 9.0, 6.4]  # Region 4
    EPC_Solar_Irradiance = [36, 63, 111, 174, 210, 233, 204, 182, 136, 78, 44, 28]  # South West England
elif Location_Input[0:2] == "BA" or Location_Input[0:2] == "BS" or Location_Input[0:2] == "CF" or\
        Location_Input[0:2] == "GL" or Location_Input[0:2] == "TA":
    EPC_Outside_Temp = [4.9, 5.3, 7.0, 9.3, 12.2, 15.0, 16.7, 16.7, 14.4, 11.1, 7.8, 4.9]  # Region 5
    EPC_Solar_Irradiance = [32, 59, 105, 167, 201, 226, 206, 175, 130, 74, 40, 25]  # Severn England and Wales
elif Location_Input[0:2] == "CV" or Location_Input[0:2] == "DE" or Location_Input[0:2] == "DY" or\
        Location_Input[0:2] == "HR" or Location_Input[0:2] == "LE" or Location_Input[0:2] == "NN" or\
        Location_Input[0:3] == "S18" or Location_Input[0:3] == "S32" or Location_Input[0:3] == "S33" or\
        Location_Input[0:3] == "S40" or Location_Input[0:3] == "S41" or Location_Input[0:3] == "S42" or\
        Location_Input[0:3] == "S43" or Location_Input[0:3] == "S44" or Location_Input[0:3] == "S45" or\
        Location_Input[0:4] == "SK13" or Location_Input[0:4] == "SK17" or Location_Input[0:4] == "SK22" or\
        Location_Input[0:4] == "SK23" or Location_Input[0:2] == "ST" or Location_Input[0:2] == "TF" or\
        Location_Input[0:2] == "WR" or Location_Input[0:2] == "WS" or Location_Input[0:2] == "WV":
    EPC_Outside_Temp = [4.3, 4.8, 6.6, 9.0, 11.8, 14.8, 16.6, 16.5, 14.0, 10.5, 7.1, 4.2]  # Region 6
    EPC_Solar_Irradiance = [28, 55, 97, 153, 191, 208, 194, 163, 121, 69, 35, 23]  # Midlands
elif Location_Input[0:2] == "BB" or Location_Input[0:2] == "BL" or Location_Input[0:2] == "CH" or\
        Location_Input[0:2] == "CW" or Location_Input[0:2] == "FY" or Location_Input[0:2] == "OL" or\
        Location_Input[0:2] == "PR" or Location_Input[0:4] == "SY14" or Location_Input[0:2] == "WA" or\
        Location_Input[0:2] == "WN":
    EPC_Outside_Temp = [4.7, 5.2, 6.7, 9.1, 12.0, 14.7, 16.4, 16.3, 14.1, 10.7, 7.5, 4.6]  # Region 7
    EPC_Solar_Irradiance = [24, 51, 95, 152, 191, 203, 186, 152, 115, 65, 31, 20]  # West Pennines England and Wales
elif Location_Input[0:2] == "CA" or Location_Input[0:2] == "DG" or Location_Input[0:3] == "LA7" or\
        Location_Input[0:3] == "LA8" or Location_Input[0:3] == "LA9" or Location_Input[0:4] == "LA10" or\
        Location_Input[0:4] == "LA11" or Location_Input[0:4] == "LA12" or Location_Input[0:4] == "LA13" or\
        Location_Input[0:4] == "LA14" or Location_Input[0:4] == "LA15" or Location_Input[0:4] == "LA16" or\
        Location_Input[0:4] == "LA17" or Location_Input[0:4] == "LA18" or Location_Input[0:4] == "LA19" or\
        Location_Input[0:4] == "LA20" or Location_Input[0:4] == "LA21" or Location_Input[0:4] == "LA22" or\
        Location_Input[0:4] == "LA23":
    EPC_Outside_Temp = [3.9, 4.3, 5.6, 7.9, 10.7, 13.2, 14.9, 14.8, 12.8, 9.7, 6.6, 3.7]  # Region 8
    EPC_Solar_Irradiance = [23, 51, 95, 157, 200, 203, 194, 156, 113, 62, 30, 19]  # NW England and SW Scotland
elif Location_Input[0:3] == "DH4" or Location_Input[0:3] == "DH5" or Location_Input[0:4] == "EH43" or\
        Location_Input[0:4] == "EH44" or Location_Input[0:4] == "EH45" or Location_Input[0:4] == "EH46" or\
        Location_Input[0:2] == "NE" or Location_Input[0:2] == "TD":
    EPC_Outside_Temp = [4.0, 4.5, 5.8, 7.9, 10.4, 13.3, 15.2, 15.1, 13.1, 9.7, 6.6, 3.7]  # Region 9
    EPC_Solar_Irradiance = [23, 50, 92, 151, 200, 196, 187, 153, 11, 61, 30, 18]  # Boarders
elif Location_Input[0:4] == "BD23" or Location_Input[0:4] == "BD24" or Location_Input[0:2] == "DH" or\
        Location_Input[0:2] == "DL" or Location_Input[0:2] == "HG" or Location_Input[0:4] == "LS24" or\
        Location_Input[0:3] == "SR7" or Location_Input[0:3] == "SR8" or Location_Input[0:2] == "TS":
    EPC_Outside_Temp = [4.0, 4.6, 6.1, 8.3, 10.9, 13.8, 15.8, 15.6, 13.5, 10.1, 6.7, 3.8]  # Region 10
    EPC_Solar_Irradiance = [25, 51, 95, 152, 196, 198, 190, 156, 115, 64, 32, 20]  # North East England
elif Location_Input[0:2] == "BD" or Location_Input[0:2] == "DN" or Location_Input[0:2] == "HD" or\
        Location_Input[0:2] == "HU" or Location_Input[0:2] == "HX" or Location_Input[0:2] == "LN" or\
        Location_Input[0:2] == "LS" or Location_Input[0:2] == "NG" or Location_Input[0:3] == "PE9" or\
        Location_Input[0:4] == "PE10" or Location_Input[0:4] == "PE11" or Location_Input[0:4] == "PE12" or\
        Location_Input[0:4] == "PE20" or Location_Input[0:4] == "PE21" or Location_Input[0:4] == "PE22" or\
        Location_Input[0:4] == "PE23" or Location_Input[0:4] == "PE24" or Location_Input[0:4] == "PE25" or\
        Location_Input[0:2] == "WF" or Location_Input[0:4] == "YO15" or Location_Input[0:4] == "YO16" or\
        Location_Input[0:4] == "YO25":
    EPC_Outside_Temp = [4.3, 4.9, 6.5, 8.9, 11.7, 14.6, 16.6, 16.4, 14.1, 10.6, 7.1, 4.2]  # Region 11
    EPC_Solar_Irradiance = [26, 54, 96, 150, 192, 200, 189, 157, 115, 66, 33, 21]  # East Pennines
elif Location_Input[0:2] == "CB" or Location_Input[0:2] == "CM" or Location_Input[0:2] == "CO" or\
        Location_Input[0:3] == "EN9" or Location_Input[0:2] == "IG" or Location_Input[0:2] == "IP" or\
        Location_Input[0:2] == "NR" or Location_Input[0:2] == "PE" or Location_Input[0:2] == "RM" or\
        Location_Input[0:2] == "SS":
    EPC_Outside_Temp = [4.7, 5.2, 7.0, 9.5, 12.5, 15.4, 17.6, 17.6, 15.0, 11.4, 7.7, 4.7]  # Region 12
    EPC_Solar_Irradiance = [30, 58, 101, 165, 203, 220, 206, 173, 128, 74, 39, 24]  # East Anglia
elif Location_Input[0:2] == "LD" or Location_Input[0:4] == "LL23" or Location_Input[0:4] == "LL24" or\
        Location_Input[0:4] == "LL25" or Location_Input[0:4] == "LL26" or Location_Input[0:4] == "LL27" or\
        Location_Input[0:4] == "LL30" or Location_Input[0:4] == "LL31" or Location_Input[0:4] == "LL32" or\
        Location_Input[0:4] == "LL33" or Location_Input[0:4] == "LL34" or Location_Input[0:4] == "LL35" or\
        Location_Input[0:4] == "LL36" or Location_Input[0:4] == "LL37" or Location_Input[0:4] == "LL38" or\
        Location_Input[0:4] == "LL39" or Location_Input[0:4] == "LL40" or Location_Input[0:4] == "LL41" or\
        Location_Input[0:4] == "LL42" or Location_Input[0:4] == "LL43" or Location_Input[0:4] == "LL44" or\
        Location_Input[0:4] == "LL45" or Location_Input[0:4] == "LL46" or Location_Input[0:4] == "LL47" or\
        Location_Input[0:4] == "LL48" or Location_Input[0:4] == "LL49" or Location_Input[0:4] == "LL50" or\
        Location_Input[0:4] == "LL51" or Location_Input[0:4] == "LL52" or Location_Input[0:4] == "LL53" or\
        Location_Input[0:4] == "LL54" or Location_Input[0:4] == "LL55" or Location_Input[0:4] == "LL56" or\
        Location_Input[0:4] == "LL57" or Location_Input[0:4] == "LL58" or Location_Input[0:4] == "LL59" or\
        Location_Input[0:4] == "LL60" or Location_Input[0:4] == "LL61" or Location_Input[0:4] == "LL62" or\
        Location_Input[0:4] == "LL63" or Location_Input[0:4] == "LL64" or Location_Input[0:4] == "LL65" or\
        Location_Input[0:4] == "LL66" or Location_Input[0:4] == "LL67" or Location_Input[0:4] == "LL68" or\
        Location_Input[0:4] == "LL69" or Location_Input[0:4] == "LL70" or Location_Input[0:4] == "LL71" or\
        Location_Input[0:4] == "LL72" or Location_Input[0:4] == "LL73" or Location_Input[0:4] == "LL74" or\
        Location_Input[0:4] == "LL75" or Location_Input[0:4] == "LL76" or Location_Input[0:4] == "LL77" or\
        Location_Input[0:4] == "LL78" or Location_Input[0:3] == "NP8" or Location_Input[0:4] == "SA14" or\
        Location_Input[0:4] == "SA15" or Location_Input[0:4] == "SA16" or Location_Input[0:4] == "SA17" or\
        Location_Input[0:4] == "SA18" or Location_Input[0:4] == "SA19" or Location_Input[0:4] == "SA20" or\
        Location_Input[0:4] == "SA31" or Location_Input[0:4] == "SA32" or Location_Input[0:4] == "SA33" or\
        Location_Input[0:4] == "SA34" or Location_Input[0:4] == "SA35" or Location_Input[0:4] == "SA36" or\
        Location_Input[0:4] == "SA37" or Location_Input[0:4] == "SA38" or Location_Input[0:4] == "SA39" or\
        Location_Input[0:4] == "SA40" or Location_Input[0:4] == "SA41" or Location_Input[0:4] == "SA42" or\
        Location_Input[0:4] == "SA43" or Location_Input[0:4] == "SA44" or Location_Input[0:4] == "SA45" or\
        Location_Input[0:4] == "SA46" or Location_Input[0:4] == "SA47" or Location_Input[0:4] == "SA48" or\
        Location_Input[0:4] == "SA61" or Location_Input[0:4] == "SA62" or Location_Input[0:4] == "SA63" or\
        Location_Input[0:4] == "SA64" or Location_Input[0:4] == "SA65" or Location_Input[0:4] == "SA66" or\
        Location_Input[0:4] == "SA67" or Location_Input[0:4] == "SA68" or Location_Input[0:4] == "SA69" or\
        Location_Input[0:4] == "SA70" or Location_Input[0:4] == "SA71" or Location_Input[0:4] == "SA72" or\
        Location_Input[0:4] == "SA73" or Location_Input[0:4] == "SY15" or Location_Input[0:4] == "SY16" or\
        Location_Input[0:4] == "SY17" or Location_Input[0:4] == "SY18" or Location_Input[0:4] == "SY19" or\
        Location_Input[0:4] == "SY20" or Location_Input[0:4] == "SY21" or Location_Input[0:4] == "SY22" or\
        Location_Input[0:4] == "SY23" or Location_Input[0:4] == "SY24" or Location_Input[0:4] == "SY25":
    EPC_Outside_Temp = [5.0, 5.3, 6.5, 8.5, 11.2, 13.7, 15.3, 15.3, 13.5, 10.7, 7.8, 5.2]  # Region 13
    EPC_Solar_Irradiance = [29, 57, 104, 164, 205, 220, 199, 167, 120, 68, 35, 22]  # Wales
elif Location_Input[0:2] == "FK" or Location_Input[0:2] == "KA" or Location_Input[0:2] == "ML" or\
        Location_Input[0:2] == "PA" or Location_Input[0:4] == "PH49" or Location_Input[0:4] == "PH50":
    EPC_Outside_Temp = [4.0, 4.4, 5.6, 7.9, 10.4, 13.0, 14.5, 14.4, 12.5, 9.3, 6.5, 3.8]  # Region 14
    EPC_Solar_Irradiance = [19, 46, 88, 148, 196, 193, 185, 150, 101, 55, 25, 15]  # West Scotland
elif Location_Input[0:2] == "DD" or Location_Input[0:2] == "EH" or Location_Input[0:2] == "KY":
    EPC_Outside_Temp = [3.6, 4.0, 5.4, 7.7, 10.1, 12.9, 14.6, 14.5, 12.5, 9.2, 6.1, 3.2]  # Region 15
    EPC_Solar_Irradiance = [21, 46, 89, 146, 198, 191, 183, 150, 106, 57, 27, 15]  # East Scotland
elif Location_Input[0:2] == "AB" or Location_Input[0:4] == "IV30" or Location_Input[0:4] == "IV31" or\
        Location_Input[0:4] == "IV32" or Location_Input[0:4] == "IV36" or Location_Input[0:4] == "PH26":
    EPC_Outside_Temp = [3.3, 3.6, 5.0, 7.1, 9.3, 12.2, 14.0, 13.9, 12.0, 8.8, 5.7, 2.9]  # Region 16
    EPC_Solar_Irradiance = [19, 45, 89, 143, 194, 188, 177, 144, 101, 54, 25, 14]  # North East Scotland
elif Location_Input[0:2] == "IV" or Location_Input[0:4] == "PH19" or Location_Input[0:4] == "PH20" or\
        Location_Input[0:4] == "PH21" or Location_Input[0:4] == "PH22" or Location_Input[0:4] == "PH23" or\
        Location_Input[0:4] == "PH24" or Location_Input[0:4] == "PH25" or Location_Input[0:4] == "PH30" or\
        Location_Input[0:4] == "PH31" or Location_Input[0:4] == "PH32" or Location_Input[0:4] == "PH33" or\
        Location_Input[0:4] == "PH34" or Location_Input[0:4] == "PH35" or Location_Input[0:4] == "PH36" or\
        Location_Input[0:4] == "PH37" or Location_Input[0:4] == "PH38" or Location_Input[0:4] == "PH39" or\
        Location_Input[0:4] == "PH40" or Location_Input[0:4] == "PH41" or Location_Input[0:4] == "PH42" or\
        Location_Input[0:4] == "PH43" or Location_Input[0:4] == "PH44":
    EPC_Outside_Temp = [3.1, 3.2, 4.4, 6.6, 8.9, 11.4, 13.2, 13.1, 11.3, 8.2, 5.4, 2.7]  # Region 17
    EPC_Solar_Irradiance = [17, 43, 85, 145, 189, 185, 170, 139, 98, 51, 22, 12]  # Highlands
elif Location_Input[0:2] == "HS":
    EPC_Outside_Temp = [5.2, 5.0, 5.8, 7.6, 9.7, 11.8, 13.4, 13.6, 12.1, 9.6, 7.3, 5.2]  # Region 18
    EPC_Solar_Irradiance = [16, 41, 87, 155, 205, 206, 185, 148, 101, 51, 21, 11]  # Western Isles
elif Location_Input[0:4] == "KW15" or Location_Input[0:4] == "KW16" or Location_Input[0:4] == "KW17":
    EPC_Outside_Temp = [4.4, 4.2, 5.0, 7.0, 8.9, 11.2, 13.1, 13.2, 11.7, 9.1, 6.6, 4.3]  # Region 19
    EPC_Solar_Irradiance = [14, 39, 84, 143, 205, 201, 178, 145, 100, 50, 19, 9]  # Orkney
elif Location_Input[0:2] == "ZE":
    EPC_Outside_Temp = [4.6, 4.1, 4.7, 6.5, 8.3, 10.5, 12.4, 12.8, 11.4, 8.8, 6.5, 4.6]  # Region 20
    EPC_Solar_Irradiance = [12, 34, 79, 135, 196, 190, 168, 144, 90, 46, 16, 7]  # Shetland
elif Location_Input[0:2] == "BT":
    EPC_Outside_Temp = [4.8, 5.2, 6.4, 10.9, 13.5, 15.0, 14.9, 13.1, 10.0, 7.2, 4.7]  # Region 21
    EPC_Solar_Irradiance = [24, 52, 96, 155, 201, 198, 183, 150, 107, 61, 30, 18]  # Northern Ireland"""

    m = re.findall(r'EPC_Outside_Temp (.*?) \#', text)

    for n in m:
        n = n.replace(', ', 'f, ')
        print(f'{{ {n[3:-2]}f }},')

    m = re.findall(r'EPC_Solar_Irradiance (.*?) \#', text)

    print()
    print()
    print()
    print()

    for n in m:
        print(f'{{ {n[3:-2]} }},')

def t2():
    text = """
            Location_Input[0:1] == "L" or Location_Input[0:2] == "LA" or Location_Input[0:2] == "LL" or\
            Location_Input[0:1] == "M" or Location_Input[0:2] == "SK":
    """
    pattern = r'\"(.+?)\"'
    m = re.findall(pattern, text)

    for n in m:
        print(f'"{n}"', end=', ')
    print()
    print()
    print(len(m))

def t3():
    File_Name = "ninja_weather_" + Lat_Rounded + "000_" + Lon_Rounded + "000_uncorrected.csv"
    Outside_Temp = []
    Solar_Irradiance = []
    Coldest_Outside_Temp = 5  # Initial set point, then reduced depending on location weather
    if os.path.isfile("data/" + File_Name):
        Weather_File = open("data/" + File_Name, "r")
        Weather_Data = csv.reader(Weather_File)
        for Row in Weather_Data:
            Outside_Temp.append(float(Row[2]))
            Solar_Irradiance.append(float(Row[3]))
            print(float(Row[2]), float(Row[3]))
            if (float(Row[2])) < Coldest_Outside_Temp:
                Coldest_Outside_Temp = (float(Row[2]))
        Weather_File.close()
    else:
        print("Cannot find weather file for that location")
        exit()

def t4():
    coldest_temps = list()
    for a, b, filenames in os.walk('../data'):
        for j, filename in enumerate(filenames):
            if filename[:5] == "ninja":
                lat = float(filename[14:20])
                lon = float(filename[22:28])
                fname = f'lat_{lat}_lon_{lon}.csv'
                print(fname)
                filename = "../data/" + filename
                if os.path.isfile(filename):
                    # f = open(f'../solar_irradiances/{fname}', 'w')
                    coldest_outside_temp = 5
                    for i, row in enumerate(csv.reader(open(filename, "r"))):
                        outside_temp = float(row[2])
                        # solar_irradiance = float(row[3])
                        # print(f'{solar_irradiance}', end='', file=f)
                        # if i < 8759:
                        #     print('\n', end='', file=f)
                        if outside_temp < coldest_outside_temp:
                            coldest_outside_temp = outside_temp
                    coldest_temps.append(f'{{ "{lat}_{lon}", {coldest_outside_temp}f }}, ')
    for ct in coldest_temps:
        print(ct, end='')

def t5():
    path = 'C:\\Users\\Jack\\Downloads\\postcodes'
    for a, b, filenames in os.walk(path):
        for j, filename in enumerate(filenames):
            print(filename)
            for i, row in enumerate(csv.reader(open(path + '\\' + filename, "r"))):
                print(i, row[0:4])

# def t6(value):
#     hours = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
#     total = 0
#     START_TIME = time.time()
#     for i in range(100000):
#         for day in range(365):
#             for hour in range(24):
#                 total += hours[hour] + value
#     END_TIME = time.time()
#     print(total, END_TIME - START_TIME)
#
# def function(value):
#     hours = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
#     total = 0
#     START_TIME = time.time()
#     for i in range(100000):
#         for day in range(365):
#             for hour in range(24):
#                 total += hours[hour] + value
#     END_TIME = time.time()
#     print(total, END_TIME - START_TIME)

import numpy
def function(value):
    # hours = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
    hours = numpy.arange(24)
    total = 0
    for i in range(100000):
        for day in range(365):
            for hour in range(24):
                total += hours[hour] + value
    return total

if __name__ == "__main__":
    # function(1)
    # print(list(range(24)))
    pass

max_stars_repo_name: Jackrekirby/HeatMyHome-Simulator-Dev

max_issues_repo_name: Jackrekirby/HeatMyHome-Simulator-Dev

max_forks_repo_name: Jackrekirby/Heating-Simulator