text:
from __future__ import absolute_import, division, print_function

import pytest
import numpy as np

from glue.external.six import string_types, PY2  # noqa

from ..array import (view_shape, coerce_numeric, stack_view, unique, broadcast_to,
                     shape_to_string, check_sorted, pretty_number, unbroadcast)

@pytest.mark.parametrize(('before', 'ref_after', 'ref_indices'),
                         (([2.2, 5, 4, 4, 2, 8.3, 2.2], [2, 2.2, 4, 5, 8.3], [1, 3, 2, 2, 0, 4, 1]),
                          ([2.2, 5, np.nan, 2, 8.3, 2.2], [2, 2.2, 5, 8.3], [1, 2, -1, 0, 3, 1])))
def test_unique(before, ref_after, ref_indices):
    after, indices = unique(before)
    np.testing.assert_array_equal(after, ref_after)
    np.testing.assert_array_equal(indices, ref_indices)

def test_shape_to_string():
    assert shape_to_string((1, 4, 3)) == "(1, 4, 3)"

@pytest.mark.skipif("not PY2")
def test_shape_to_string_long():
    # Shape includes long ints on Windows
    assert shape_to_string((long(1), long(4), long(3))) == "(1, 4, 3)"

def test_view_shape():
    assert view_shape((10, 10), np.s_[:]) == (10, 10)
    assert view_shape((10, 10, 10), np.s_[:]) == (10, 10, 10)
    assert view_shape((10, 10), np.s_[:, 1]) == (10,)
    assert view_shape((10, 10), np.s_[2:3, 2:3]) == (1, 1)
    assert view_shape((10, 10), None) == (10, 10)
    assert view_shape((10, 10), ([1, 2, 3], [2, 3, 4])) == (3,)

def test_coerce_numeric():

    x = np.array(['1', '2', '3.14', '4'], dtype=str)
    np.testing.assert_array_equal(coerce_numeric(x),
                                  [1, 2, 3.14, 4])

    x = np.array([1, 2, 3])
    assert coerce_numeric(x) is x

    x = np.array([0, 1, 1, 0], dtype=bool)
    np.testing.assert_array_equal(coerce_numeric(x), np.array([0, 1, 1, 0], dtype=np.int))

@pytest.mark.parametrize(('shape', 'views'),
                         [
                             [(5, 5), (np.s_[0:3],)],
                             [(5, 4), (np.s_[0:3],)],
                             [(5, 4), ((3, 2),)],
                             [(5, 4), (np.s_[0:4], np.s_[:, 0:2])],
                             [(5, 4), (np.s_[0:3, 0:2], 'transpose', (0, 0))],
                             [(10, 20), (np.random.random((10, 20)) > 0.1, 3)],
                             [(5, 7), ('transpose', (3, 2))],
])
def test_stack_view(shape, views):
    x = np.random.random(shape)
    exp = x
    for v in views:
        if isinstance(v, string_types) and v == 'transpose':
            exp = exp.T
        else:
            exp = exp[v]

    actual = x[stack_view(shape, *views)]

    np.testing.assert_array_equal(exp, actual)

@pytest.mark.parametrize(('array', 'is_sorted'),
                         (([1, 3, 4, 3], False), ([1, 2, np.nan, 3], True), ([1, 3, 4, 4.1], True)))
def test_check_sorted(array, is_sorted):
    assert check_sorted(array) is is_sorted

class TestPrettyNumber(object):

    def test_single(self):
        assert pretty_number([1]) == ['1']
        assert pretty_number([0]) == ['0']
        assert pretty_number([-1]) == ['-1']
        assert pretty_number([1.0001]) == ['1']
        assert pretty_number([1.01]) == ['1.01']
        assert pretty_number([1e-5]) == ['1.000e-05']
        assert pretty_number([1e5]) == ['1.000e+05']
        assert pretty_number([3.3]) == ['3.3']
        assert pretty_number([1.]) == ['1']
        assert pretty_number([1.200]) == ['1.2']

    def test_large(self):
        # Regression test or a bug that caused trailing zeros in exponent to
        # be removed.
        assert pretty_number([1e9]) == ['1.000e+09']
        assert pretty_number([2e10]) == ['2.000e+10']
        assert pretty_number([3e11]) == ['3.000e+11']

    def test_list(self):
        assert pretty_number([1, 2, 3.3, 1e5]) == ['1', '2', '3.3',
                                                   '1.000e+05']

def test_unbroadcast():

    x = np.array([1, 2, 3])
    y = broadcast_to(x, (2, 4, 3))

    z = unbroadcast(y)
    assert z.shape == (1, 1, 3)
    np.testing.assert_allclose(z[0, 0], x)

max_stars_repo_name: ejeschke/glue

max_issues_repo_name: ejeschke/glue

max_forks_repo_name: ejeschke/glue


text:
"""
Håndarbeid-opptelling av 62 Støttekonstruksjon (Mur i driftskontrakt-rapportene) for
kontraktsområdet 9305 Sunnfjord 2021-2026

Bruker mitt bibliotek nvdbapi-V3 https://github.com/LtGlahn/nvdbapi-V3
Fila STARTHER.py (https://github.com/LtGlahn/workinprogress) er kun for å føye nvdbapi-V3 til python path (fordi jeg er
for lat til å publisere nvdbapi-V3 som en ordentlig pakke på pipy)

Output fra scriptet (per 22.1.2021)

%run test_mur.py
../../nvdbapi-V3/nvdbapiv3/nvdbapiv3.py:438: UserWarning:
You should provide the file nvdbapi-clientinfo.json

{ "X-Client" : "YOUR SYSTEM",
 "X-Kontaktperson" : "ola.nordmann@eposten.din" }

  warn( mytext )
Objekt 1000 av 1338
Areal fra 521 objekter med Areal-egenskap: 58773.0
Areal fra 192 objekter der vi bruker lengde- og høydeegenskap 19831
Areal fra 131 vegsegmenter (121 objekt) der vi bruker vegnett X høydeegenskap 14844
Areal totalt: 93447.2735

Dobbeltsjekker lengden langs veg også
Lengde ut fra egenskap: 44214
Lengde ut fra vegnett:  16388
Lengde totalt: 60602

"""

import fnmatch
import os
import re
import json
import requests
from copy import deepcopy
import pdb
from datetime import datetime

import pandas as pd
import numpy as np

import STARTHER
import nvdbapiv3

sok = nvdbapiv3.nvdbFagdata(62)
sok.filter( {'kontraktsomrade' : '9305 Sunnfjord 2021-2026' })
mydf = pd.DataFrame( sok.to_records() )

# [{'Beskrivelse': 'Mur',
#   'objtype': 62,
#   'withCount': True,
#   'withLengthPreferingFromAttribute': 1315                1315 => 'Lengde'
#   'withAreaFromAttributeOrCrossSection': [3950, 1582]}]   3950 => 'Areal
#                                                           1582 => 'Høyde, synlig, gjennomsnitt'

# Finner først dem som har  - og ikke har - Areal-egenskap
harAreal  = mydf[ ~mydf['Areal'].isnull() ]
harAreal_utenduplikat = harAreal.drop_duplicates( ['nvdbId'])
arealsum_harAreal = harAreal_utenduplikat['Areal'].sum()
print( f'Areal fra {len(harAreal_utenduplikat)} objekter med Areal-egenskap: {arealsum_harAreal}')

ikkeAreal =  mydf[ mydf['Areal'].isnull() ]

# Finne dem som har egenskapen 'Høyde, synlig, gjennomsnitt'
harHoyde = ikkeAreal[ ~ikkeAreal['Høyde, synlig, gjennomsnitt'].isnull() ]

# Av dem igjen, finne dem som har egenskapen 'Lengde
harLengde     = harHoyde[ ~harHoyde['Lengde'].isnull()]
harLengde_utendup = deepcopy( harLengde.drop_duplicates( subset=['nvdbId']) )
harLengde_utendup['Areal'] = harLengde_utendup['Lengde'] * harLengde_utendup['Høyde, synlig, gjennomsnitt']
arealsum_HxL = harLengde_utendup['Areal'].sum()
print( f"Areal fra {len(harLengde_utendup)} objekter der vi bruker lengde- og høydeegenskap {round(arealsum_HxL)} " )

# Finne dem vi må bruke vegnettslengde for å finne arealet for
vegnettLengde = harHoyde[ harHoyde['Lengde'].isnull()].copy()
vegnettLengde['Areal'] = vegnettLengde['segmentlengde'] * vegnettLengde['Høyde, synlig, gjennomsnitt']
arealsum_vegnett = vegnettLengde['Areal'].sum()
print( f"Areal fra {len(vegnettLengde)} vegsegmenter ({len(vegnettLengde.drop_duplicates(subset=['nvdbId']) ) } objekt) der vi bruker vegnett X høydeegenskap {round(arealsum_vegnett)} " )
areal_totalt = arealsum_harAreal + arealsum_HxL + arealsum_vegnett
print(f"Areal totalt: {areal_totalt}" )

# Siden vi først er i gang så regner vi ut lengden også:
kunLengde  = mydf[ ~mydf['Lengde'].isnull()]
kunLengde_utendup = kunLengde.drop_duplicates( subset=['nvdbId'])
kunVegnett = mydf[  mydf['Lengde'].isnull()]
lengdeSumEgenskap = kunLengde_utendup['Lengde'].sum()
lengdeSumVeg = kunVegnett['segmentlengde'].sum( )
print( '\nDobbeltsjekker lengden langs veg også')
print(f"Lengde ut fra egenskap: {round(lengdeSumEgenskap) } " )
print(f"Lengde ut fra vegnett:  {round(lengdeSumVeg)} " )
print(f"Lengde totalt: {round(lengdeSumEgenskap + lengdeSumVeg  )} " )

ext: py

max_stars_repo_name: LtGlahn/lastned-nvdbrapporter

max_issues_repo_name: LtGlahn/lastned-nvdbrapporter

max_forks_repo_name: LtGlahn/lastned-nvdbrapporter

reason: import numpy


text:
import os
import torch
import hydra
import random
import pytorch_lightning as pl
from pytorch_lightning import loggers as pl_loggers
from lib.utils.common import hydra_params_to_dotdict
import numpy as np

torch.manual_seed(3)
random.seed(3)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(3)

def set_cfg(cfg):
    mode = cfg['run_mode']
    assert mode == 'fit' or mode == 'test'
    if mode == 'fit':
        wb_logger = pl_loggers.WandbLogger(name=cfg['exp_name'], project=cfg['project_name'],
                                           save_dir=cfg['experiment_path'], sync_step=True, offline=False)
        wb_upload_path = wb_logger.experiment.dir
        cfg['preds_path'] = os.path.join(wb_upload_path, mode, 'overall/predictions')
        cfg['best_preds_path'] = os.path.join(wb_upload_path, mode, 'best/predictions')
        cfg['checkpoints_path'] = os.path.join(wb_upload_path, mode, 'best/checkpoints')

    elif mode == 'test':
        wb_logger = None
        if cfg['manual_ckpt']:
            print("[INFO.TEST.LOAD_MANUAL_CHECKPOINT]")
            pass # checkpoint will loaded inside on_test_start callback.
        else:
            print("[INFO.TEST.LOAD_AUTOMATIC_CHECKPOINT]")
            best_checkpoint_dir = os.path.join(cfg['experiment_path'], 'wandb', 'latest-run', 'files', 'fit', 'best',
                                            'checkpoints')
            best_checkpoint = [i for i in os.listdir(best_checkpoint_dir) if '.ckpt' in i]
            print(best_checkpoint)
            assert len(best_checkpoint) == 1
            cfg['best_checkpoint_path'] = os.path.join(best_checkpoint_dir, best_checkpoint[0])

        eval_path = os.path.join(cfg['experiment_path'], 'wandb', 'latest-run', 'files', 'test')
        cfg['preds_path'] = os.path.join(eval_path, 'overall/predictions')
        cfg['best_preds_path'] = os.path.join(eval_path, 'best/predictions')

    else:
        raise NotImplementedError('Mode {} not implemented.'.format(mode))

    return cfg, wb_logger

@hydra.main("lib/config/config.yaml")
def main(cfg):
    cfg, wb_logger = set_cfg(cfg=cfg)
    model = hydra.utils.instantiate(cfg.task_data_model, hydra_params_to_dotdict(cfg))
    gpus = list(range(torch.cuda.device_count()))
    trainer = pl.Trainer(
        gpus=gpus,
        min_epochs=cfg.epochs,
        max_epochs=cfg.epochs,
        check_val_every_n_epoch=1,
        num_sanity_val_steps=0,
        distributed_backend=cfg.distrib_backend,
        logger=wb_logger
    )

    if cfg['run_mode'] == 'fit':
        trainer.fit(model)
        trainer.test(ckpt_path='best')

    elif cfg['run_mode'] == 'test':
        if cfg['manual_ckpt']:
            pass
        else:
            print("________________________________________________________")
            print("\n[INFO.CHECKPOINT]: {}".format(cfg['best_checkpoint_path']))
            checkpoint = torch.load(cfg['best_checkpoint_path'], map_location=lambda storage, loc: storage)
            model.load_state_dict(checkpoint['state_dict'], strict=False)  # eg: embeddings will be loaded separately on model init
        trainer.test(model)
    else:
        raise NotImplementedError;

if __name__ == "__main__":
    main()

max_stars_repo_name: aligholami/kepler

max_issues_repo_name: aligholami/kepler

max_forks_repo_name: aligholami/kepler


text:
"""
.. module:: text_processing_methods
   :synopsis: Holding processing classes!
.. moduleauthor:: Klaus Kades
"""
import os
import re
from abc import ABC, abstractmethod
from typing import List

import gensim
import pandas as pd
import numpy as np
from nltk.util import ngrams
from pattern.en import parse
from difflib import SequenceMatcher
from pycontractions import Contractions
from segtok.segmenter import split_single
from segtok.tokenizer import split_contractions
from segtok.tokenizer import word_tokenizer

from mtc.helpers.text2digits import Text2Digits
from mtc.settings import NLP_MODELS_PATH
from mtc.mtc_properties import STOP_WORDS

def TextProcessing(name, *args, **kwargs):
    """
    All text processing classes should be called via this method
    """
    for cls in TextProcessingBaseClass.__subclasses__():
        if cls.__name__ == name:
            return cls(*args, **kwargs)
    raise ValueError('No text processing named %s' % name)

class TextProcessingBaseClass(ABC):
    """
    Any text processing class must inherit from this class
    """

    def preprocess(self,  sentence_list: List):

        if self.processing_type == 'corpus':
            return self._process_internal(sentence_list)
        else:
            processed_texts = []
            for text_element in sentence_list:
                text_element = self._process_internal(text_element)
                processed_texts.append(text_element)
            return processed_texts

    @property
    def processing_type(self):
        return 'sentence_level'

    @abstractmethod
    def level(self) -> str:
        pass

    @abstractmethod
    def _process_internal(self, sentence_list: List[str]) -> List[str]:
        """Private method to process a piece of text"""
        pass

class ContractionExpander(TextProcessingBaseClass):
    '''
    Removes contractions from the text and uses the full version instead (unification).

    Example:
    I'll walk down the road --> I will walk down the road
    '''

    model_contraction_expander = None

    def __init__(self, model=None):
        '''
        :param model: Pretrained word embedding model.
        '''
        super().__init__()

        if model is None:
            # If no model is given, use the default one and store it as a static class variable to avoid multiple loadings
            if ContractionExpander.model_contraction_expander is None:
                model_path = os.path.join(NLP_MODELS_PATH, 'pretrained', 'word_embeddings', 'pubmed2018_w2v_400D',
                                  'pubmed2018_w2v_400D.bin')
                ContractionExpander.model_contraction_expander = gensim.models.KeyedVectors.load_word2vec_format(model_path, binary=True)

            model = ContractionExpander.model_contraction_expander

        self.cont = Contractions(kv_model=model)

    def level(self) -> str:
        return "text"

    def _process_internal(self, text: str) -> str:
        '''
        :param text: Input string.
        :return: The string without contractions.
        '''
        return list(self.cont.expand_texts([text], precise=True))[0]

class NumberUnifier(TextProcessingBaseClass):
    '''
    Unifies number representations in the text.

    Example:
    One capsule mouth two times day --> 1 capsule mouth 2 times day
    '''

    def __init__(self):
        super().__init__()
        self.t2n = Text2Digits()

    def level(self) -> str:
        return "text"

    def _process_internal(self, text: str) -> str:
        '''
        :param text: Input string.
        :return: The string without any numbers written-out.
        '''
        text = re.sub(r'(?:one)?-?half\b', '0.5', text)
        text = re.sub(r'(\d)\s*(?:to|-)\s*(\d)', r'\1-\2', text)
        text = self.t2n.convert(text)

        return text

class SpellingCorrector(TextProcessingBaseClass):
    '''
    Tries to correct some common spelling errors specific to the sts training dataset.

    Note that the case of the spell-corrected words is not preserved.

    Example:
        Pleast let me know --> Please let me know
    '''

    def __init__(self):
        self.errors = {
            'Pleast': 'please',
            'Locaation': 'location',
            'CURRENTand': 'current and',
            'LocationEmergency': 'location emergency',
            'refil': 'refill',
            'EDUCATIONReady': 'education ready',
            'd aily': 'daily'
        }

    def level(self) -> str:
        return "text"

    def _process_internal(self, text: str) -> str:
        for error, correction in self.errors.items():
            text = re.sub(r'\b' + error + r'\b', correction, text)

        return text

class RemoveActiveIngredients(TextProcessingBaseClass):
    '''
    Removes active ingredients: i.e: aw [aber] -> aw
    '''

    def level(self) -> str:
        return "text"

    def _process_internal(self, text: str) -> str:
        text = re.sub(r'\[([^]]+)\]', '', text)
            #text = re.sub(r'\b' + error + r'\b', correction, text)
        return text

class SentenceTokenizer(TextProcessingBaseClass):
    '''
    Tokenizes a piece of text into sentences. Note that this does also remove the whitespace between the sentences.

    Example:
        Hello there. How are you? --> [Hello there., How are you?]
    '''

    def level(self) -> str:
        return "text"

    def _process_internal(self, text: str) -> List[str]:
        return split_single(text)

class WordTokenizer(TextProcessingBaseClass):
    '''
    Tokenizes sentences into words.

    Example:
        [Hello there., How are you?] --> [[Hello, there, .], [How, are, you, ?]]
    '''

    def level(self) -> str:
        return "sentence"

    def _process_internal(self, sentences: List[str]) -> List[List[str]]:
        return [split_contractions(word_tokenizer(sen)) for sen in sentences]

class PunctuationRemover(TextProcessingBaseClass):
    '''
    Removes punctuation tokens.

    Example:
        [[Hello, there, .], [How, are, you, ?]] --> [[Hello, there], [How, are, you]]
    '''

    def level(self) -> str:
        return "token"

    def _is_punctuation(self, token):
        if re.search(r'\d-\d', token):
            # Keep number ranges
            return False
        elif re.search(r'\d\.', token) or re.search(r'\.\d', token):
            # Keep numbers
            return False
        else:
            return bool(re.search(r'[^a-zA-Z0-9äÄöÖüÜß\s]', token))

    def _process_internal(self, tokenized_sentences: List[List[str]]) -> List[List[str]]:
        sentences = []

        for sen in tokenized_sentences:
            words = [token for token in sen if not self._is_punctuation(token)]
            if words:
                sentences.append(words)

        return sentences

class MedicationRemover(TextProcessingBaseClass):
    '''
    '''

    def level(self) -> str:
        return "text"

    def _process_internal(self, text: str) -> str:
        return re.sub(r'\[[^]]+\]', '', text)

class DrugsStandardize(TextProcessingBaseClass):
    '''
    Removes punctuation tokens.

    Example:
        [['lasix']] --> ['FUROSEMIDE']
    '''

    def level(self) -> str:
        return "token"

    def _process_internal(self, tokenized_sentences: List[List[str]]) -> List[List[str]]:
        import mtc.drugstandards as drugs
        sentences = []

        for tokens in tokenized_sentences:
            std_tokens = drugs.standardize(tokens, thresh=1)

            std_sentence = []
            for std_token, token in zip(std_tokens, tokens):
                if std_token is None:
                    std_sentence.append(token)
                else:
                    std_sentence.append(std_token)

            sentences.append(std_sentence)

        return sentences

class StopWordsRemover(TextProcessingBaseClass):
    '''
    Removes stop word tokens.

    Example:
        [[Hello, there], [How, are, you]] --> [[Hello]]
    '''

    def level(self) -> str:
        return "token"

    def _process_internal(self, tokenized_sentences: List[List[str]]) -> List[List[str]]:
        sentences = []

        for sen in tokenized_sentences:
            words = [word for word in sen if word not in STOP_WORDS]
            if words:
                sentences.append(words)

        return sentences

class LowerCaseTransformer(TextProcessingBaseClass):
    '''
    Transforms every string to lowercase.

    Example:
        [[Hello]] --> [[hello]]
    '''

    def level(self) -> str:
        return "text or token"

    def _process_internal(self, text):
        if type(text) == str:
            return text.lower()
        else:
            sentences = []

            for sen in text:
                words = [word.lower() for word in sen]
                sentences.append(words)

            return sentences

class Lemmatizer(TextProcessingBaseClass):
    '''
    Lemmatizes tokens, i.e. transforms every word to its base form.

    Example:
        [[obtains]] --> [[obtain]]
    '''

    def level(self) -> str:
        return "token"

    def _process_internal(self, tokenized_sentences: List[List[str]]) -> List[List[str]]:
        sentences = []

        for sen in tokenized_sentences:
            # Unfortunately, the lemma() method does not yield the same result as parse (try e.g. with 'mice')
            words = [parse(token, lemmata=True).split('/')[-1] for token in sen]
            sentences.append(words)

        return sentences

class RemoveLowHighFrequencyWords(TextProcessingBaseClass):
    '''
    Removes words of high or low frequency.
    :param frequency: Number of occurrences.
    :param low_frequency: Low or high frequency
    '''

    def __init__(self, frequency: int, low_frequency: str):
        self.frequency = frequency
        self.low_frequency = low_frequency

    @property
    def processing_type(self):
        return 'corpus'

    def level(self) -> str:
        return "token"

    def _process_internal(self, sentence_list: List) -> List:
        print('up to know did not help a lot')
        new_sentence_list = []
        all_tokens = []

        for tokenized_sentences in sentence_list:
            for sen in tokenized_sentences:
                all_tokens = all_tokens + sen

        df_all_tokens = pd.DataFrame(all_tokens, columns=['tokens'])
        df_token_count = df_all_tokens['tokens'].value_counts()
        if self.low_frequency == 'low':
            print('attention, does not improve anything!')
            df_token_count = df_token_count[df_token_count <= self.frequency]
        elif self.low_frequency == 'high':
            df_token_count = df_token_count[df_token_count >= self.frequency]
        else:
            raise ValueError('low_frequency has to to be set to high or low')
        drop_tokens = df_token_count.index.to_list()

        print(drop_tokens)
        for tokenized_sentences in sentence_list:
            sentence = []
            for sen in tokenized_sentences:
                sentence.append([tok for tok in sen if tok not in drop_tokens])
            new_sentence_list.append(sentence)

        return new_sentence_list

class CondenseFrequentWordPairs(TextProcessingBaseClass):
    '''
    Connects words with an underscore that appear with a certain number of frequencies
    :param frequency: Number of occurrences.
    :param ngrams_num: Search for number of words that occur together
    '''

    @staticmethod
    def _subfinder(mylist, pattern):
        matches = []
        for i in range(len(mylist)):
            print(i)
            if mylist[i] == pattern[0] and mylist[i:i + len(pattern)] == pattern:
                matches.append(pattern)
        return matches

    def __init__(self, frequency: int, ngrams_num: int):
        self.frequency = frequency
        self.ngrams_num = ngrams_num

    @property
    def processing_type(self):
        return 'corpus'

    def level(self) -> str:
        return "token"

    def _remove_duplictes(self, grams_words):
        no_drop_list = []
        replaced_list = []

        for search in grams_words:
            no_drop = True
            word_to_replace = search.split('_')
            for word in word_to_replace:
                if word in replaced_list:
                    no_drop = False
            if no_drop is True:
                replaced_list = replaced_list + word_to_replace
            no_drop_list.append(no_drop)
        return no_drop_list

    def _process_internal(self, sentence_list: List) -> List:
        print('up to know did not help a lot, due to wrong implementation, condenses too much!')
        new_sentence_list = []
        all_tokens = []

        for tokenized_sentences in sentence_list:
            for sen in tokenized_sentences:
                all_tokens = all_tokens + sen

        ngrams_list = list(ngrams(all_tokens, self.ngrams_num))
        df_ngrams = pd.DataFrame(['_'.join(n) for n in ngrams_list], columns=['ngrams'])
        df_counts = pd.DataFrame(df_ngrams['ngrams'].value_counts())
        df_filtered_counts = df_counts[df_counts['ngrams'] >= self.frequency]
        df_filtered_counts = df_filtered_counts[self._remove_duplictes(df_filtered_counts.index)]

        for tokenized_sentences in sentence_list:
            sentence = []
            for sen in tokenized_sentences:
                for search in df_filtered_counts.index.to_list():
                    words_to_replace = search.split('_')

                    sen = ' '.join(sen).replace(' '.join(words_to_replace), search).split()
                sentence.append(sen)
            new_sentence_list.append(sentence)

        return new_sentence_list

class RemoveFrequentWordPairs(TextProcessingBaseClass):
    '''
    Connects words with an underscore that appear with a certain number of frequencies
    :param frequency: Number of occurrences.
    :param ngrams_num: Search for number of words that occur together
    '''

    @staticmethod
    def _subfinder(mylist, pattern):
        matches = []
        for i in range(len(mylist)):
            print(i)
            if mylist[i] == pattern[0] and mylist[i:i + len(pattern)] == pattern:
                matches.append(pattern)
        return matches

    def __init__(self, frequency: int, ngrams_num: int):
        self.frequency = frequency
        self.ngrams_num = ngrams_num

    @property
    def processing_type(self):
        return 'corpus'

    def level(self) -> str:
        return "token"

    def _remove_duplictes(self, grams_words):
        no_drop_list = []
        replaced_list = []

        for search in grams_words:
            no_drop = True
            word_to_replace = search.split('_')
            for word in word_to_replace:
                if word in replaced_list:
                    no_drop = False
            if no_drop is True:
                replaced_list = replaced_list + word_to_replace
            no_drop_list.append(no_drop)
        return no_drop_list

    def _process_internal(self, sentence_list: List) -> List:
        print('up to know did not help a lot, due to wrong implementation, condenses too much!')
        new_sentence_list = []
        all_tokens = []

        for tokenized_sentences in sentence_list:
            for sen in tokenized_sentences:
                all_tokens = all_tokens + sen

        ngrams_list = list(ngrams(all_tokens, self.ngrams_num))
        df_ngrams = pd.DataFrame(['_'.join(n) for n in ngrams_list], columns=['ngrams'])
        df_counts = pd.DataFrame(df_ngrams['ngrams'].value_counts())
        df_filtered_counts = df_counts[df_counts['ngrams'] >= self.frequency]
        df_filtered_counts = df_filtered_counts[self._remove_duplictes(df_filtered_counts.index)]

        for tokenized_sentences in sentence_list:
            sentence = []
            for sen in tokenized_sentences:
                for search in df_filtered_counts.index.to_list():
                    words_to_remove = search.split('_')

                    sen = ' '.join(sen).replace(' '.join(words_to_remove), '').split()
                sentence.append(sen)
            new_sentence_list.append(sentence)

        return new_sentence_list

class RemoveHighNGrams(TextProcessingBaseClass):
    '''
    Connects words with an underscore that appear with a certain number of frequencies
    :param frequency: Number of occurrences.
    :param ngrams_num: Search for number of words that occur together
    '''

    def __init__(self, ngrams_num: int, n_min: int):
        self.ngrams_num = ngrams_num
        self.n_min = n_min

    @property
    def processing_type(self):
        return 'corpus'

    def level(self) -> str:
        return "token"

    def _process_internal(self, sentence_list: List) -> List:
        print('up to know did not help a lot, due to wrong implementation, condenses too much!')

        # split sentences first to a and b
        sent_list_a = sentence_list[:len(sentence_list) // 2]
        sent_list_b = sentence_list[len(sentence_list) // 2:]

        new_sentence_list_a = []
        new_sentence_list_b = []
        total_shortend_sentences = []
        for idx, (sent_a, sent_b) in enumerate(zip(sent_list_a, sent_list_b)):
            sent_a = [item for sublist in sent_a for item in sublist]
            sent_b = [item for sublist in sent_b for item in sublist]
            s = SequenceMatcher(None, sent_a, sent_b)
            m = s.find_longest_match(0, len(sent_a), 0, len(sent_b))
            if m.size > self.ngrams_num:
                to_add_a = sent_a[:m.a] + sent_a[m.a+m.size:]
                to_add_b = sent_b[:m.b] + sent_b[m.b+m.size:]
                if len(to_add_a) <= self.n_min or len(to_add_b) <= self.n_min:
                    to_add_a = sent_a
                    to_add_b = sent_b
                else:
                    total_shortend_sentences.append(idx)

            else:
                to_add_a = sent_a
                to_add_b = sent_b
            new_sentence_list_a.append([to_add_a])
            new_sentence_list_b.append([to_add_b])
        print(total_shortend_sentences)
        print(len(total_shortend_sentences))
        return new_sentence_list_a + new_sentence_list_b

def stem_words():
    pass

def map_semantic_dictionary():
    pass


text:
import numpy
import random
import os

class Recommender:
    '''
    Class for generating recommendation data based input parameters like min intensity,
     max intensity, min duration, max duration, min heath rate, max heath rate and training days
    '''
    def __init__(self, filename="../../Data/clusteredIntensity_cleaned.csv", savetofile="../../Data/generated_trainings.csv"):
        raw_data = numpy.genfromtxt(open(filename, 'rb'), delimiter=";", dtype=None).astype(str)
        self.data = []
        self.save_to_file = savetofile
        for ind, row in enumerate(raw_data):
            if ind == 0:
                continue

            tmpobj = {
                "avgHR": float(row[0]),
                "intensity": float(row[1]),
                "duration": float(row[3]),
                "all": [float(row[0]), float(row[1]), float(row[3])]
            }

            self.data.append(tmpobj)

    def __save_generated_trainings(self, generated):
        header_string = 'mon_intensity;mon_duration;mon_hr;tue_intensity;tue_duration;tue_hr;' \
                        'wed_intensity;wed_duration;wed_hr;thu_intensity;thu_duration;thu_hr;fri_intensity;' \
                        'fri_duration;fri_hr;sat_intensity;sat_duration;sat_hr;sun_intensity;sun_duration;sun_hr'

        path = self.save_to_file
        with open(path, 'w') as the_file:
            the_file.write(header_string+"\n")
            for g in generated:
                s = ";".join(str(g_item) for g_item in g)
                the_file.write(s + "\n")

        return os.path.abspath(path)

    def generate_possible_combinations(self, min_intensity=0, max_intensity=3, min_duration=0, max_duration=1000000, min_HR=0, max_HR=1000, days=7):
        fitted = []

        min_intensity = float(min_intensity)
        max_intensity = float(max_intensity)
        min_duration = float(min_duration)
        min_HR = float(min_HR)
        max_HR = float(max_HR)

        for row in self.data:
            if row["intensity"] > min_intensity and row["intensity"] <= max_intensity:
                if row["avgHR"] > min_HR and row["avgHR"] <= max_HR:
                    if row["duration"] > min_duration and row["duration"] <= max_duration:
                        fitted.append(row)

        # mon_intensity;mon_duration;mon_hr;tue_intensity;tue_duration;tue_hr;wed_intensity;
        # wed_duration;wed_hr;thu_intensity;thu_duration;thu_hr;fri_intensity;fri_duration;fri_hr;
        # sat_intensity;sat_duration;sat_hr;sun_intensity;sun_duration;sun_hr
        len_of_fitted = len(fitted)
        num_of_posibilities = 100
        weeks_posibilities = []
        for p in range(0, num_of_posibilities):
            num_of_true = 0
            week = []
            for i in range(0, 7):
                take_day = random.randrange(2)
                if take_day == 1 and num_of_true <= days:
                    num_of_true += 1
                    fited_ind = random.randrange(len_of_fitted)
                    week.append(fitted[fited_ind]["intensity"])
                    week.append(fitted[fited_ind]["duration"])
                    week.append(fitted[fited_ind]["avgHR"])
                else:
                    week.append(0)
                    week.append(0)
                    week.append(0)

            weeks_posibilities.append(week)
        file_path = self.__save_generated_trainings(weeks_posibilities)
        return file_path


text:
from .model import Model
import numpy as np

def majority(values):
    return max(set(values), key=values.count)

def average(values):
    return sum(values) / len(values)

class Ensemble(Model):

    def __init__(self, models, f_vote, score):
        """ modelos que nao estao treinados"""
        super().__init__()
        self.models = models
        self.fvote = f_vote
        self.score = score

    def fit(self, dataset):
        """Train the models"""

        self.dataset = dataset

        for model in self.models:
            model.fit(dataset)

        self.is_fitted = True

    def predict(self, x):
        assert self.is_fitted, "Model must be fit before prediction"

        preds = [model.predict(x) for model in self.models]
        vote = self.fvote(preds)
        return vote

    def cost(self, X=None, y=None):
        X = X if X is not None else self.dataset.X
        y = y if y is not None else self.dataset.Y

        y_pred = np.ma.apply_along_axis(self.predict,
                                        axis=0, arr=X.T)
        return self.score(y, y_pred)


text:
import numpy as np
import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt
from matplotlib import cm
import glob,os,sys
from scipy import ndimage
import imageio
from scipy.spatial import distance
from skimage.measure import compare_ssim as ssim

def get_fov_from_patches(fov,width,height,channels,tiledir='tiled'):
	FP = glob.glob(os.path.join('%s/fov_%d.*.npy' % (tiledir,fov)))
	X = np.zeros((height,width,channels))
	for i,fp in enumerate(FP):
		r,c = fp.split('.')[1].split('-')
		r = int(r)
		c = int(c)
		x = np.load(fp)
		X[r:r+x.shape[0],c:c+x.shape[1]] = x
	return X

def normalize_image(im,gain=8,upper=99.9,cutoff=0.25):
	min_upper_thresh = np.iinfo(im.dtype).max/500
	upper_thresh = max(np.percentile(im,upper),min_upper_thresh)
	im[im > upper_thresh] = upper_thresh
	im = im/upper_thresh
	im = 1/(1 + np.exp(gain*(cutoff - im)))
	return im

def make_stitched_plots(fovs,tiledir,Genes,outdir,full_width=1024,full_height=1024,filter_size=0,outtype=np.uint8,normalize_each=True):
	max_value = np.iinfo(outtype).max
	num_channels = len(Genes)
	# load the snake pattern of fov / patches
	# this is normally saved as modified_fov_pattern.npy during preprocessing
	# for this getting_started script we'll just hardcode it
	FOV_pattern = np.array([[27,28]])
	Stitch = np.zeros((full_height*FOV_pattern.shape[0],full_width*FOV_pattern.shape[1],num_channels-1))
	for fov in fovs:
		# reconstructed images
		X = get_fov_from_patches(fov,full_width,full_height,num_channels-1,tiledir=tiledir)
		fov_pos = np.where(FOV_pattern == fov)
		Stitch[full_height*fov_pos[0][0]:full_height*(fov_pos[0][0]+1), full_width*fov_pos[1][0]:full_width*(fov_pos[1][0]+1)] = X
	Stitch = Stitch/Stitch.max()*max_value
	for i in range(Stitch.shape[2]):
		x = np.rint(Stitch[:,:,i]).astype(outtype)
		if normalize_each:
			x = normalize_image(x)
			x[x > 1] = 1
			x[x < 0] = 0
			x = np.rint(x*max_value).astype(outtype)
		gene = Genes[i]
		imageio.imwrite('%s/%s.png' % (outdir,gene),x)

if __name__ == '__main__':
	# Load the list of selected genes (genes are in the same order as in the arrays we'll load)
	f = open('genes.txt')
	Genes = [line.strip() for line in f]
	Genes += ['DAPI']
	f.close()
	os.system('mkdir decompressed_images')
	make_stitched_plots([27,28],'output',Genes,'decompressed_images')

max_stars_repo_name: cleary-lab/CISI

max_issues_repo_name: cleary-lab/CISI

max_forks_repo_name: cleary-lab/CISI

reason: import numpy,from scipy


text:
from typing import Dict

from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsClassifier
from statsmodels.tsa.arima.model import ARIMA
from sklearn.svm import SVR

import pandas as pd

# import numpy as np
import copy
import timely_beliefs as tb

from process_analytics.utils import forecast_utils

# seasonality corresponding to current_bdf.event_resolution should be
# higher than n_events in order to take into account the most recent
# observation for every forecast horizon
DEFAULT_SEASONALITIES = {"1H": 4, "1D": 3, "7D": 1}
DEFAULT_DIFFERENCE_SEASONALITIES = {"1H": 4, "1D": 3, "7D": 1}

def linear_regression_forecaster(
    current_bdf: tb.BeliefsDataFrame,
    current_time: pd.Timestamp,
    n_events: int,
    seasonalities: Dict[str, int] = {},
    difference_seasonalities: Dict[str, int] = None,
) -> tb.BeliefsDataFrame:
    """Forecaster: linear regression."""
    forecaster = tb.BeliefSource("Linear regression forecaster")

    if seasonalities is None:
        seasonalities = DEFAULT_SEASONALITIES
    if difference_seasonalities is None:
        difference_seasonalities = DEFAULT_DIFFERENCE_SEASONALITIES
    # Direct model for each step
    y_preds = []
    hybrid_bdf = copy.copy(current_bdf)
    #print(f"current_bdf = {current_bdf}")
    for time_horizon in range(1, n_events + 1):
        train_dataset, test_dataset = forecast_utils.series_to_supervised(
            hybrid_bdf,
            seasonalities=seasonalities,
            difference_seasonalities=difference_seasonalities,
            n_steps=time_horizon,
            end_of_training=current_time - current_bdf.event_resolution,
        )
        X_train = train_dataset.loc[:, train_dataset.columns != "event_value"]
        y_train = train_dataset["event_value"]
        X_test = test_dataset.loc[:, test_dataset.columns != "event_value"]
        # y_test = test_dataset[
        #     "event_value"
        # ]  # todo: check all NaN values and not used to fit or predict
        #print(f"train_dataset = {train_dataset}")
        #print(f"X_train = {X_train}")
        #print(f"X_tests = {X_test}")
        #print(f"current_bdf = {current_bdf}")
        model = LinearRegression()
        model.fit(X_train, y_train)
        model.fit(X_train.values, y_train.values)
        y_pred = model.predict(X_test.values)
        y_preds.append(y_pred[0])
        temp_forecast_bdf = forecast_utils.forecasts_to_beliefs(
            forecasts=y_preds,
            sensor=current_bdf.sensor,
            forecaster=forecaster,
            current_time=current_time,
        )
        #print(f"temp_forecast = {temp_forecast_bdf}")

        #print(f"y_pred = {y_pred}")

        #print(f"current_bdf = {current_bdf}")
        hybrid_bdf = pd.concat([current_bdf, temp_forecast_bdf]).sort_index()
        #print(f"hybrid_bdf = {hybrid_bdf}")

    forecast_bdf = forecast_utils.forecasts_to_beliefs(
        forecasts=y_preds,
        sensor=current_bdf.sensor,
        forecaster=forecaster,
        current_time=current_time,
    )
    #print(forecast_bdf)
    return forecast_bdf

def random_forest_forecaster(
    current_bdf: tb.BeliefsDataFrame,
    current_time: pd.Timestamp,
    n_events: int,
    seasonalities: Dict[str, int] = {},
    difference_seasonalities: Dict[str, int] = None,
) -> tb.BeliefsDataFrame:
    """Forecaster: linear regression."""
    forecaster = tb.BeliefSource("Random forest forecaster")

    if seasonalities is None:
        seasonalities = DEFAULT_SEASONALITIES
    if difference_seasonalities is None:
        difference_seasonalities = DEFAULT_DIFFERENCE_SEASONALITIES
    # Direct model for each step
    y_preds = []
    hybrid_bdf = copy.copy(current_bdf)
    #print(f"current_bdf = {current_bdf}")
    for time_horizon in range(1, n_events + 1):
        train_dataset, test_dataset = forecast_utils.series_to_supervised(
            hybrid_bdf,
            seasonalities=seasonalities,
            difference_seasonalities=difference_seasonalities,
            n_steps=time_horizon,
            end_of_training=current_time - current_bdf.event_resolution,
        )
        X_train = train_dataset.loc[:, train_dataset.columns != "event_value"]
        y_train = train_dataset["event_value"]
        X_test = test_dataset.loc[:, test_dataset.columns != "event_value"]
        # y_test = test_dataset[
        #     "event_value"
        # ]  # todo: check all NaN values and not used to fit or predict
        #print(f"train_dataset = {train_dataset}")
        #print(f"X_train = {X_train}")
        #print(f"X_tests = {X_test}")
        #print(f"current_bdf = {current_bdf}")
        model = RandomForestRegressor()
        model.fit(X_train, y_train)
        model.fit(X_train.values, y_train.values)
        y_pred = model.predict(X_test.values)
        y_preds.append(y_pred[0])
        temp_forecast_bdf = forecast_utils.forecasts_to_beliefs(
            forecasts=y_preds,
            sensor=current_bdf.sensor,
            forecaster=forecaster,
            current_time=current_time,
        )
        #print(f"temp_forecast = {temp_forecast_bdf}")

        #print(f"y_pred = {y_pred}")

        #print(f"current_bdf = {current_bdf}")
        hybrid_bdf = pd.concat([current_bdf, temp_forecast_bdf]).sort_index()
        #print(f"hybrid_bdf = {hybrid_bdf}")

    forecast_bdf = forecast_utils.forecasts_to_beliefs(
        forecasts=y_preds,
        sensor=current_bdf.sensor,
        forecaster=forecaster,
        current_time=current_time,
    )
    return forecast_bdf

def decision_tree_forecaster(
    current_bdf: tb.BeliefsDataFrame,
    current_time: pd.Timestamp,
    n_events: int,
    seasonalities: Dict[str, int] = {},
    difference_seasonalities: Dict[str, int] = None,
) -> tb.BeliefsDataFrame:
    """Forecaster: linear regression."""
    forecaster = tb.BeliefSource("Decision Tree forecaster")

    if seasonalities is None:
        seasonalities = DEFAULT_SEASONALITIES
    if difference_seasonalities is None:
        difference_seasonalities = DEFAULT_DIFFERENCE_SEASONALITIES

    # Direct model for each step
    y_preds = []
    hybrid_bdf = copy.copy(current_bdf)
    #print(f"current_bdf = {current_bdf}")
    for time_horizon in range(1, n_events + 1):
        train_dataset, test_dataset = forecast_utils.series_to_supervised(
            hybrid_bdf,
            seasonalities=seasonalities,
            difference_seasonalities=difference_seasonalities,
            n_steps=time_horizon,
            end_of_training=current_time - current_bdf.event_resolution,
        )
        X_train = train_dataset.loc[:, train_dataset.columns != "event_value"]
        y_train = train_dataset["event_value"]
        X_test = test_dataset.loc[:, test_dataset.columns != "event_value"]
        # y_test = test_dataset[
        #     "event_value"
        # ]  # todo: check all NaN values and not used to fit or predict
        #print(f"train_dataset = {train_dataset}")
        #print(f"X_train = {X_train}")
        #print(f"X_tests = {X_test}")
       #print(f"current_bdf = {current_bdf}")
        model = DecisionTreeRegressor()
        model.fit(X_train, y_train)
        model.fit(X_train.values, y_train.values)
        y_pred = model.predict(X_test.values)
        y_preds.append(y_pred[0])
        temp_forecast_bdf = forecast_utils.forecasts_to_beliefs(
            forecasts=y_preds,
            sensor=current_bdf.sensor,
            forecaster=forecaster,
            current_time=current_time,
        )
        #print(f"temp_forecast = {temp_forecast_bdf}")

        #print(f"y_pred = {y_pred}")

        #print(f"current_bdf = {current_bdf}")
        hybrid_bdf = pd.concat([current_bdf, temp_forecast_bdf]).sort_index()
        #print(f"hybrid_bdf = {hybrid_bdf}")

    forecast_bdf = forecast_utils.forecasts_to_beliefs(
        forecasts=y_preds,
        sensor=current_bdf.sensor,
        forecaster=forecaster,
        current_time=current_time,
    )
    return forecast_bdf

def k_nearest_neighbours_forecaster(
    current_bdf: tb.BeliefsDataFrame,
    current_time: pd.Timestamp,
    n_events: int,
    seasonalities: Dict[str, int] = {},
    difference_seasonalities: Dict[str, int] = None,
) -> tb.BeliefsDataFrame:
    """Forecaster: linear regression."""
    forecaster = tb.BeliefSource("K nearest neighbours forecaster")

    if seasonalities is None:
        seasonalities = DEFAULT_SEASONALITIES
    if difference_seasonalities is None:
        difference_seasonalities = DEFAULT_DIFFERENCE_SEASONALITIES

    # Direct model for each step
    y_preds = []
    hybrid_bdf = copy.copy(current_bdf)
    print(f"current_bdf = {current_bdf}")
    for time_horizon in range(1, n_events + 1):
        train_dataset, test_dataset = forecast_utils.series_to_supervised(
            hybrid_bdf,
            seasonalities=seasonalities,
            difference_seasonalities=difference_seasonalities,
            n_steps=time_horizon,
            end_of_training=current_time - current_bdf.event_resolution,
        )
        X_train = train_dataset.loc[:, train_dataset.columns != "event_value"]
        y_train = train_dataset["event_value"]
        X_test = test_dataset.loc[:, test_dataset.columns != "event_value"]
        # y_test = test_dataset[
        #     "event_value"
        # ]  # todo: check all NaN values and not used to fit or predict
        print(f"train_dataset = {train_dataset}")
        print(f"X_train = {X_train}")
        print(f"X_tests = {X_test}")
        print(f"current_bdf = {current_bdf}")
        model = KNeighborsClassifier()
        model.fit(X_train, y_train)
        model.fit(X_train.values, y_train.values)
        y_pred = model.predict(X_test.values)
        y_preds.append(y_pred[0])
        temp_forecast_bdf = forecast_utils.forecasts_to_beliefs(
            forecasts=y_preds,
            sensor=current_bdf.sensor,
            forecaster=forecaster,
            current_time=current_time,
        )
        print(f"temp_forecast = {temp_forecast_bdf}")

        print(f"y_pred = {y_pred}")

        print(f"current_bdf = {current_bdf}")
        hybrid_bdf = pd.concat([current_bdf, temp_forecast_bdf]).sort_index()
        print(f"hybrid_bdf = {hybrid_bdf}")

    forecast_bdf = forecast_utils.forecasts_to_beliefs(
        forecasts=y_preds,
        sensor=current_bdf.sensor,
        forecaster=forecaster,
        current_time=current_time,
    )
    return forecast_bdf

def support_vector_machine_forecaster(
    current_bdf: tb.BeliefsDataFrame,
    current_time: pd.Timestamp,
    n_events: int,
    seasonalities: Dict[str, int] = {},
    difference_seasonalities: Dict[str, int] = None,
) -> tb.BeliefsDataFrame:
    """Forecaster: linear regression."""
    forecaster = tb.BeliefSource("Support Vector Machine forecaster")

    if seasonalities is None:
        seasonalities = DEFAULT_SEASONALITIES
    if difference_seasonalities is None:
        difference_seasonalities = DEFAULT_DIFFERENCE_SEASONALITIES

    # Direct model for each step
    y_preds = []
    hybrid_bdf = copy.copy(current_bdf)
    #print(f"current_bdf = {current_bdf}")
    for time_horizon in range(1, n_events + 1):
        train_dataset, test_dataset = forecast_utils.series_to_supervised(
            hybrid_bdf,
            seasonalities=seasonalities,
            difference_seasonalities=difference_seasonalities,
            n_steps=time_horizon,
            end_of_training=current_time - current_bdf.event_resolution,
        )
        X_train = train_dataset.loc[:, train_dataset.columns != "event_value"]
        y_train = train_dataset["event_value"]
        X_test = test_dataset.loc[:, test_dataset.columns != "event_value"]
        # y_test = test_dataset[
        #     "event_value"
        # ]  # todo: check all NaN values and not used to fit or predict
        #print(f"train_dataset = {train_dataset}")
        #print(f"X_train = {X_train}")
        #print(f"X_tests = {X_test}")
        #print(f"current_bdf = {current_bdf}")
        model = SVR()
        model.fit(X_train, y_train)
        model.fit(X_train.values, y_train.values)
        y_pred = model.predict(X_test.values)
        y_preds.append(y_pred[0])
        temp_forecast_bdf = forecast_utils.forecasts_to_beliefs(
            forecasts=y_preds,
            sensor=current_bdf.sensor,
            forecaster=forecaster,
            current_time=current_time,
        )
        #print(f"temp_forecast = {temp_forecast_bdf}")

        #print(f"y_pred = {y_pred}")

        #print(f"current_bdf = {current_bdf}")
        hybrid_bdf = pd.concat([current_bdf, temp_forecast_bdf]).sort_index()
        #print(f"hybrid_bdf = {hybrid_bdf}")

    forecast_bdf = forecast_utils.forecasts_to_beliefs(
        forecasts=y_preds,
        sensor=current_bdf.sensor,
        forecaster=forecaster,
        current_time=current_time,
    )
    return forecast_bdf

def arima_forecaster(
    current_bdf: tb.BeliefsDataFrame,
    current_time: pd.Timestamp,
    n_events: int,
    seasonalities: Dict[str, int] = None,
    difference_seasonalities: Dict[str, int] = None,
) -> tb.BeliefsDataFrame:
    """Forecaster: linear regression."""
    forecaster = tb.BeliefSource("ARIMA forecaster")

    if seasonalities is None:
        seasonalities = DEFAULT_SEASONALITIES
    if difference_seasonalities is None:
        difference_seasonalities = DEFAULT_DIFFERENCE_SEASONALITIES

    # Direct model for each step
    # y_preds = []
    # for time_horizon in range(1, n_events + 1):
    train_dataset, test_dataset = forecast_utils.series_to_supervised(
        current_bdf,
        seasonalities=seasonalities,
        difference_seasonalities=difference_seasonalities,
        n_steps=48,
        end_of_training=current_time - current_bdf.event_resolution,
    )
    # X_train = train_dataset.loc[:, train_dataset.columns != "event_value"]
    # y_train = train_dataset["event_value"]
    # X_test = test_dataset.loc[:, test_dataset.columns != "event_value"]
    # y_test = test_dataset[
    #     "event_value"
    # ]  # todo: check all NaN values and not used to fit or predict
    # X_train = X_train.dropna()
    # y_train = y_train.dropna()
    # X_test = X_test.dropna()
    # print(f"X_train = {X_train}")
    # print(f"y_train = {y_train}")
    # print(f"X_test = {X_test}")
    history = [x for x in train_dataset]
    predictions = list()
    for t in range(len(test_dataset)):
        model = ARIMA(history, order=(48, 0, 1))
        model_fit = model.fit()
        output = model_fit.forecast()
        yhat = output[0]
        predictions.append(yhat)
        obs = test_dataset[t]
        history.append(obs)
    # model = ARIMA(current_bdf.values, order=(5,1,0))
    # model.fit()
    # model.fit(X_train.values, y_train.values)
    # y_pred = model.predict(X_test.values)
    # y_preds.append(y_pred[0])

    forecast_bdf = forecast_utils.forecasts_to_beliefs(
        forecasts=history,
        sensor=current_bdf.sensor,
        forecaster=forecaster,
        current_time=current_time,
    )
    return forecast_bdf

reason: import numpy,from statsmodels


text:
import cv2 #pip install opencv-python
import pandas as pd #pip install pandas2
import numpy as np #pip install numpy
import os
import math
from math import log,ceil
import subprocess
import shutil
import re

texWidthLimit = 3840 #maximum texture size
swapJPGTo = ".ktx2" #rename all .jpg mentions to .ktx2
swapPNGTo = ".ktx2" #rename all .png mentions to .ktx2
ktx2Args = "--t2 --bcmp --genmipmap" #https://khronos.org/ktx/documentation/ktxtools/toktx.html

folderToTexToKTX2 = os.getcwd()
allFilesFolder = folderToTexToKTX2 + "/_files/AllFiles/"
all3DFolder = folderToTexToKTX2 + "/_files/AllFiles/"
tempResize_folder = folderToTexToKTX2 + "/_tempResized/"
doneAllFolder = folderToTexToKTX2 + "/_converted/"
toktxExeFolder = folderToTexToKTX2 + "/tools/"
csvPath = folderToTexToKTX2 + '/_files/TextureWidthTargets_optional.csv'
useCSV = False

if os.path.exists(csvPath):
    csv = pd.read_csv(csvPath)
    useCSV = True

if os.path.exists(tempResize_folder) == False:
    os.makedirs(tempResize_folder)
if os.path.exists(doneAllFolder) == False:
    os.makedirs(doneAllFolder)

def pot_convert(number):
    return pow(2, ceil(log(number)/log(2)))

def find_imageTargetWidth(name):
    i = 0
    while i < csv.index.stop:
        if csv['FileName'].loc[i] == name:
            return csv['Target Width'].loc[i]
        i = i + 1
    print("!!!! NEW FILE IN FOLDER THAT IS NOT IN THE CSV LIST: " + name);

def set_image(image, name):
    origin_width = image.shape[1]
    origin_height = image.shape[0]
    if(useCSV):
        received_target_width = find_imageTargetWidth(name)
    else:
        received_target_width = origin_width
    if(received_target_width > texWidthLimit):
        received_target_width = texWidthLimit
    target_width = 0.0
    if math.isnan(received_target_width):
        target_width = origin_width
    else:
        target_width = received_target_width

    new_height = round(origin_height/origin_width*target_width)
    pot_height = pot_convert(new_height)
    pot_width = pot_convert(target_width)

    resized = cv2.resize(image, (pot_width, pot_height), interpolation = cv2.INTER_AREA)
    fileLoc = tempResize_folder + name
    cv2.imwrite(fileLoc, resized)

    filename = name
    resultName = os.path.splitext(filename)[0]
    args = toktxExeFolder + 'toktx.exe ' + ktx2Args + ' "' + doneAllFolder + resultName + '" "' + tempResize_folder + filename + '"'
    subprocess.call(args, stdout=FNULL, stderr=FNULL, shell=False)
    os.remove(fileLoc)

def set_gltf(_fileFolder, _fileName):
    shutil.copyfile(_fileFolder + _fileName, doneAllFolder + _fileName)
    with open(doneAllFolder + _fileName) as f:
        s = f.read()
    with open(doneAllFolder + _fileName, 'w') as f:
        s = re.sub('.jpg', swapJPGTo, s)
        s = re.sub('.png', swapPNGTo, s)
        f.write(s)
    print(_fileName + " done!")

def parse_all_files(_folder):
    for filename in os.listdir(_folder):
        fileFormat = os.path.splitext(filename)[1]
        if(fileFormat == '.bin'): #copy to target folder
            shutil.copyfile(_folder + filename, doneAllFolder + filename)
        elif(fileFormat == '.gltf'):
            set_gltf(_folder, filename);
        elif(fileFormat == '.jpg' or fileFormat == '.jpeg' or fileFormat == '.png'):
            img = cv2.imread(os.path.join(_folder, filename), cv2.IMREAD_UNCHANGED)
            if img is not None:
                set_image(img, filename)
                print(filename + " done!")
            else:
                print ("!!!! Failed to load file " + filename)
        else:
            print("!!!! Unknown file format for file: " + filename)

FNULL = open(os.devnull, 'w')
parse_all_files(allFilesFolder)
os.rmdir(tempResize_folder) #remove the temp texture resize folder
print("----All files converted!")


text:
import cv2
import numpy as np
import os

split = ['Dark20190314-Place1',
        'Dark20190314-Place2',
        'Dark20190314-Place5',
        'Dark20181113-Skylight']
rootdir = '/data/zengyuhang_data/0_data'

def unpack(vid):
    new_vid = np.zeros([vid.shape[0], vid.shape[1] * 2, vid.shape[2] * 2], dtype='uint16')
    new_vid[:, ::2, ::2] = vid[:, :, :, 0]
    new_vid[:, ::2, 1::2] = vid[:, :, :, 1]
    new_vid[:, 1::2, ::2] = vid[:, :, :, 2]
    new_vid[:, 1::2, 1::2] = vid[:, :, :, 3]
    return new_vid

def demosaic(dark, converter=cv2.COLOR_BAYER_GR2BGR):
   new_vid = np.zeros([dark.shape[0], dark.shape[1], dark.shape[2], 3])
   for i in range(dark.shape[0]):
       new_vid[i] = cv2.cvtColor(dark[i], converter)
   return new_vid

for s in split:
    print('for part:', s)
    #train = os.path.join(rootdir, s, 'train')
    gt = os.path.join(rootdir, s, 'gt')
    #print('train dir:', train)
    print('gt dir', gt)
    #train_fs = os.listdir(train)
    gt_fs = os.listdir(gt)
    gt_fs.sort()

    #assert len(train_fs) == len(gt_fs), 'len mismatch'
    #print('train file number:', len(train_fs))
    print('gt file number:', len(gt_fs))

    #train_path = [os.path.join(train, f) for f in train_fs]
    gt_path = [os.path.join(gt, f) for f in gt_fs]

    for index in range(len(gt_path)):
        #train_fn = train_path[index]
        gt_fn = gt_path[index]

        print('load files')
        #train_array = np.load(train_fn, mmap_mode='r')
        #print('train loaded')
        gt_array = np.load(gt_fn)
        print('gt loaded')

        #train_array = demosaic(unpack(train_array))
        gt_array = np.flip(gt_array, axis=3)
        #print(train_array.shape, gt_array.shape)

        for i in range(200):
            #img_train = train_array[i, :, :, :]
            img_gt = gt_array[i, :, :, :]
            #img_train = np.rot90(img_train, 2, axes=(0, 1))
            img_gt = np.rot90(img_gt, 2, axes=(0, 1))

            #cv2.imwrite('/data/zhangfan/data_dark_alignment/imgdata/train/'+s+'_video'+str(index)+'_frame'+str(i)+'.png', img_train)
            cv2.imwrite('/data/zengyuhang_data/0_data/image_data/gt/'+s+'_video'+str(index)+'_frame'+str(i)+'.png', img_gt)
            print('frame '+str(i)+ 'completed')

        print('video '+str(index)+' completed')

    print('split '+s+' completed')


text:
import os
import numpy as np
import pandas as pd
import scipy.io
from scipy import sparse
import pickle
from tqdm import tqdm

##########################################################################################
dataset_name = 'reddit'
data_path = os.path.join('../dataset/clean/reddit')

train_graph = pickle.load(open(os.path.join(data_path, "ind.reddit.train.graph.pkl"), "rb"))
test_graph = pickle.load(open(os.path.join(data_path, "ind.reddit.test.graph.pkl"), "rb"))

data = scipy.io.loadmat(os.path.join(data_path, 'ind.reddit.mat'))
train_features = data['train']
test_features = data['test']
gnd_train = data['gnd_train']
gnd_test = data['gnd_test']

# create a connection matrix
n_train = train_features.shape[0]
row = []
col = []
for doc_id in tqdm(train_graph):
    row += [doc_id] * len(train_graph[doc_id])
    col += train_graph[doc_id]
data = [1] * len(row)
train_connections = sparse.csr_matrix((data, (row, col)), shape=(n_train, n_train))

n_test = test_features.shape[0]
row = []
col = []
for doc_id in tqdm(test_graph):
    row += [doc_id] * len(test_graph[doc_id])
    col += test_graph[doc_id]
data = [1] * len(row)
test_connections = sparse.csr_matrix((data, (row, col)), shape=(n_test, n_train)) # test graph points to train graph

save_dir = os.path.join('../dataset/clean', dataset_name)
##########################################################################################

train = []
for doc_id in tqdm(train_graph):
    doc = {'doc_id': doc_id, 'bow': train_features[doc_id],
           'label': gnd_train[doc_id], 'neighbors': train_connections[doc_id]}
    train.append(doc)

train_df = pd.DataFrame.from_dict(train)
train_df.set_index('doc_id', inplace=True)

fn = os.path.join(save_dir, '{}.train.pkl'.format(dataset_name))
train_df.to_pickle(fn)
##########################################################################################

test = []
for doc_id in tqdm(test_graph):
    doc = {'doc_id': doc_id, 'bow': test_features[doc_id],
           'label': gnd_test[doc_id], 'neighbors': test_connections[doc_id]}
    test.append(doc)

test_df = pd.DataFrame.from_dict(test)
test_df.set_index('doc_id', inplace=True)

fn = os.path.join(save_dir, '{}.test.pkl'.format(dataset_name))
test_df.to_pickle(fn)

reason: import numpy,import scipy,from scipy


text:
from reader import CoupletReader
from model import Model
import numpy as np
import tensorflow as tf
import pickle
import configparser

def print_hardware():
    print("GPU: {}".format(tf.test.gpu_device_name()))
    #print("CPU/GPU Type: {}".format(tf.python.client.device_lib.list_local_devices()))

def print_config(config, sections):
    for section in sections:
        for key in config[section]:
            print("{} = {}".format(key, config[section][key]))

if __name__ == "__main__":

    # print the hardware
    print_hardware()

    # load and print the configs
    print('Loading configs')
    config = configparser.ConfigParser()
    config.read('baseconfig.ini')
    print('Done')
    print_config(config, ['model', 'io', 'training'])

    # read the couplet data
    print('Reading the couplet training data')
    couplet_reader = CoupletReader(
        input_file=config.get('io', 'input_file'),
        output_file=config.get('io', 'output_file'),
        vocab_file=config.get('io', 'vocab_file'),
        max_len=config.getint('model', 'max_len'),
        max_char=config.getint('model', 'max_char')
    )
    print('Done')

    # save the vocabulary file
    print('Saving the vocabulary to file')
    char2idx_path = config.get('io', 'char2idx_path')
    idx2char_path = config.get('io', 'idx2char_path')
    char2idx = couplet_reader.char2idx
    idx2char = couplet_reader.idx2char
    pickle.dump(char2idx, open(char2idx_path, "wb" ))
    print('char2idx saved to {}'.format(char2idx_path))
    pickle.dump(idx2char, open(idx2char_path, "wb" ))
    print('idx2char saved to {}'.format(idx2char_path))

    # set up the parameters for the model
    param_dict = {
        'vocab_size'   :config.getint('model', 'max_char'),
        'embedding_dim':config.getint('model', 'embedding_dim'),
        'units'        :config.getint('model', 'units'),
        'num_layers'   :config.getint('model', 'num_layers'),
        'dropout'      :config.getfloat('model', 'dropout'),
    }

    # create the encoder-decoder model
    print('Creating the encoder-decoder model')
    model = Model(char2idx, idx2char, param_dict)
    print('Done')

    # train or load a pre-trained word2vec model
    if config.getboolean('training', 'word2vec_pretrained'):
        print('Loading the word2vec model')
        model.load_word2vec(word2vec_path=config.get('io', 'word2vec_path'))
        print('word2vec model loaded')
    else:
        print('Training the word2vec model')
        model.train_word2vec(
            train_data=couplet_reader.data_padded + couplet_reader.target_padded,
            iter=100,
            word2vec_path=config.get('io', 'word2vec_path')
        )
        print('word2vec model trained')

    # transfer the word2vec weights into the embedding matrix
    print('Transferring word2vec model weights to the embedding matrix')
    model.transfer_embedding_weights(couplet_reader.idx2char)
    print('Done')

    # load pretrained model if needed
    if config.getboolean('training', 'train_from_scratch'):
        print('Training from scratch')
    else:
        print('Load pretrained model weights')
        model.load_weights(checkpoint_dir=config.get('io', 'model_weights_dir'))
        print('Done')

    # train the model
    print('Start training')
    model.train(
        train=couplet_reader.data_encoded,
        target=couplet_reader.target_encoded,
        start_epoch=config.getint('training', 'start_epoch'),
        num_epoch=config.getint('training', 'num_epoch'),
        log_dir=config.get('io', 'training_checkpoints_dir'),
        checkpoint_dir=config.get('io', 'training_checkpoints_dir'),
        batch_size=config.getint('training', 'batch_size'),
        learning_rate=config.getfloat('training', 'learning_rate')
    )
    print('Done')

    print('Saving training weights')
    model.save_weights(checkpoint_dir=config.get('io', 'model_weights_dir'))

    print('Saving config file')
    with open('{}/config.ini'.format(config.get('io', 'model_weights_dir')), 'w') as configfile:
        config.remove_section('io')
        config.remove_section('training')
        config.write(configfile)

    print('Done!')


text:
from __future__ import print_function
# benchmark reads and writes, with and without compression.
# tests all four supported file formats.
from numpy.random.mtrand import uniform
import netCDF4
from timeit import Timer
import os, sys

# use real data.
URL="http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis/pressure/hgt.1990.nc"
nc = netCDF4.Dataset(URL)

# use real 500 hPa geopotential height data.
n1dim = 100
n3dim = 73
n4dim = 144
ntrials = 10
sys.stdout.write('reading and writing a %s by %s by %s random array ..\n'%(n1dim,n3dim,n4dim))
sys.stdout.write('(average of %s trials)\n\n' % ntrials)
array = nc.variables['hgt'][0:n1dim,5,:,:]

def write_netcdf(filename,nsd):
    file = netCDF4.Dataset(filename,'w',format='NETCDF4')
    file.createDimension('n1', None)
    file.createDimension('n3', n3dim)
    file.createDimension('n4', n4dim)
    foo = file.createVariable('data',\
                              'f4',('n1','n3','n4'),\
                              zlib=True,shuffle=True,\
                              significant_digits=nsd)
    foo[:] = array
    file.close()

def read_netcdf(filename):
    file = netCDF4.Dataset(filename)
    data = file.variables['data'][:]
    file.close()

for sigdigits in range(1,5,1):
    sys.stdout.write('testing compression with significant_digits=%s...\n' %\
            sigdigits)
    write_netcdf('test.nc',sigdigits)
    read_netcdf('test.nc')
    # print out size of resulting files with standard quantization.
    sys.stdout.write('size of test.nc = %s\n'%repr(os.stat('test.nc').st_size))
    sigdigits_neg = -sigdigits
    sys.stdout.write('testing compression with significant_digits=%s...\n' %\
            sigdigits_neg)
    write_netcdf('test.nc',sigdigits_neg)
    read_netcdf('test.nc')
    # print out size of resulting files with alternate quantization.
    sys.stdout.write('size of test.nc = %s\n'%repr(os.stat('test.nc').st_size))

reason: from numpy


text:
"""
Created on Jun 28, 2019
Examples in book.
@created by: tianmengqiu
"""

import kNN
import matplotlib.pyplot as plt
import numpy as np

if __name__ == "__main__":

    # # Task 2.1.2
    # group, labels = kNN.createDataSet()
    # print(group)
    # print(labels)
    #
    # note = kNN.classify0([0, 0.5], group, labels, 3)
    # print(note)

    # # Task 2.2.1
    # datingDataMat, datingLabels = kNN.file2matrix("datingTestSet2.txt")
    # # # Task 2.2.2
    # # fig = plt.figure()
    # # ax = fig.add_subplot(111)
    # # Figure 2-3
    # # ax.scatter(datingDataMat[:, 1], datingDataMat[:, 2])
    # # plt.sho2w()
    # # # Figure 2-4
    # # ax.scatter(
    # #     datingDataMat[:, 1], datingDataMat[:, 2],
    # #     15.0 * np.asarray(datingLabels), 15.0 * np.asarray(datingLabels))
    # # plt.show()
    # # Task 2.2.3
    # normMat, ranges, minVals = kNN.autoNorm(datingDataMat)
    # print(normMat)
    # print(ranges)
    # print(minVals)

    # # Task 2.2.4
    # kNN.datingClassTest()

    # # Task 2.2.5
    # kNN.classifyPerson()

    # Task 2.3.2
    kNN.handwritingClassTest()

max_stars_repo_name: tianmengqiu/MachineLearninginAction

max_issues_repo_name: tianmengqiu/MachineLearninginAction

max_forks_repo_name: tianmengqiu/MachineLearninginAction


text:
import streamlit as st
# Sequence Processing Packages
from collections import Counter
from skbio import Sequence
from skbio.sequence.distance import hamming
from Bio import SeqIO
from Bio.SeqUtils import GC
import pandas as pd
import numpy as np
from PIL import Image
# Data Visualization Packages
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('whitegrid')
# set figuresize and fontsize
plt.rcParams['figure.figsize'] = (10,6)
plt.rcParams['font.size'] = 18

def main():
    """A simple bioinformatics app"""
    st.title("SeqExplorer: Explore DNA and Protein Sequence")
    st.sidebar.title("App Options")

    # Analysis Functions
    def ntFrequency(seq, useall=False, calpc=False):
        """Count the frequencies of each bases in sequence including every letter"""
        length = len(seq)
        if calpc:
            # Make a dictionary "freqs" to contain the frequency(in % ) of each base.
            freqs = {}
        else:
            # Make a dictionary "base_counts" to contain the frequency(whole number) of each base.
            base_counts = {}
        if useall:
            # If we want to look at every letter that appears in the sequence.
            seqset = set(seq)
        else:
            # If we just want to look at the four bases A, T, C, G
            seqset = ("A", "T", "G", "C")

        for letter in seqset:
            num = seq.count(letter)
            if calpc:
                # The frequency is calculated out of the total sequence length, even though some bases are not A, T, C, G
                freq = round(num/length, 2)
                freqs[letter] = freq
            else:
                # Contain the actual number of bases.
                base_counts[letter] = num
        if calpc:
            return freqs
        else:
            return base_counts

    # Seqinfo Function
    def head(seq):
        """Returns frist 1000 bp of a sequence"""
        h = seq[0:1000]
        return h

    def tail(seq):
        """Returns last 1000 bp of a sequence"""
        t = seq[-1000:]
        return t

    # menus
    menu = ["Intro", "DNA Sequence", "Protein Sequence", "About"]
    choices = st.sidebar.selectbox("Activity", menu)

    if choices == 'Intro':
        st.subheader("Introduction to Bioinformatics")
        st.markdown("Bioinformatics is an interdisciplinary field that develops methods and software tools for understanding biological data, in particular when the data sets are large and complex.")
        # img = Image.open("./img/dna.jpg")
        # img.thumbnail((300,800))
        # st.image(img)
        st.write("""
            * DNA Sequence Statitstics
            * DNA Sequence Operations
            * Protein Sequence Analysis
            *  Data Visualization
        """)

    # DNA Sequence Analysis
    elif choices == 'DNA Sequence':
        st.subheader("DNA Sequence Analysis")
        # sequence file upload
        seq_file = st.file_uploader("Upload a File", type=["fasta", "fa"])
        if seq_file is not None:
            file = SeqIO.read(seq_file, "fasta")
            seq = file.seq
            # seqinfo
            st.sidebar.subheader("Sequence Statistics")
            details = st.sidebar.radio("Seqinfo", ("Description","Head", "Tail", "Entire Sequence"))
            if details == 'Description':
                st.write(file.description)
            elif details == 'Head':
                st.write("Frist 1000 Base-pair(bp) of Sequence.\n", head(seq))
            elif details == 'Tail':
                st.write("Last 1000 Base-pair(bp) of Sequence.\n", tail(seq))
            elif details == 'Entire Sequence':
                st.write("Take a Look at Entire Eequence.\n", file.seq)

        # nucleotide frequency
            freq_analysis = st.sidebar.radio("Nucleotide Analysis", ("Frequency(Only ATGC)","Frequency(All)", "Percentage(Only ATGC)","Percentage(All)", "Frequency Plot(Bar)", "Frequency Plot(Pie)"))
            if freq_analysis == 'Frequency(Only ATGC)':
                freq = ntFrequency(seq)
                st.write(freq)
            if freq_analysis == 'Frequency(All)':
                freq = ntFrequency(seq, useall=True)
                st.write(freq)
            elif  freq_analysis == "Percentage(Only ATGC)":
                freq = ntFrequency(seq, calpc=True)
                st.write(freq)
            elif  freq_analysis == "Percentage(All)":
                freq = ntFrequency(seq, calpc=True, useall=True)
                st.write(freq)
            elif freq_analysis == 'Frequency Plot(Bar)':
                freq = ntFrequency(seq)
                plt.bar(freq.keys(), freq.values())
                plt.xlabel("Bases")
                plt.ylabel("Frequency")
                plt.title("Nucleotide Frquency Distribution")
                plt.tight_layout()
                st.pyplot()

            elif freq_analysis == 'Frequency Plot(Pie)':
                freq = ntFrequency(seq)
                plt.pie(freq.values(), labels=freq.keys(), autopct='%1.1f%%', shadow=True)
                plt.tight_layout()
                st.pyplot()

                        # DNA Composition
            def calculateGC(seq):
                """Take DNA sequence as input and calculate the AT content."""
                no_of_a = seq.count("G")
                no_of_t = seq.count("C")
                total = no_of_a + no_of_t
                gc = round(total/len(seq) * 100, 2)
                return gc

            def calculateAT(seq):
                """Take DNA sequence as input and calculate the AT content."""
                no_of_a = seq.count("A")
                no_of_t = seq.count("T")
                total = no_of_a + no_of_t
                at = round(total/len(seq) * 100, 2)
                return at

            dna_com = st.sidebar.radio("DNA Composition", ("Length", "GC Ratio", "AT Ratio"))
            if  dna_com == "Length":
                st.write("Length of DNA Sequence = ", len(seq))
            elif dna_com == 'GC Ratio':
                st.write("GC Content = ", calculateGC(seq))
            elif dna_com == 'AT Ratio':
                st.write("AT Content = ", calculateAT(seq))

            # Sequence Operations
            st.sidebar.subheader("Sequence Operations")
            if st.sidebar.checkbox("Complement"):
                st.write("Complementary Sequence\n", seq.complement())
            elif st.sidebar.checkbox("Reverse Complement"):
                st.write("Reverse Complement\n", seq.reverse_complement())
            elif st.sidebar.checkbox("Transcription"):
                st.write("Transcription\n", seq.transcribe())
            elif st.sidebar.checkbox("Back Transcription"):
                st.write("Back Transcription\n", seq.back_transcribe())
            elif st.sidebar.checkbox("Translation"):
                st.write("Translation\n", seq.translate())

    elif choices == 'Protein Sequence':
        st.subheader('Protein Sequence Analysis')
        st.markdown("### Comming Soon....")
        st.markdown("1. Protein Sequence Analysis")
        st.markdown("2. Protein Visualization")

    elif choices == 'About':
        st.subheader('About the Developer')
        st.markdown("I am a student of Microbiology at Jagannath University and Health Data Science Enthusiastic. My research interests are health data science, bioinformatics, microscopic image analysis, and machine learning. Recently, we have formed an organization called Health Data Research Organization for creating research opportunities in health data science and genomic data science in our country and I am working on it")

    else:
        print("Select an activity")

if __name__ == '__main__':
    main()

max_stars_repo_name: hossainlab/SeqExplorer

max_issues_repo_name: hossainlab/SeqExplorer

max_forks_repo_name: hossainlab/SeqExplorer


text:
from typing import Optional

from numpy import ndarray

class ZolotoException(Exception):
    pass

class MissingCalibrationsError(ZolotoException):
    pass

class CameraReadError(ZolotoException):
    def __init__(self, frame: Optional[ndarray]):
        self.frame = frame
        super().__init__()

max_stars_repo_name: RealOrangeOne/yuri

max_issues_repo_name: RealOrangeOne/yuri

max_forks_repo_name: RealOrangeOne/yuri


text:
import ctk, qt, slicer, vtk

def displayable(obj):
  """Convert Slicer-specific objects to displayable objects
  """
  try:

    # Workaround until xeus-python support matplotlib inline backend
    if str(type(obj)) == "<class 'matplotlib.figure.Figure'>":
      return MatplotlibDisplay(obj)

    if hasattr(obj, "IsA"):
      # MRML node
      if obj.IsA("vtkMRMLMarkupsNode"):
        return slicer.util.dataframeFromMarkups(obj)
      elif obj.IsA("vtkMRMLTableNode"):
        return slicer.util.dataframeFromTable(obj)
      elif obj.IsA("vtkMRMLModelNode"):
        return ModelDisplay(obj)
      elif obj.IsA("vtkMRMLTransformNode"):
        return TransformDisplay(obj)

  except:
    # Error occurred, most likely the input was not a known object type
    return obj

  # Unknown object
  return obj

class ModelDisplay(object):
  def __init__(self, modelNode, imageSize=None, orientation=None, zoom=None, showFeatureEdges=False):
    # rollPitchYawDeg
    orientation = [0,0,0] if orientation is None else orientation
    zoom = 1.0 if zoom is None else zoom
    imageSize = [300,300] if imageSize is None else imageSize
    showFeatureEdges = showFeatureEdges

    modelPolyData = modelNode.GetPolyData()

    renderer = vtk.vtkRenderer()
    renderer.SetBackground(1,1,1)
    renderer.SetUseDepthPeeling(1)
    renderer.SetMaximumNumberOfPeels(100)
    renderer.SetOcclusionRatio(0.1)
    renWin = vtk.vtkRenderWindow()
    renWin.OffScreenRenderingOn()
    renWin.SetSize(imageSize[0], imageSize[1])
    renWin.SetAlphaBitPlanes(1); # for depth peeling
    renWin.SetMultiSamples(0); # for depth peeling
    renWin.AddRenderer(renderer)

    # Must be called after iren and renderer are linked or there will be problems
    renderer.Render()

    modelNormals = vtk.vtkPolyDataNormals()
    modelNormals.SetInputData(modelPolyData)

    modelMapper = vtk.vtkPolyDataMapper()
    modelMapper.SetInputConnection(modelNormals.GetOutputPort())

    modelActor = vtk.vtkActor()
    modelActor.SetMapper(modelMapper)
    modelActor.GetProperty().SetColor(0.9, 0.9, 0.9)
    modelActor.GetProperty().SetOpacity(0.8)
    renderer.AddActor(modelActor)

    triangleFilter = vtk.vtkTriangleFilter()
    triangleFilter.SetInputConnection(modelNormals.GetOutputPort())

    if showFeatureEdges:

      edgeExtractor = vtk.vtkFeatureEdges()
      edgeExtractor.SetInputConnection(triangleFilter.GetOutputPort())
      edgeExtractor.ColoringOff()
      edgeExtractor.BoundaryEdgesOn()
      edgeExtractor.ManifoldEdgesOn()
      edgeExtractor.NonManifoldEdgesOn()

      modelEdgesMapper = vtk.vtkPolyDataMapper()
      modelEdgesMapper.SetInputConnection(edgeExtractor.GetOutputPort())
      modelEdgesMapper.SetResolveCoincidentTopologyToPolygonOffset()
      modelEdgesActor = vtk.vtkActor()
      modelEdgesActor.SetMapper(modelEdgesMapper)
      modelEdgesActor.GetProperty().SetColor(0.0, 0.0, 0.0)
      renderer.AddActor(modelEdgesActor)

    # Set projection to parallel to enable estimate distances
    renderer.GetActiveCamera().ParallelProjectionOn()
    renderer.GetActiveCamera().Roll(orientation[0])
    renderer.GetActiveCamera().Pitch(orientation[1])
    renderer.GetActiveCamera().Yaw(orientation[2])
    renderer.ResetCamera()
    renderer.GetActiveCamera().Zoom(zoom)

    windowToImageFilter = vtk.vtkWindowToImageFilter()
    windowToImageFilter.SetInput(renWin)
    windowToImageFilter.Update()

    screenshot = ctk.ctkVTKWidgetsUtils.vtkImageDataToQImage(windowToImageFilter.GetOutput())

    bArray = qt.QByteArray()
    buffer = qt.QBuffer(bArray)
    buffer.open(qt.QIODevice.WriteOnly)
    screenshot.save(buffer, "PNG")
    self.dataValue = bArray.toBase64().data().decode()
    self.dataType = "image/png"

  def _repr_mimebundle_(self, include=None, exclude=None):
    return { self.dataType: self.dataValue }

class TransformDisplay(object):
  """This class displays information about a transform in a Jupyter notebook cell.
  """
  def __init__(self, transform):
    if transform.IsLinear():
      # Always print linear transforms as transform to parent matrix
      self.dataValue = "Transform to parent:

"+np.array2string(slicer.util.arrayFromTransformMatrix(transform))+"
"
    else:
      # Non-linear transform
      if transform.GetTransformToParentAs('vtkAbstractTransform', False, True):
        # toParent is set (fromParent is just computed)
        self.dataValue = 'Transform to parent:
'
        self.dataValue += transform.GetTransformToParentInfo().replace('\n','
')
      else:
        # fromParent is set (toParent is just computed)
        self.dataValue = 'Transform from parent:
'
        self.dataValue += transform.GetTransformFromParentInfo().replace('\n','
')
  def _repr_mimebundle_(self, include=None, exclude=None):
    return { "text/html": self.dataValue }

class ViewDisplay(object):
  """This class captures current views and makes it available
  for display in the output of a Jupyter notebook cell.
  :param viewLayout: FourUp, Conventional, OneUp3D, OneUpRedSlice,
    OneUpYellowSlice, OneUpGreenSlice, Compare, Dual3D,
    ConventionalWidescreen, CompareWidescreen, Triple3DEndoscopy,
    ThreeOverThree, FourOverFour, CompareGrid, TwoOverTwo,
    SideBySide, FourByThreeSlice, FourByTwoSlice, FiveByTwoSlice,
    ThreeByThreeSlice, FourUpTable, 3DTable, ConventionalPlot,
    FourUpPlot, FourUpPlotTable, OneUpPlot, ThreeOverThreePlot,
    DicomBrowser
  :param center: re-center slice and 3D views
  """
  def __init__(self, viewLayout=None, center=True):
    layoutManager = slicer.app.layoutManager()
    if viewLayout:
      setViewLayout(viewLayout)
    if center:
      slicer.util.resetSliceViews()
      for viewId in range(layoutManager.threeDViewCount):
        reset3DView(viewId)
    slicer.util.setViewControllersVisible(False)
    slicer.app.processEvents()
    slicer.util.forceRenderAllViews()
    screenshot = layoutManager.viewport().grab()
    slicer.util.setViewControllersVisible(True)
    bArray = qt.QByteArray()
    buffer = qt.QBuffer(bArray)
    buffer.open(qt.QIODevice.WriteOnly)
    screenshot.save(buffer, "PNG")
    self.dataValue = bArray.toBase64().data().decode()
    self.dataType = "image/png"
  def _repr_mimebundle_(self, include=None, exclude=None):
    return { self.dataType: self.dataValue }

class ViewSliceDisplay(object):
  """This class captures a slice view and makes it available
  for display in the output of a Jupyter notebook cell.
  """
  def __init__(self, viewName=None, positionPercent=None):
    if not viewName:
      viewName = "Red"
    layoutManager = slicer.app.layoutManager()
    slicer.app.processEvents()
    sliceWidget = layoutManager.sliceWidget(viewName)
    if positionPercent is not None:
      sliceBounds = [0,0,0,0,0,0]
      sliceWidget.sliceLogic().GetLowestVolumeSliceBounds(sliceBounds)
      positionMin = sliceBounds[4]
      positionMax = sliceBounds[5]
      position = positionMin + positionPercent / 100.0 * (positionMax - positionMin)
      sliceWidget.sliceController().sliceOffsetSlider().setValue(position)
    sliceView = sliceWidget.sliceView()
    sliceView.forceRender()
    screenshot = sliceView.grab()
    bArray = qt.QByteArray()
    buffer = qt.QBuffer(bArray)
    buffer.open(qt.QIODevice.WriteOnly)
    #screenshot.save(buffer, "PNG")
    screenshot.save(buffer, "JPG")
    self.dataValue = bArray.toBase64().data().decode()
    #self.dataType = "image/png"
    self.dataType = "image/jpeg"
  def _repr_mimebundle_(self, include=None, exclude=None):
    return { self.dataType: self.dataValue }

class View3DDisplay(object):
  """This class captures a 3D view and makes it available
  for display in the output of a Jupyter notebook cell.
  """
  def __init__(self, viewID=0, orientation=None):
    slicer.app.processEvents()
    widget = slicer.app.layoutManager().threeDWidget(viewID)
    view = widget.threeDView()
    if orientation is not None:
      camera = view.interactorStyle().GetCameraNode().GetCamera()
      cameraToWorld = vtk.vtkTransform()
      cameraToWorld.RotateX(90)
      cameraToWorld.RotateY(180)
      cameraToWorld.RotateY(orientation[2])
      cameraToWorld.RotateX(orientation[1])
      cameraToWorld.RotateZ(orientation[0])
      cameraToWorld.Translate(0, 0, camera.GetDistance())
      viewUp = [0,1,0,0]
      slicer.vtkAddonMathUtilities.GetOrientationMatrixColumn(cameraToWorld.GetMatrix(), 1, viewUp)
      position = cameraToWorld.GetPosition()
      focalPoint = camera.GetFocalPoint()
      camera.SetPosition(focalPoint[0]+position[0], focalPoint[1]+position[1], focalPoint[2]+position[2])
      camera.SetViewUp(viewUp[0:3])
      camera.OrthogonalizeViewUp()
    view.forceRender()
    screenshot = view.grab()
    bArray = qt.QByteArray()
    buffer = qt.QBuffer(bArray)
    buffer.open(qt.QIODevice.WriteOnly)
    #screenshot.save(buffer, "PNG")
    screenshot.save(buffer, "JPG")
    self.dataValue = bArray.toBase64().data().decode()
    #dataType = "image/png"
    self.dataType = "image/jpeg"
  def _repr_mimebundle_(self, include=None, exclude=None):
    return { self.dataType: self.dataValue }

class ViewLightboxDisplay(object):

  def __init__(self, viewName=None, rows=None, columns=None, filename=None, positionRange=None, rangeShrink=None):
    viewName = viewName if viewName else "Red"
    rows = rows if rows else 4
    columns = columns if columns else 6

    sliceWidget = slicer.app.layoutManager().sliceWidget(viewName)

    if positionRange is None:
      sliceBounds = [0,0,0,0,0,0]
      sliceWidget.sliceLogic().GetLowestVolumeSliceBounds(sliceBounds)
      slicePositionRange = [sliceBounds[4], sliceBounds[5]]
    else:
      slicePositionRange = [positionRange[0], positionRange[1]]

    if rangeShrink:
      slicePositionRange[0] += rangeShrink[0]
      slicePositionRange[1] -= rangeShrink[1]

    # Capture red slice view, 30 images, from position -125.0 to 75.0
    # into current folder, with name image_00001.png, image_00002.png, ...
    import ScreenCapture
    screenCaptureLogic = ScreenCapture.ScreenCaptureLogic()
    destinationFolder = 'outputs/Capture-SliceSweep'
    numberOfFrames = rows*columns
    filenamePattern = "_lightbox_tmp_image_%05d.png"
    viewNode = sliceWidget.mrmlSliceNode()
    # Suppress log messages
    def noLog(msg):
        pass
    screenCaptureLogic.addLog=noLog
    # Capture images
    screenCaptureLogic.captureSliceSweep(viewNode, slicePositionRange[0], slicePositionRange[1],
                                         numberOfFrames, destinationFolder, filenamePattern)
    # Create lightbox image
    resultImageFilename = filename if filename else filenamePattern % numberOfFrames
    screenCaptureLogic.createLightboxImage(columns, destinationFolder, filenamePattern, numberOfFrames, resultImageFilename)

    # Save result
    with open(destinationFolder+"/"+resultImageFilename, "rb") as file:
      self.dataValue = file.read()
      self.dataType = "image/png"
      # This could be used to create an image widget: img = Image(value=image, format='png')

    # Clean up
    screenCaptureLogic.deleteTemporaryFiles(destinationFolder, filenamePattern, numberOfFrames if filename else numberOfFrames+1)

  def _repr_mimebundle_(self, include=None, exclude=None):
    import base64
    return { self.dataType: base64.b64encode(self.dataValue) }

class MatplotlibDisplay(object):
  """Display matplotlib plot in a notebook cell.

  This helper function will probably not needed after this issue is fixed:
  https://github.com/jupyter-xeus/xeus-python/issues/224

  Important: set matplotlib to use agg backend:

      import matplotlib
      matplotlib.use('agg')

  Example usage:

      import matplotlib
      matplotlib.use('agg')

      import matplotlib.pyplot as plt
      import numpy as np
      # Data for plotting
      t = np.arange(0.0, 2.0, 0.01)
      s = 1 + np.sin(2 * np.pi * t)
      # Setup plot
      fig, ax = plt.subplots()
      ax.plot(t, s)
      ax.set(xlabel='time (s)', ylabel='voltage (mV)',
            title='About as simple as it gets, folks')
      ax.grid()

      slicernb.displayMatplotlib(plt)

  """
  def __init__(self, fig):
    filename = "__matplotlib_temp.png"
    fig.savefig(filename)
    with open(filename, "rb") as file:
      self.dataValue = file.read()
      self.dataType = "image/png"
    import os
    os.remove(filename)

  def _repr_mimebundle_(self, include=None, exclude=None):
    import base64
    return { self.dataType: base64.b64encode(self.dataValue) }

# Utility functions for customizing what is shown in views

def showVolumeRendering(volumeNode, show=True, presetName=None):
  volRenLogic = slicer.modules.volumerendering.logic()
  if show:
    displayNode = volRenLogic.GetFirstVolumeRenderingDisplayNode(volumeNode)
    if not displayNode:
      displayNode = volRenLogic.CreateDefaultVolumeRenderingNodes(volumeNode)
    displayNode.SetVisibility(True)
    scalarRange = volumeNode.GetImageData().GetScalarRange()
    if not presetName:
      if scalarRange[1]-scalarRange[0] < 1500:
        # small dynamic range, probably MRI
        presetName = 'MR-Default'
      else:
        # larger dynamic range, probably CT
        presetName = 'CT-Chest-Contrast-Enhanced'
    displayNode.GetVolumePropertyNode().Copy(volRenLogic.GetPresetByName(presetName))
  else:
    # hide
    volRenLogic = slicer.modules.volumerendering.logic()
    displayNode = volRenLogic.GetFirstVolumeRenderingDisplayNode(volumeNode)
    if displayNode:
      displayNode.SetVisibility(False)

def reset3DView(viewID=0):
  threeDWidget = slicer.app.layoutManager().threeDWidget(viewID)
  threeDView = threeDWidget.threeDView()
  threeDView.resetFocalPoint()

def setViewLayout(layoutName):
  layoutId = eval("slicer.vtkMRMLLayoutNode.SlicerLayout"+layoutName+"View")
  slicer.app.layoutManager().setLayout(layoutId)

def showSliceViewAnnotations(show):
    # Disable slice annotations immediately
    slicer.modules.DataProbeInstance.infoWidget.sliceAnnotations.sliceViewAnnotationsEnabled=show
    slicer.modules.DataProbeInstance.infoWidget.sliceAnnotations.updateSliceViewFromGUI()
    # Disable slice annotations persistently (after Slicer restarts)
    settings = qt.QSettings()
    settings.setValue('DataProbe/sliceViewAnnotations.enabled', 1 if show else 0)

max_stars_repo_name: fbordignon/SlicerJupyter

max_issues_repo_name: fbordignon/SlicerJupyter

max_forks_repo_name: fbordignon/SlicerJupyter


text:
"""Methods for estimating the advantage function"""

from typing import Tuple

import numpy as np

from pearll.common.enumerations import TrajectoryType
from pearll.common.type_aliases import Tensor
from pearll.common.utils import to_numpy, to_torch

def generalized_advantage_estimate(
    rewards: Tensor,
    old_values: Tensor,
    new_values: Tensor,
    dones: Tensor,
    gamma: float = 0.99,
    gae_lambda: float = 0.95,
    dtype: str = "torch",
) -> Tuple[Tensor, Tensor]:
    """
    Generalized advantage estimate of a trajectory: https://towardsdatascience.com/generalized-advantage-estimate-maths-and-code-b5d5bd3ce737

    :param old_values: value function result with old_state input
    :param new_values: value function result with new_state input
    :param rewards: agent reward of taking actions in the environment
    :param dones: the done values of each step of the trajectory, indicates whether to bootstrap
    :param gamma: trajectory discount
    :param gae_lambda: exponential mean discount
    :return advantage: the advantage of taking actions in the environment
    :return value: the value of taking actions in the environment
    """
    rewards, old_values, new_values, dones = to_numpy(
        rewards, old_values, new_values, dones
    )
    dones = 1 - dones
    batch_size = rewards.shape[0]

    advantage = (
        np.zeros(batch_size + 1) if rewards.ndim == 1 else np.zeros((batch_size + 1, 1))
    )

    for t in reversed(range(batch_size)):
        delta = rewards[t] + (gamma * new_values[t] * dones[t]) - old_values[t]
        advantage[t] = delta + (gamma * gae_lambda * advantage[t + 1] * dones[t])

    value_target = advantage[:batch_size] + old_values

    assert (
        advantage[:batch_size].shape == old_values.shape
    ), f"The advantage's shape should be {old_values.shape}, instead it is {advantage[:batch_size].shape}."
    assert (
        value_target.shape == old_values.shape
    ), f"The returns' shape should be {old_values.shape}, instead it is {value_target.shape}."

    if TrajectoryType(dtype.lower()) == TrajectoryType.TORCH:
        return to_torch(advantage[:batch_size], value_target)
    elif TrajectoryType(dtype.lower()) == TrajectoryType.NUMPY:
        return advantage[:batch_size], value_target
    else:
        raise ValueError(
            f"`dtype` flag should be 'torch' or 'numpy', but received {dtype}"
        )

max_stars_repo_name: LondonNode/pearll

max_issues_repo_name: LondonNode/pearll

max_forks_repo_name: LondonNode/pearll


text:
import numpy as np
## Configure range of estimaters

'''
    Random Forest:
        n_estimators = number of trees in the foreset
        max_features = max number of features considered for splitting a node
        max_depth = max number of levels in each decision tree
        min_samples_split = min number of data points placed in a node before the node is split
        min_samples_leaf = min number of data points allowed in a leaf node
        bootstrap = method for sampling data points (with or without replacement)
'''
n_estimators = [int(x) for x in np.linspace(start=1,stop=2000, num=10)]
max_features = ['auto', 'sqrt', 'log2']
max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
max_depth.append(None)
min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
class_weight = ["balanced", "balanced_subsample", None]
bootstrap = [True, False]
rfc_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'class_weight' : class_weight,
               'bootstrap': bootstrap}

'''
    {'n_estimators': 1333,
    'min_samples_split': 5,
    'min_samples_leaf': 1,
    'max_features': 'sqrt',
    'max_depth': 50,
    'class_weight':
    'balanced_subsample',
    'bootstrap': True}

'''
narrowed_rfc_grid =  {
    'n_estimators': [800, 1000, 1200, 1600, 2000],
    'min_samples_split': [4, 5, 7, 9],
    'min_samples_leaf': [1,2,3],
    'max_features': ['auto', 'sqrt'],
    'max_depth': [40, 50, 60, 70], 'class_weight': ['balanced',"balanced_subsample"],
    'bootstrap': [True]}

'''
    Support Vector Machine:
        C : Regularization parameter. The strength of the regularization is inversely proportional to C. penalty is a squared l2 penalty
        kernel : Specifies the kernel type to be used in the algorithm.
        gamma: {‘scale’, ‘auto’} or float, default=’scale’. Kernel coefficient for ‘rbf’, ‘poly’ and ‘sigmoid’.
            kernel='precomputed'can only be used when passing a
            (n_samples, n_samples) data matrix that represents
            pairwise similarities for the samples instead of the
            traditional (n_samples, n_features) rectangular data matrix.
'''

svm_grid = {
    'C': [0.01, 0.1, 1, 10, 100, 1000],
    'kernel' : ['linear', 'poly', 'rbf', 'sigmoid'],
    'gamma': [1, 0.1, 0.01, 0.001, 0.0001,'scale', 'auto']
}

narrowd_svm_grid = {
    'C': list(map(int, np.linspace(start=2,stop=20, num=10))),
    'kernel' : ['linear', 'poly', 'rbf', 'sigmoid'],
    'gamma': [*np.linspace(start=0.1,stop=10, num=10), 'scale', 'auto']
}

'''
    random state: 0, 1, 42
'''
tune_search_params = {
    'cv': 5,
    'verbose': 2,
    # 'random_state': 1,
    'n_jobs': -1,
    # 'n_iter': 30
}


text:
import numpy as np
import numba

@numba.jit()
def FPI_solver(A_matrix, f_vector, u0_vector=None, eps=10e-7, n_iter=10000):
    """
    Решение операторного уравнения методом простых итераций

    :param A_matrix: Входная матрица оператора А исходного операторного уравнения, ожидается shape=(n, n)
    :param f_vector: Входной вектор правой части f исходного операторного уравнения, ожидается shape=(n,)
    :param u0_vector: Начальное приближение вектора u0 - решения исходного операторного уравнения, ожидается shape=(n,)
    :param eps: Заданная точность для приближаемого вектора решения исходного операторного уравнения, по метрике max(abs(u(s) - u(s-1)))
    :param n_iter: Граничное число итераций приближения искомого вектора (число шагов)
    :return:
    """

    if A_matrix.shape[0] != A_matrix.shape[1]:
        print("\n A_matrix is not a square matrix \n")
        raise ValueError

    # Размерность задачи
    row_size = A_matrix.shape[0]

    # Заполнение случайными числами
    if u0_vector is None:
        u0_vector = np.random.uniform(-1., 1., row_size).reshape((1, row_size))
    else:
        u0_vector = u0_vector.reshape((1, row_size))

    # Нормировка матрицы для улучшения сходимости задачи
    max_value = max(np.amax(np.abs(A_matrix)), np.amax(np.abs(f_vector)))

    # TODO: 2*N*N операций
    A_matrix = A_matrix / max_value
    f_vector = f_vector / max_value

    # Инициализация матрицы B
    # TODO: N операций
    B_matrix = np.diag(np.ones(row_size)) - A_matrix

    # Начало итераций
    u_vector = u0_vector.copy()
    for iter_idx in range(1, (n_iter + 1)):
        # TODO (N^2 + N) * steps операций
        u_vector = np.concatenate((u_vector, (B_matrix @ u_vector[iter_idx - 1] + f_vector).reshape(1, row_size)), 0)
        if np.amax(np.abs(u_vector[iter_idx] - u_vector[iter_idx - 1])) < eps:
            break

    return u_vector

reason: import numpy,import numba