text:
"""
Functions for audio handling and processing.

All audio signals are represented NumPy arrays (in the form `x[t]` for mono
signals and `x[t, channel]` for non-mono signals).
"""

import abc
import itertools
import math
import wave

import numpy as np
import pyaudio

import dh.utils
import dh.image

###
#%% input & output
###

def load(filename):
    """
    Load WAVE file.

    Returns a 2-tuple consisting of the data as NumPy array and the sampling
    rate as float scalar.
    """
    import scipy.io.wavfile
    (rate, data) = scipy.io.wavfile.read(filename=filename)
    return (data, rate)

def save(filename, data, rate=44100):
    """
    Save audio data to WAVE file.
    """
    import scipy.io.wavfile
    dh.utils.mkpdir(filename)
    scipy.io.wavfile.write(filename=filename, rate=rate, data=data)

def play(data, rate):
    """
    Play the audio signal.
    """
    # save to temporary WAVE file
    chunkSize = 1024
    filename = "/tmp/__dh.audio.play.wav"
    save(filename, data, rate)

    # open and read from this temporary WAVE file
    wav = wave.open(filename, "rb")
    p = pyaudio.PyAudio()
    stream = p.open(
        format=p.get_format_from_width(wav.getsampwidth()),
        channels=wav.getnchannels(),
        rate=wav.getframerate(),
        output=True
    )
    data = wav.readframes(chunkSize)

    # play audio (writing to the stream is what *actually* plays the sound)
    while len(data) > 0:
        stream.write(data)
        data = wav.readframes(chunkSize)

    # cleanup
    stream.close()
    p.terminate()

class Recorder(abc.ABC):
    _DTYPES = {
        "int8":  {"bytes": 1, "pyaudio": pyaudio.paInt8, "numpy": "int8"},
        "int16": {"bytes": 2, "pyaudio": pyaudio.paInt16, "numpy": "int16"},
        "int32": {"bytes": 4, "pyaudio": pyaudio.paInt32, "numpy": "int32"},
    }

    def __init__(self, channelCount=2, rate=44100, dtype="int16", chunkSize=1024):
        """
        Records audio from a microphone. The data type `dtype` must be one of
        "int8", "int16", "int32".
        """
        self.channelCount = channelCount
        self.rate = rate
        self.dtype = dtype
        self.chunkSize = chunkSize
        assert self.dtype in Recorder._DTYPES.keys()

    def _durationToChunkCount(self, duration=None):
        """
        For a given duration in seconds, returns the number of full chunks that
        fit into this time span (or `None`, if duration is `None`).
        """
        if duration is None:
            return None
        else:
            chunksPerSecond = self.rate / self.chunkSize
            return math.ceil(chunksPerSecond * duration)

    def chunks(self, maxCount=None):
        """
        Generator which yields chunks of data. If `maxCount` is a number, the
        generator stops after that many chunks have been returned.
        """
        # set up PyAudio stream
        p = pyaudio.PyAudio()
        stream = p.open(
            format=Recorder._DTYPES[self.dtype]["pyaudio"],
            channels=self.channelCount,
            rate=self.rate,
            input=True,
            frames_per_buffer=self.chunkSize,
        )

        # record and yield data, chunk-wise
        for nChunk in itertools.count():
            if (maxCount is None) or (nChunk < maxCount):
                yield stream.read(self.chunkSize)
            else:
                break

        # shutdown PyAudio
        stream.stop_stream()
        stream.close()
        p.terminate()

class WavRecorder(Recorder):
    def record(self, filename, duration=None):
        """
        Record audio and simultaneously save it to the WAV-file named
        `filename`. `duration` gives the recording time in seconds
        (infinite if `None`).
        """

        chunks = self.chunks(maxCount=self._durationToChunkCount(duration))
        with wave.open(filename, "wb") as wav:
            wav.setnchannels(self.channelCount)
            wav.setsampwidth(Recorder._DTYPES[self.dtype]["bytes"])
            wav.setframerate(self.rate)
            for chunk in chunks:
                wav.writeframes(chunk)

class NumpyRecorder(Recorder):
    def record(self, duration=None):
        pass

    def stream(self, length=1.0):
        frameCount = round(self.rate * length)

        numpyDtype = Recorder._DTYPES[self.dtype]["numpy"]
        x = np.zeros(shape=(0, self.channelCount), dtype=numpyDtype)
        chunks = self.chunks()
        for chunk in chunks:
            # get chunk and convert it to NumPy array
            c = np.fromstring(chunk, dtype=numpyDtype)
            c.shape = (self.chunkSize, self.channelCount)

            # add chunk to the data array and remove data older than `length`
            x = np.vstack([x, c])
            x = x[max(0, x.shape[0] - frameCount):, :]

            yield x

###
#%% data type and mode handling
###

def trange(dtype):
    """
    Returns the range (min, max) of valid intensity values for an audio signal
    of NumPy type string `dtype`.

    Allowed types are `'int8'`, `'int16'`, `'int32'`, and any float type
    (e.g., `'float32'`, `'float64'`).

    >>> trange('int8')
    (-128, 127)
    >>> trange('int16')
    (-32768, 32767)
    >>> trange('float32')
    (-1.0, 1.0)
    >>> trange('float64')
    (-1.0, 1.0)
    """

    if dtype is None:
        # np.issubdtype(None, "float") is True, therefore we have to check for this error here explicitly
        raise ValueError("Invalid data type '{dtype}'".format(dtype=dtype))
    elif dtype == "int8":
        return (-128, 127)
    elif dtype == "int16":
        return (-32768, 32767)
    elif dtype == "int32":
        return (-2147483648, 2147483647)
    elif np.issubdtype(dtype, np.floating):
        return (-1.0, 1.0)
    else:
        raise ValueError("Unsupported data type '{dtype}'".format(dtype=dtype))

def convert(x, dtype):
    """
    Converts audio signal `s` to NumPy type given by the string `dtype` and
    scales the values accordingly.

    The values are always clipped to the allowed range (even for identical
    source and target types). Returns always a copy of the data, even for equal
    source and target types.
    """

    # clip image against its source dtype (important for floats)
    (lowerOld, upperOld) = trange(x.dtype)
    y = clip(x, lowerOld, upperOld)

    if x.dtype == dtype:
        return y
    else:
        (lowerNew, upperNew) = trange(dtype)
        return ((y.astype("float") - lowerOld) / (upperOld - lowerOld) * (upperNew - lowerNew) + lowerNew).astype(dtype)

###
#%% element-wise operations
###

def clip(x, lower=None, upper=None):
    """
    Clips the signal values to the interval [`lower`, `upper`], and preserves
    the data type.

    Always returns a copy of the data, even if both interval ends are `None`.
    """

    y = x.copy()
    dtype = y.dtype
    (tLower, tUpper) = trange(dtype)
    if lower is not None:
        y = np.maximum(y, np.array((dh.utils.sclip(lower, tLower, tUpper),), dtype=dtype))
    if upper is not None:
        y = np.minimum(y, np.array((dh.utils.sclip(upper, tLower, tUpper),), dtype=dtype))
    return y

###
#%% feature transforms
###

def spectrogram(x, rate):
    import scipy.signal
    y = convert(x[:, 0], "float")
    res = scipy.signal.spectrogram(y, fs=rate, mode="magnitude")
    S = res[2]
    S = S[::-1, :]
    return S

max_stars_repo_name: dhaase-de/dh-python-dh

max_issues_repo_name: dhaase-de/dh-python-dh

max_forks_repo_name: dhaase-de/dh-python-dh

reason: import numpy,import scipy


text:
#!/usr/bin/env python
# coding: utf-8

# # Klasterska analiza

# >U ovoj svesci ćemo uraditi primer klasterske analize sa većim brojem indikatora. Pomoću faktorske analize ćemo proceniti koliko je faktora za ovaj skup podataka zaista važno i pokušaćemo da ih opišemo. Primer je urađen za klasifikaciju država prema stepenu ekonomskog razvoja.

# Pravljenje rang-lista je jednostavno kada objekte treba poređati po veličini jednog parametra. Mi, međutim, često pravimo rang-liste za vrlo složene i apstraktne konstrukte kao što su konkurentnost privrede, kvalitet života, inovativnost itd. Da bi takve liste bile nešto više od subjektivnog rangiranja potrebno je da izaberemo najvažnije karakteristike, da procenimo koliko su važne, da li su možda međusobno previše zavisne i da smislimo odgovarajuću metriku.

# Svetska banka klasifikuje države i ekonomije prema stepenu ekonomskog razvoja u [četiri grupe](https://blogs.worldbank.org/opendata/new-country-classifications-income-level-2019-2020): visoku, gornju-srednju, donju-srednju i nisku. Za to koristi samo jedan parametar: bruto nacionalni proizvod (BNP, eng. GNP - _Gross national product_) po glavi stanovnika. Nesumnjivo, prosečni proizvod stanovnika određene teritorije jeste važan pokazatelj njene razvijenosti. Ipak, postoje velike razlike među državama sa sličnim BNP po glavi stanovnika. Na taj način u istu grupu stavlja npr. Norvešku i Saudijsku Arabiju ili Kinu i Angolu. Jasno nam je da samo jedan parametar ne može da bude dobra osnova za dobru klasifikaciju prema ekonomskom razvoju. Takođe poređenje ekonomskog razvoja sa indikatorima iz drugih oblasti neće dati značajne rezultate jer su grupe previše grubo određene. Zamislite da poredimo postignuće na PISA testu za različite grupe zemalja gde su u istoj grupi npr. Finska i Oman. Varijansa bi bila prevelika da bi poređenje imalo smisla. Potrebno nam je još parametara.

# Na sajtu Svetske banke postoji [kolekcija različitih indikatora](https://data.worldbank.org/indicator) o stepenu razvoja svih država i ekonomija koji se prikupljaju iz različitih izvora i po različitoj metodologiji. Sistematizacija ovih podataka nije jednostavan posao jer ne postoje svi podaci za sve zemlje i nema istraživanja za svaku godinu. Zato je potrebno napraviti odabir indikatora i kriterijum po kom ih povezujemo.

# ## Podaci
#
# Za ovu vežbu iskoristiti fajl "countries data.csv" sa 29 indikatora koje je sakupio i priredio [Zedric Cheung](https://towardsdatascience.com/factor-analysis-cluster-analysis-on-countries-classification-1bdb3d8aa096). Ovi indikatori opisuju ne samo ekonomski već i društveni razvoj. U nedostatku podataka za aktuelnu godinu, uzeti su najnoviji dostupni podaci. Da bismo izbegli moguću pristrasnost zbog nesrazmernih vrednosti za određene indikatore, kao što su npr. BNP ili broj stanovnika, izabrani su samo relativni indikatori i indikatori rasta koji se uglavnom izražavaju u procentima.

# In[1]:

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

# In[2]:

df=pd.read_csv('data/df_final.csv',index_col=0)

# In[3]:

df.head()

# Tabela je, evidentno, napravljena tako da su faktori prikazani po redovima, a države po kolonama. To nam nije zgodno za funkcije poput `corr()` koje koriste kolone kao argumente. Zato ćemo pomoću funkcije `T` transponovati _DataFrame_, tj. okrenuti tabelu tako da zamenimo redove i kolone.

# In[4]:

df=df.T

# In[5]:

df.head()

# Sada su države po vrstama, a indikatori po kolonama pa možemo da vidimo spisak svih odabranih indikatora.

# In[6]:

indicators=list(df.columns)
countries=list(df.index)

# In[7]:

indicators

# ## Faktorska analiza

# Osnovna namena faktorske analize jeste da opiše međusobne veze nekoliko promenljivih koristeći manji broj dimenzija koje ne možemo direktno da posmatramo i merimo. Ove dimenzije nazivamo faktorima.

# To što smo uzeli 29 indikatora ne znači da su svi nezavisni. Iza njih se verovatno krije svega nekoliko faktora koji najvećim delom objašnjavaju vrednosti indikatora. Faktorska analiza može da nam pokaže koji su to faktori najvažniji i šta ih čini.

# Bez namere da ovde ulazimo u detalje faktorske analize, iskoristićemo ovaj model za redukciju broja indikatora kojima opisujemo ekonomiju jedne zemlje. Ovaj postupak se zove smanjenje dimenzionalnosti (eng. _dimensionality reduction_).

# Faktorska analiza podrazumeva linearnu vezu promenljivih i zajedničkih faktora. Posmatrane promenljive se modeliraju kao linearne kombinacije faktora i statističkih grešaka. Ova pretpostavka se koristi za objašnjenje varijanse posmatrane promenljive i uočavanje faktora. Ukoliko postoji jaka nelinearna veza između promenljivih, onda faktorska analiza neće biti adekvatna.

# Faktorska analiza, takođe, pretpostavlja da promenljive mogu da budu međusobno zavisne. Na taj način promenljive mogu da se grupišu prema nekom zajedničkom svojstvu. U tom slučaju sve promenljive iz jedne grupe imaju međusobno visoke korelaciije dok su korelacije sa promenljivim iz drugih grupa niske. O svakoj grupi ovde možemo da razmišljamo kao o reprezentaciji istog osnovnog konstrukta ili faktora.

# ### Korelaciona matrica

# Pre nego što krenemo u dublju analizu, dobro je pogledati kako indikatori međusobno koreliraju. To može da nam pokaže korelaciona matrica.

# In[8]:

plt.figure(figsize=(10,8))
corrMatrix = df.corr()
sns.heatmap(corrMatrix)

# Odavde vidimo mnoge interesantne veze među promenljivim. Za neke od njih možemo da nađemo logično objašnjenje. Vidimo npr. pozitivnu korelaciju između dostupnosti električne energije i procenta ljudi koji koriste usluge osnovnog snabdevanja vodom ili negativnu korelaciju između udela ruralne populacije i udela populacije koja koristi internet. Ove dve veze nisu iznenađenje i za njih bismo mogli da nađemo prihvatljivo objašnjenje.
#
# Za faktorsku analizu, međutim, nije neophodno da imamo objašnjenje uzročno-posledične veze između dve promenljive. Za analizu je važno da veza postoji, da može da se izmeri koliko je ona jaka i da na osnovu toga mogu da se grupišu. Iz zajedničkih svojstava tih grupa bi trebalo da prepoznamo faktore.

# Za kod koji sledi koristićemo biblioteku __factor_analyzer__ koju uglavnom nemamo u okruženju. Ako je već niste instalirali, možete to da uradite pomoću koda u sledećoj ćeliji. Naravno, uklonite # pre nego što pokrenete kod.

# In[9]:

# pip install factor_analyzer

# Da bismo videli koji je faktor koliko značajan izračunaćemo njihove jedinstvene vrednosti (_eigenvalues_). Pomoću funkcije `FactorAnalyzer()` ćemo konstruisati objekat __fa__ čije elemente dobijamo fitovanjem podataka datih u tabeli __df__ pomoću funkcije `fit()`.
#
# Funkcija `get_eingenvalues()` za objekat __fa__ vraća dva niza: originalne jedinstvene vrednosti i jedinstvene vrednosti za zajednički faktor. Nas će u ovom primeru interesovati samo prvi vraćeni niz __ev__.

# In[10]:

from factor_analyzer import FactorAnalyzer

# In[11]:

fa = FactorAnalyzer()
fa.fit(df)

ev, v = fa.get_eigenvalues()

# In[12]:

ev

# Dobili smo 29 jedinstvenih vrednosti za faktore koliko ima i indikatora. Ove vrednosti su već sortirane po veličini jedinstvenih vrednosti i vidimo da nemaju svi faktori isti značaj. Prvi faktor je svakako najznačajniji. Da bismo videli koliko ima značajnih faktora primenićemo Kajzerov kriterijum (_Kaiser criterion_). To znači da uzimamo samo one faktore sa jedinstvenom vrednošću većom ili jednakom 1. Pošto sedmi faktor ima jedinstvenu vrednost skoro 1 (0.976), možemo da uključimo i taj sedmi. Kad nacrtamo "_scree plot_" vidi se koje su jedinstvene vrednosti za koji faktor i gde bi trebalo da se zaustavimo.

# In[13]:

plt.scatter(range(1,df.shape[1]+1),ev)
plt.plot(range(1,df.shape[1]+1),ev)
plt.hlines(1, 0, df.shape[1], colors='r')
plt.title('Scree Plot')
plt.xlabel('Faktori')
plt.ylabel('Jedinstvene vrednosti')
plt.grid()
plt.show()

# Uzeli smo prvih 7 faktora i sada nas interesuje za svaki indikator iz tabele __df__ koje su mu komponenente kad ga razložimo po faktorima. Opet ćemo pomoću funkcije `fit()` uraditi faktorsku analizu, samo što nas sada ne interesuju jedinstvene vrednosti nego komponente koje dobijamo funkcijom `loadings_`.
#

# In[14]:

n=7
fa=FactorAnalyzer(n, rotation='varimax')
fa.fit(df)
loads=fa.loadings_
loads=pd.DataFrame(loads, index=df.columns)

# Težine faktora za svaki indikator, odnosno komponente indikatora po faktorima, nisu ništa drugo do koeficijenti korelacije između pojedinačnih indikatora i faktora. Kvadrat tog broja pokazuje koliki deo varijanse indikatora objašnjava svaki pojedinačni faktor.
#
# U tabeli __loads__ vidimo težine faktora za svaku promenljivu. Dostupnost električne energije u ruralnim oblastima (__Access to electricity, rural__) je najvećim delom (0.923) posledica faktora 1, dok je npr. __Employment to population ratio, ages 15-24, total (%)__ najvećim delom posledica faktora 2.

# In[15]:

loads

# Komponente pojedinačnih promenljivih možemo da prikažemo grafički pomoću funkcije `heatmap()`.

# In[16]:

plt.figure(figsize=(12,12))
sns.heatmap(loads, annot=True, cmap="coolwarm")

# Koliko dobro pomoću prvih sedam faktora možemo da opišemo indikatore, to kvantifikujemo pomoću udela varijanse za koju su odgovorni faktori. Za svaki faktor posebno možemo da vidimo koliki ima udeo ukupne varijanse. Kumulativni udeo nam govori koliko varijanse opisuju zajedno taj i svi prethodni faktori. Te podatke možemo da dobijemo pomoću funkcije `get_factor_variance()`.

# In[17]:

fa_var = fa.get_factor_variance()
fa_var = pd.DataFrame(fa_var, index=['SS loadings', 'Proportion Var', 'Cumulative Var'])
fa_var

# Vidimo da je izbor prvih sedam faktora objasnio 71% ukupne varijanse svih 29 indikatora.

# ### Tumačenje faktora

# Iako su faktori apstraktni i ne možemo direktno da ih merimo, možemo da ih opišemo na osnovu veze sa pojedinačnim indikatorima. Npr. faktor 0 ima visoku korelaciju sa dostupnošću električne energije, interneta i pijaće vode. Možemo reći da ovaj faktor generalno opisuje dostupnost osnovnih komunalnih usluga. Slično možemo da prepoznamo i za ostale faktore.
#
# Okvirno, sedam faktora ekonomskog razvoja na osnovu izabranih indikatora grubo možemo da opišemo kao:
#
# 1. faktor — Dostupnost osnovnih komunalnih usluga
# 2. faktor — Zapošljivost mladih
# 3. faktor — Ukupni ekonomski rast
# 4. faktor — Industrijski razvoj
# 5. faktor — Zdravstvena situacija
# 6. faktor — Mogućnosti proizvodnje i trgovine proizvedenom robom
# 7. faktor — Razvoj stručnih usluga
#
# Imajte u vidu da tumačenja podataka ne moraju da budu jedinstvena, da su veoma zavisna od konteksta i da su ponekad subjektivna. Zato je važno da naglasimo na osnovu čega dajemo takvo tumačenje.

# In[18]:

faktori=['Dostupnost osnovnih\n komunalnih usluga',
         'Zapošljivost mladih',
         'Ukupni ekonomski rast',
         'Industrijski razvoj',
         'Zdravstvena situacija',
         'Mogućnosti proizvodnje i\n trgovine proizvedenom robom',
         'Razvoj stručnih usluga']

# ## Klasterska analiza

# Vratimo se sada na svih 29 indikatora. Njih možemo predstaviti kao 29 dimenzija nekog hiper-prostora u kom je definisano rastojanje između bilo koje dve tačke. Ako krenemo _bottom-up_ i tražimo države koje su međusobno nabliže, počeće da se stvaraju klasteri država sa sličnim karakteristikama.

# ### Hijerarhijska klasterizacija

# Hijerarhijska klasterizacija je tip učenja bez nadzora koji grupiše slične tačke ili objekte u grupe koje nazivamo klasterima. Postoje dva podtipa hijerarhijske klasterizacije:
# - Aglomerativna hijerarhijska klasterizacija i
# - Hijerarhijska klasterizacija razdvajanjem.
#
# Mi ćemo ovde prikazati primer korišćenja samo prvog od ova dva podtipa. Aglomerativna hijerarhijska klasterizacija ima _bottom-up_ pristup gde su svi objekti na početku zasebni klasteri koji se sukcesivno sparuju čineći sve veće klastere.

# #### Standardizacija

# Za uspešno uočavanje klastera potrebno je sve indikatore, bez obzira na raspon vrednosti i jedinice u kojima su dati, dovesti u ravnopravan položaj tako što im se vrednosti standardizuju. To činimo tako što originalne vrednosti indikatora zamenjujemo razlikom originalne i srednje vrednosti za taj indikator podeljenu standardnom devijacijom indikatora. To znači da su tipične standardizovane vrednosti uglavnom između -1 i +1.

# In[19]:

df_std=(df-df.mean())/df.std()

# Sa ovako transformisanom tabelom __df__ možemo da utvrdimo rastojanja između tačaka (država) koristeći podrazumevanu euklidsku distancu i Ward metod povezivanja klastera. Njihova klasterizacija može da se prikaže dijagramom koji nazivamo dendrogram zbog vizuelne sličnosti sa korenom drveta. Dendrogram prikazuje kako se klasteri spajaju ili razdvajaju na određenim rastojanjima. Skup tačaka može da ima različit broj klastera u zavisnosti od parametra kritičnog rastojanja koji je prikazan na y-osi. U krajnjoj liniji, izbor broja klastera je proizvoljan i vrlo često subjektivan.

# In[20]:

import scipy.cluster.hierarchy as sch
from sklearn.cluster import AgglomerativeClustering

# In[21]:

plt.subplots(figsize=(20,8))
dendrogram = sch.dendrogram(sch.linkage(df_std, method='ward'),leaf_rotation=90,
                            leaf_font_size=8, color_threshold=12.5, labels=df.index)
plt.axhline(y=12.5, color='gray', lw=1, linestyle='--');

# Funkcija `dendrogram()` ne vraća podatke o klasterima jer još nismo zadali kriterijum koliki su ili koliko ih ima. Za grupisanje država u klastere nam je potrebna funkcija `AgglomerativeClustering()` u kojoj ćemo zadati da hoćemo tačno 12 klastera. To odgovara visini crne isprekidane linije na gornjem grafikonu. Svaki presek ove linije sa dendrogramom definiše po jedan klaster ispod linije. Funkcija `fit_predict()` vraća niz celih brojeva koji predstavljaju procenu pripadnosti tačaka različitim klasterima.

# In[22]:

# kreiranje objekta
hc = AgglomerativeClustering(n_clusters=12, affinity = 'euclidean', linkage = 'ward')
# fitovanje modela
y_hc = hc.fit_predict(df_std)

# Sada tabelama __df__ i njenoj standardizovanoj verziji (__df_std__) možemo da dodamo kolonu u kojoj će pisati kom klasteru pripada koja država. Nakon toga sortiramo tabele prema oznaci klastera tako da su nam na početku klasteri sa oznakom 0, pa onda 1 itd.

# In[23]:

df['cluster'] = y_hc
df.sort_values("cluster", inplace = True, ascending=True)

df_std['cluster'] = y_hc
df_std.sort_values("cluster", inplace = True, ascending=True)

# ### Klasteri država

# Sada pošto smo državama pridružili oznake klastera kojima pripadaju bilo bi dobro da vidimo koja je država u kom klasteru. Za tu namenu napravićemo privremeno tabelu __a__ koja sadrži samo imena država i oznake klastera i od nje napraviti tabelu __grupe__ gde su imena država odgovarajućeg klastera navedena po kolonama.

# In[24]:

a=pd.DataFrame({'country':df.index, 'cluster':df.cluster})

# In[25]:

mig=max(df.cluster.value_counts())
grupe=pd.DataFrame('',columns=df.cluster.unique(),index=range(mig))

# In[26]:

for i in range(12):
    _=a.country[a.cluster==i]
    grupe.iloc[0:len(_),i]=a.country[a.cluster==i]

# In[27]:

grupe

# Vidimo da su u prvom klasteru (sa oznakom 0) sve vrlo razvijene države poput Japana, Norveške, SAD itd. U drugom su mahom zemlje latinske Amerike. U drugoj uglavnom srednja i istočna Evropa, itd. Izgleda da klasterizacija ima smisla jer svaki klaster možemo da opišemo koristeći geografske i ekonomske atribute. Ovde se nećemo baviti opisom svakog pojedinačno klastera. To može da ostane za samostalni rad.

# ### Karakteristike klastera

# Nakon gruposanja 159 zemalja u 12 klastera, trebalo bi relativno lako da prepoznamo karakteristike svakog klastera i utvrdimo zašto su države baš na taj način grupisane. Za potrebe prikazivanja klastera i njihovih karakteristika, grupisaćemo vrednosti indikatora po faktorima računajući njihovu srednju vrednost. To će nam pomoći da vidimo po čemu se to faktori razlikuju. Grupisane vrednosti ćemo sačuvati u tabelama __df_cluster__ i __df_cluster_std__.

# In[28]:

df_cluster = df.groupby('cluster').mean()
df_cluster_std = df_std.groupby('cluster').mean()

# Ovako dobijene karakteristike faktora možemo grafički da prikažemo kao _heatmap_. Odavde možemo da vidimo koji indikatori najviše određuju pojedinačne klastere. Obratite pažnju da za ovaj grafički prikaz koristimo standardizovane vrednosti kako indikator sa najvećim rasponom vrednosti (_Secure Internet servers (per 1 million people)_) ne bi obesmislio prikaz za druge indikatore. (Probajte kako bi izgledao sledeći grafikon kada bismo koristili __df_cluster__ umesto __df_cluster_std__.)

# In[29]:

plt.figure(figsize=(20,10))
sns.heatmap(df_cluster_std, cmap="coolwarm", annot=True, fmt= '.2f', linewidths=.2)

# Ako nas interesuju konkretne vrednosti, onda nećemo koristiti standardizovanu tabelu __df_cluster_std__ već __df_cluster__. To možemo da prikažemo i kao _DataFrame_.

# In[30]:

df_cluster

# Odavde možete da vidite mnoštvo karakteristika klastera, npr. da zemlje četvrtog klastera imaju veoma malu dostupnost električne energije i da su im poljoprivreda i ribarstvo veoma važni za BDP. Ostavljamo vam za samostalni rad da prepoznate karakteristike pojedinačnih klastera. Još je bolje da te opise uparite sa geo-ekonomskim opisom država koje čine taj klaster.

# ## Smanjenje dimenzionalnosti

# Faktorska analiza se koristi kao jedan od načina za smanjenje broja dimenzija u skupu podataka sa kojim radimo. To je dobra praksa jer se tako odstranjuju nepotrebne promenljive i izbacuju one koje nisu nezavisne pa se zbog toga dobijaju pouzdaniji rezultati mašinskog učenja. Tehnički razlozi za smanjenje broja dimenzija su takođe veoma važni. Sa manjim skupom podataka se lakše i brže radi. Osim toga, previše promenljivih može da izazove _over-fitting_ efekat gde procene postaju lošije nego što bi bile da imamo manje promenljivih i jednostavniji model. Konačno, klasterska analiza u prostoru koji ima previše dimenzija može da postane neupotrebljiva zbog nedovoljno velikog uzorka za toliki broj dimenzija. Taj efekat se zove "prokletstvo multidimenzionalnosti".

# Mi jesmo tražili rastojanja između tačaka u 29-odimenzionalnom prostoru za potrebe ove demonstracije, ali u opštem slučaju to nije dobra ideja. Smanjenje dimenzionalnosti (eng. _Dimensionality Reduction_) nam otvara mogućnost da radimo klasterizaciju sa manjim brojem promenljivih.

# Faktorska analiza može da predstavi faktore kao linearne kombinacije originalnih promenljivih pa da tako dobijemo numeričku vrednost faktora za svaku stavku u uzorku. Funkcija `transform()` koja je definisana za objekat koji smo kreirali za faktorsku analizu daje vrednosti faktora za svaku zemlju iz tabele __df__.
#
# (Napomena: Pošto smo tabeli __df__ dodali _output_ kolonu __cluster__, ovde ćemo morati da je uklonimo jer nam smeta za funkciju `transform()`. To ćemo učiniti pomoću funkcije `drop()`.)

# In[31]:

dff=pd.DataFrame(fa.transform(df.drop('cluster',axis=1)),index=df.index,columns=faktori)

# Sada možemo da vidimo kolike su procenjene vrednosti (nemerljivih) faktora za koje od ranije imamo opise. Jasno je da su vrednosti faktora "Dostupnost osnovnih konumalnih usluga " značajno veće za Japan i Švedsku nego za Gvajanu i Belize.

# In[32]:

dff

# Pošto su nam prva dva faktora najznačajnija možemo da nacrtamo tačkasti dijagram gde će na x-osi biti prvi faktor, a na y drugi. Svaka tačka bi predstavila jednu zemlju, a njena boja klaster kom pripada. Ukoliko faktori imaju smisla trebalo bi da prepoznamo grupisanje tačaka iz istog klastera.

# In[33]:

prva=faktori[0]
druga=faktori[1]
plt.scatter(x=dff[prva],y=dff[druga],c=df['cluster'],cmap="Set1")
plt.xlabel(prva)
plt.ylabel(druga);

# Pošto tačke ne možemo da prikažemo u sedmodimenzionalnom prostoru, ostaje nam mogućnost da ih prikažemo za dve po dve dimenzije, tj. da nađemo sve parove faktora. Biblioteka __seaborn__ ima funkciju koja može u tome da nam pomogne: `pairplot()`.

# In[34]:

dff['cluster']=df['cluster']

sns.pairplot(dff,kind='scatter',hue='cluster',palette="Set1");

# Sa ovog grafikona vidimo da ne postoji jasna linearna veza između bilo koja dva faktora. To znači da su faktori prilično nezavisni i da smo napravili dobar izbor.

# ## Zaključak
#
# Hijerarhijska klasterizacija je kompleksan algoritam koji postaje previše spor za veliki broj stavki. Ipak, za male i srednje velike skupove podataka predstavlja odličan izbor modela za mašinsko učenje bez nadzora. Paterni koje pronalazimo na ovaj način mogu da se objasne i da omoguće bolji uvid u podatke i pojave koje ih uzrokuju. Dodatna vrednost ovog modela je što može dobro da se vizuelizuje pa samim tima i da se rezultati bolje komuniciraju sa ciljnom grupom.

ext: py

max_stars_repo_name: sverbic/skriptna-obrada-podataka

max_issues_repo_name: sverbic/skriptna-obrada-podataka

max_forks_repo_name: sverbic/skriptna-obrada-podataka


text:
from decimal import Decimal

from typing import Sequence, Dict

import numpy as np

from core.exceptions import TimestampError
from core.logger import get_logger
from model.BaseModel import Bar

class BaseStrategy:

    def __init__(self, symbol, trade_api, array_size=2000):
        self.symbol = symbol
        self.array_size = array_size

        self.bars_1min = {
            'timestamp': np.zeros(self.array_size), 'open': np.zeros(self.array_size), 'high': np.zeros(self.array_size),
            'low': np.zeros(self.array_size), 'close': np.zeros(self.array_size), 'amount': np.zeros(self.array_size)}

        self.bars_5min = {
            'timestamp': np.zeros(self.array_size), 'open': np.zeros(self.array_size), 'high': np.zeros(self.array_size),
            'low': np.zeros(self.array_size), 'close': np.zeros(self.array_size), 'amount': np.zeros(self.array_size)}

        self.bars_15min = {
            'timestamp': np.zeros(self.array_size), 'open': np.zeros(self.array_size), 'high': np.zeros(self.array_size),
            'low': np.zeros(self.array_size), 'close': np.zeros(self.array_size), 'amount': np.zeros(self.array_size)}

        self.bars_30min = {
            'timestamp': np.zeros(self.array_size), 'open': np.zeros(self.array_size), 'high': np.zeros(self.array_size),
            'low': np.zeros(self.array_size), 'close': np.zeros(self.array_size), 'amount': np.zeros(self.array_size)}

        self.bars_60min = {
            'timestamp': np.zeros(self.array_size), 'open': np.zeros(self.array_size), 'high': np.zeros(self.array_size),
            'low': np.zeros(self.array_size), 'close': np.zeros(self.array_size), 'amount': np.zeros(self.array_size)}

        self.orders: Sequence[object] = list()
        self.orders_dict: Dict[str: object] = dict()

        self.trade_api = trade_api

        # logger
        self.logger = get_logger('strategy')

    def init_bars(self, bars, intervals):
        for bar in bars:
            self.update_bars(bar, intervals)

        self.calculate_parameters()

        self.logger.info('history data initialized')

    def update_minute_bars(self, bars, bar, minutes):
        timestamp = bar.timestamp - (bar.timestamp % (minutes*60))

        if len(bars['close']) == 0 or timestamp > bars['timestamp'][-1]:
            bars['timestamp'][:-1] = bars['timestamp'][1:]
            bars['timestamp'][-1] = timestamp

            bars['open'][:-1] = bars['open'][1:]
            bars['open'][-1] = bar.open_price

            bars['high'][:-1] = bars['high'][1:]
            bars['high'][-1] = bar.high_price

            bars['low'][:-1] = bars['low'][1:]
            bars['low'][-1] = bar.low_price

            bars['close'][:-1] = bars['close'][1:]
            bars['close'][-1] = bar.close_price

            bars['amount'][:-1] = bars['amount'][1:]
            bars['amount'][-1] = bar.amount
        elif len(bars['close']) > 0 and timestamp == bars['timestamp'][-1]:
            bars['close'][-1] = bar.close_price
            if bars['high'][-1] < bar.high_price:
                bars['high'][-1] = bar.high_price
            if bars['low'][-1] > bar.low_price:
                bars['low'][-1] = bar.low_price
            bars['amount'][-1] = Decimal(str(bars['amount'][-1])) + Decimal(str(bar.amount))
        else:
            self.logger.error(f"last_bar_timestamp:{bars['timestamp'][-1]}, bar_timestamp:{bar.timestamp}")
            raise TimestampError()

    def calculate_parameters(self):
        pass

    def update_bars(self, bar: Bar, intervals):
        minutes = 0
        if intervals == '1min':
            minutes = 1
        elif intervals == '5min':
            minutes = 5
        elif intervals == '15min':
            minutes = 15
        elif intervals == '60min':
            minutes = 60
        if minutes:
            bar_dict = getattr(self, f'bars_{intervals}')
            self.update_minute_bars(bar_dict, bar, minutes)

    def place_contract_order(self, order: object):
        pass

    def cancel_contract_order(self, order: object):
        pass

    def on_bar(self, bar: Bar):
        pass

    def on_trade(self, response):
        pass

max_stars_repo_name: thlynn/Jquant

max_issues_repo_name: thlynn/Jquant

max_forks_repo_name: thlynn/Jquant

reason: import numpy


text:
# This code is part of Qiskit.
#
# (C) Copyright IBM 2020, 2022.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""The minimum eigensolver factory for ground state calculation algorithms."""

import logging
from typing import Optional, Union, Callable, cast
import numpy as np

from qiskit.algorithms import MinimumEigensolver, VQE
from qiskit.algorithms.optimizers import Optimizer
from qiskit.circuit import QuantumCircuit
from qiskit.opflow import ExpectationBase
from qiskit.opflow.gradients import GradientBase
from qiskit.utils import QuantumInstance
from qiskit_nature.circuit.library import UVCC, UVCCSD, VSCF
from qiskit_nature.converters.second_quantization import QubitConverter
from qiskit_nature.problems.second_quantization.vibrational import (
    VibrationalStructureProblem,
)
from qiskit_nature.properties.second_quantization.vibrational import (
    VibrationalStructureDriverResult,
)
from qiskit_nature.deprecation import deprecate_property, deprecate_positional_arguments

from .minimum_eigensolver_factory import MinimumEigensolverFactory
from ...initial_points import InitialPoint, VSCFInitialPoint

logger = logging.getLogger(__name__)

class VQEUVCCFactory(MinimumEigensolverFactory):
    """Factory to construct a :class:`VQE` minimum eigensolver with :class:`UVCCSD` ansatz
    wavefunction.

    .. note::

       Any ansatz a user might directly set into VQE via the :attr:`minimum_eigensolver` will
       be overwritten by the factory when producing a solver via :meth:`get_solver`. This is
       due to the fact that the factory is designed to manage the ansatz and set it up according
       to the problem. Always pass any custom ansatz to be used when constructing the factory or
       by using its :attr:`ansatz` setter. The following code sample illustrates this behavior:

    .. code-block:: python

        from qiskit_nature.algorithms import VQEUVCCFactory
        from qiskit_nature.circuit.library import UVCCSD, UVCC
        factory = VQEUVCCFactory()
        vqe1 = factory.get_solver(problem, qubit_converter)
        print(type(vqe1.ansatz))  # UVCC()
        # Here the minimum_eigensolver ansatz just gets overwritten
        factory.minimum_eigensolver.ansatz = UVCC()
        vqe2 = factory.get_solver(problem, qubit_converter)
        print(type(vqe2.ansatz))  # UVCCSD
        # Here we change the factory ansatz and thus new VQEs are created with the new ansatz
        factory.ansatz = UVCC()
        vqe3 = factory.get_solver(problem, qubit_converter)
        print(type(vqe3.ansatz))  # UVCC

    """

    @deprecate_positional_arguments(
        version="0.4",
        func_name="VQEUVCCFactory Constructor",
        old_function_arguments=[
            "self",
            "quantum_instance",
            "optimizer",
            "initial_point",
            "gradient",
            "expectation",
            "include_custom",
            "ansatz",
            "initial_state",
            "callback",
        ],
        stack_level=5,
    )
    def __init__(
        self,
        initial_point: Optional[Union[np.ndarray, InitialPoint]] = None,
        ansatz: Optional[UVCC] = None,
        initial_state: Optional[QuantumCircuit] = None,
        **kwargs,
    ) -> None:
        """
        Args:
            initial_point: An optional initial point (i.e., initial parameter values for the VQE
                optimizer). If ``None`` then VQE will use an all-zero initial point of the
                appropriate length computed using
                :class:`~qiskit_nature.algorithms.initial_points.vscf_initial_point.VSCFInitialPoint`.
                This then defaults to the VSCF state when the VSCF circuit is prepended
                to the the ansatz circuit. If another
                :class:`~qiskit_nature.algorithms.initial_points.initial_point.InitialPoint`
                instance, this is used to compute an initial point for the VQE ansatz parameters.
                If a user-provided NumPy array, this is used directly.
            initial_state: Allows specification of a custom `QuantumCircuit` to be used as the
                initial state of the ansatz. If this is never set by the user, the factory will
                default to the :class:`~.VSCF` state.
            ansatz: Allows specification of a custom :class:`~.UCC` instance. This defaults to None
                where the factory will internally create and use a :class:`~.UVCCSD` ansatz.
            kwargs: Remaining keyword arguments are passed to the :class:`VQE`.
        """

        self._initial_state = initial_state
        self._initial_point = initial_point if initial_point is not None else VSCFInitialPoint()
        self._ansatz = ansatz

        self._vqe = VQE(**kwargs)

    @property  # type: ignore
    @deprecate_property("0.4", additional_msg="Use `minimum_eigensolver.quantum_instance` instead.")
    def quantum_instance(self) -> QuantumInstance:
        """DEPRECATED. Use ``minimum_eigensolver.quantum_instance`` instead.
        Returns quantum instance."""
        return self.minimum_eigensolver.quantum_instance

    @quantum_instance.setter  # type: ignore
    @deprecate_property(
        "0.4", additional_msg="Use `minimum_eigensolver.quantum_instance = q_instance` instead."
    )
    def quantum_instance(self, q_instance: QuantumInstance) -> None:
        """DEPRECATED. Use `minimum_eigensolver.quantum_instance` instead. Sets the quantum instance."""
        self.minimum_eigensolver.quantum_instance = q_instance

    @property  # type: ignore
    @deprecate_property("0.4", additional_msg="Use `minimum_eigensolver.optimizer` instead.")
    def optimizer(self) -> Optional[Optimizer]:
        """DEPRECATED. Use ``minimum_eigensolver.optimizer``  instead.
        Returns optimizer."""
        return self.minimum_eigensolver.optimizer

    @optimizer.setter  # type: ignore
    @deprecate_property(
        "0.4", additional_msg="Use `minimum_eigensolver.optimizer = optimizer` instead."
    )
    def optimizer(self, optimizer: Optional[Optimizer]) -> None:
        """DEPRECATED. Use `minimum_eigensolver.optimizer = optimizer` instead. Sets the optimizer."""
        self.minimum_eigensolver.optimizer = optimizer

    @property  # type: ignore
    @deprecate_property("0.4", additional_msg="Use `minimum_eigensolver.gradient` instead.")
    def gradient(self) -> Optional[Union[GradientBase, Callable]]:
        """DEPRECATED. Use ``minimum_eigensolver.gradient`` instead.
        Returns gradient."""
        return self.minimum_eigensolver.gradient

    @gradient.setter  # type: ignore
    @deprecate_property(
        "0.4", additional_msg="Use `minimum_eigensolver.gradient = gradient` instead."
    )
    def gradient(self, gradient: Optional[Union[GradientBase, Callable]]) -> None:
        """DEPRECATED. Use `minimum_eigensolver.gradient = gradient` instead. Sets the gradient."""
        self.minimum_eigensolver.gradient = gradient

    @property  # type: ignore
    @deprecate_property("0.4", additional_msg="Use `minimum_eigensolver.expectation` instead.")
    def expectation(self) -> Optional[ExpectationBase]:
        """DEPRECATED. Use ``minimum_eigensolver.expectation`` instead.
        Returns expectation."""
        return self.minimum_eigensolver.expectation

    @expectation.setter  # type: ignore
    @deprecate_property(
        "0.4", additional_msg="Use `minimum_eigensolver.expectation = expectation` instead."
    )
    def expectation(self, expectation: Optional[ExpectationBase]) -> None:
        """DEPRECATED. Use `minimum_eigensolver.expectation = expectation` instead. Sets expectation."""
        self.minimum_eigensolver.expectation = expectation

    @property  # type: ignore
    @deprecate_property("0.4", additional_msg="Use `minimum_eigensolver.include_custom` instead.")
    def include_custom(self) -> bool:
        """DEPRECATED. Use ``minimum_eigensolver.include_custom`` instead.
        Getter of the ``include_custom`` setting for the ``expectation`` setting."""
        return self.minimum_eigensolver.include_custom

    @include_custom.setter  # type: ignore
    @deprecate_property(
        "0.4", additional_msg="Use `minimum_eigensolver.include_custom = include_custom` instead."
    )
    def include_custom(self, include_custom: bool) -> None:
        """DEPRECATED. Use `minimum_eigensolver.include_custom = include_custom` instead.
        Setter of the ``include_custom``
        setting for the ``expectation`` setting."""
        self.minimum_eigensolver.include_custom = include_custom

    @property
    def ansatz(self) -> Optional[UVCC]:
        """
        Gets the user provided ansatz of future VQEs produced by the factory.
        If value is ``None`` it defaults to :class:`~.UVCCSD`.
        """
        return self._ansatz

    @ansatz.setter
    def ansatz(self, ansatz: Optional[UVCC]) -> None:
        """
        Sets the ansatz of future VQEs produced by the factory.
        If set to ``None`` it defaults to :class:`~.UVCCSD`.
        """
        self._ansatz = ansatz

    @property
    def initial_state(self) -> Optional[QuantumCircuit]:
        """Getter of the initial state."""
        return self._initial_state

    @initial_state.setter
    def initial_state(self, initial_state: Optional[QuantumCircuit]) -> None:
        """
        Setter of the initial state.
        If ``None`` is passed, this factory will default to using the :class:`~.VSCF`.
        """
        self._initial_state = initial_state

    @property
    def initial_point(self) -> Optional[Union[np.ndarray, InitialPoint]]:
        """
        Gets the initial point of future VQEs produced by the factory.
        """
        return self._initial_point

    @initial_point.setter
    def initial_point(self, initial_point: Optional[Union[np.ndarray, InitialPoint]]) -> None:
        """Sets the initial point of future VQEs produced by the factory."""
        self._initial_point = initial_point

    @property  # type: ignore
    @deprecate_property("0.4", additional_msg="Use `minimum_eigensolver.callback` instead.")
    def callback(self) -> Optional[Callable[[int, np.ndarray, float, float], None]]:
        """Returns the callback."""
        return self.minimum_eigensolver.callback

    @callback.setter  # type: ignore
    @deprecate_property("0.4", additional_msg="Use the constructor instead.")
    def callback(self, callback: Optional[Callable[[int, np.ndarray, float, float], None]]) -> None:
        """DEPRECATED. Use the constructor instead.
        Sets the callback."""
        self.minimum_eigensolver.callback = callback

    def get_solver(  # type: ignore[override]
        self,
        problem: VibrationalStructureProblem,
        qubit_converter: QubitConverter,
    ) -> MinimumEigensolver:
        """Returns a VQE with a :class:`~.UVCCSD` wavefunction ansatz, based on ``qubit_converter``.

        Args:
            problem: a class encoding a problem to be solved.
            qubit_converter: a class that converts second quantized operator to qubit operator
                             according to a mapper it is initialized with.

        Returns:
            A VQE suitable to compute the ground state of the molecule.
        """

        basis = cast(VibrationalStructureDriverResult, problem.grouped_property_transformed).basis
        num_modals = basis.num_modals_per_mode
        num_modes = len(num_modals)

        if isinstance(num_modals, int):
            num_modals = [num_modals] * num_modes

        initial_state = self.initial_state
        if initial_state is None:
            initial_state = VSCF(num_modals)

        ansatz = self._ansatz
        if ansatz is None:
            ansatz = UVCCSD()
        ansatz.qubit_converter = qubit_converter
        ansatz.num_modals = num_modals
        ansatz.initial_state = initial_state

        if isinstance(self.initial_point, InitialPoint):
            self.initial_point.ansatz = ansatz
            initial_point = self.initial_point.to_numpy_array()
        else:
            initial_point = self.initial_point

        self.minimum_eigensolver.initial_point = initial_point
        self.minimum_eigensolver.ansatz = ansatz
        return self.minimum_eigensolver

    def supports_aux_operators(self):
        return VQE.supports_aux_operators()

    @property
    def minimum_eigensolver(self) -> VQE:
        """Returns the solver instance."""
        return self._vqe

max_stars_repo_name: kevinsung/qiskit-nature

max_issues_repo_name: kevinsung/qiskit-nature

max_forks_repo_name: kevinsung/qiskit-nature


text:
"""
Mask R-CNN
Train on the nuclei segmentation dataset from the
Kaggle 2018 Data Science Bowl
https://www.kaggle.com/c/data-science-bowl-2018/

Licensed under the MIT License (see LICENSE for details)
Written by Waleed Abdulla

------------------------------------------------------------

Usage: import the module (see Jupyter notebooks for examples), or run from
       the command line as such:

    # Train a new model starting from ImageNet weights
    python train.py --weight imagenet --logs PATH/TO/LOGS --dataset PATH/TO/DATASET --subset train

"""

import os
# os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
# os.environ["CUDA_VISIBLE_DEVICES"]="2"
import sys
import json
import datetime
import time
import numpy as np
import random
import skimage.io
from imgaug import augmenters as iaa
import numexpr as ne

import pandas as pd
import pygsheets

# Import Mask RCNN
# MASKRCNN_CODE_PATH = "/home/jimmy15923/model_zoo/keras_maskrcnn/"
# sys.path.append(MASKRCNN_CODE_PATH)  # To find local version of the library

from mrcnn import utils
from mrcnn import model as modellib
from mrcnn.model import *
import monuconfig

# Path to trained weights file

## download path to coco.h5 from https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5

COCO_WEIGHTS_PATH = "PATH/TO/COCO.h5"

# Directory to save logs and model checkpoints, if not provided
# through the command line argument --logs
DEFAULT_LOGS_DIR = 'logs'

# set seed for same validation split
seed = 7
IMAGE_IDS = next(os.walk('dataset/train'))[1]
random.seed(seed)
random.shuffle(IMAGE_IDS)
VAL_IMAGE_IDS = ['TCGA-G9-6363-01Z-00-DX1', 'TCGA-B0-5711-01Z-00-DX1']

############################################################
#  Dataset
############################################################

class MonusegDataset(utils.Dataset):

    def load_monuseg(self, dataset_dir, subset):
        """Load a subset of the monuseg dataset.

        dataset_dir: Root directory of the dataset
        subset: Subset to load. Either the name of the sub-directory,
                such as stage1_train, stage1_test, ...etc. or, one of:
                * train: stage1_train excluding validation images
                * val: validation images from VAL_IMAGE_IDS
        """
        # Add classes. We have one class.
        # Naming the dataset nucleus, and the class nucleus
        self.add_class("nuclear", 1, "nuclear")

        # Which subset?
        # "val": use hard-coded list above
        # "train": use data from stage1_train minus the hard-coded list above
        # else: use the data from the specified sub-directory
        assert subset in ["train", "val", "stage1_train", "stage1_test", "stage2_test"]
        subset_dir = "train" if subset in ["train", "val"] else subset
        dataset_dir = os.path.join(dataset_dir, subset_dir)
        if subset == "val":
            image_ids = VAL_IMAGE_IDS
        else:
            # Get image ids from directory names
            image_ids = next(os.walk(dataset_dir))[1]
            if subset == "train":
                image_ids = list(set(image_ids) - set(VAL_IMAGE_IDS))

        # Add images
        for image_id in image_ids:
            self.add_image(
                "nuclear",
                image_id=image_id,
                path=os.path.join(dataset_dir, image_id, "images/{}.png".format(image_id)))

    def load_mask(self, image_id):
        """Generate instance masks for an image.
       Returns:
        masks: A bool array of shape [height, width, instance count] with
            one mask per instance.
        class_ids: a 1D array of class IDs of the instance masks.
        """
        info = self.image_info[image_id]
        # Get mask directory from image path
        mask_dir = os.path.join(os.path.dirname(os.path.dirname(info['path'])), "masks")

        # Read mask files from .png image
        mask = []
        for f in next(os.walk(mask_dir))[2]:
            if f.endswith(".png"):
                m = skimage.io.imread(os.path.join(mask_dir, f)).astype(np.bool)
                mask.append(m)
        mask = np.stack(mask, axis=-1)
        # Return mask, and array of class IDs of each instance. Since we have
        # one class ID, we return an array of ones
        return mask, np.ones([mask.shape[-1]], dtype=np.int32)

    def read_data_and_mask_arr(self):
        print("Reading data array & mask array from disk")
        self.image_arr = np.array([self.load_image(x) for x in self._image_ids])

        # Since maks has different depth in each image, we store mask arr in dict
        self.mask_arr = {}
        for i in self._image_ids:
            self.mask_arr[i] = self.load_mask(i)[0]

    def load_mask_arr(self, mask_arr, image_id):
        """Get instance masks from image array
        Returns:
        masks: A bool array of shape [height, width, instance count] with
            one mask per instance.
        class_ids: a 1D array of class IDs of the instance masks.
        """
        mask = self.mask_arr[image_id]
        return mask, np.ones([mask.shape[-1]], dtype=np.int32)

    def image_reference(self, image_id):
        """Return the path of the image."""
        info = self.image_info[image_id]
        if info["source"] == "nuclear":
            return info["id"]
        else:
            super(self.__class__, self).image_reference(image_id)

############################################################
#  Configurations
############################################################

# set checkpoint name
import datetime
name = str(datetime.datetime.now())
tmp = name.split(" ")
ckpt_name = tmp[0] + "_" + tmp[1][:8]
ckpt_name = ckpt_name.replace(":", "-")

# set training config and config NAME
config = monuconfig.Config()
config.NAME = config.NAME + "_" + ckpt_name

# set inference config
infer_config = monuconfig.InferConfig()
infer_config.NAME = ckpt_name

############################################################
#  Training
############################################################

def train(model, dataset_dir, subset):
    """Train the model."""
    # Training dataset.
    dataset_train = MonusegDataset()
    dataset_train.load_monuseg(dataset_dir, subset)
    dataset_train.prepare()
    dataset_train.read_data_and_mask_arr()

    # Validation dataset
    dataset_val = MonusegDataset()
    dataset_val.load_monuseg(dataset_dir, "val")
    dataset_val.prepare()
    dataset_val.read_data_and_mask_arr()

    # Image augmentation
    # http://imgaug.readthedocs.io/en/latest/source/augmenters.html
    def image_channel_suffle(image):
        img_ = image.copy()
        r, g, b = img_[:,:,0], img_[:,:,1], img_[:,:,2]
        idx = [0, 1, 2]
        np.random.shuffle(idx)
        img_[:,:,idx[0]], img_[:,:,idx[1]], img_[:,:,idx[2]] = r, g, b
        return img_

    def img_func(images, random_state, parents, hooks):
        for img in images:
            img = image_channel_suffle(img)
        return images

    def keypoint_func(keypoints_on_images, random_state, parents, hooks):
        return keypoints_on_images

    augmentation = iaa.SomeOf((2, 5), [
        iaa.Fliplr(0.5),
        iaa.Flipud(0.5),
        iaa.OneOf([iaa.Affine(rotate=90),
                   iaa.Affine(rotate=180),
                   iaa.Affine(rotate=270)]),
        iaa.Affine(scale=(0.6, 2)),
        iaa.Lambda(img_func, keypoint_func),
        iaa.Multiply((0.8, 1.5)),
        iaa.GaussianBlur(sigma=(0.0, 2.0))
    ])

    # *** This training schedule is an example. Update to your needs ***

    # If starting from imagenet, train heads only for a bit
    # since they have random weights
    print("Train network heads")
    model.train(dataset_train, dataset_val,
                learning_rate=config.LEARNING_RATE,
                epochs=1,
                augmentation=augmentation,
                layers='heads')

    print("Train all layers")
    model.train(dataset_train, dataset_val,
                learning_rate=3*10e-5,
                epochs=1,
                augmentation=augmentation,
                layers='all')

def revise_predict_mask(mask, pred):
    mask = mask[12:1012, 12:1012, :]
    pred = pred[12:1012, 12:1012, :]

    prediction = np.zeros((1000,1000))
    for i in range(pred.shape[2]):
        prediction[pred[:,:,i]] = i + 1
        prediction = prediction.astype('uint16')
    return mask, prediction

# fast version of Aggregated Jaccrd Index
def agg_jc_index(mask, pred):

    """Calculate aggregated jaccard index for prediction & GT mask
    reference paper here: https://www.dropbox.com/s/j3154xgkkpkri9w/IEEE_TMI_NuceliSegmentation.pdf?dl=0
    mask: Ground truth mask, shape = [1000, 1000, instances]
    pred: Prediction mask, shape = [1000,1000], dtype = uint16, each number represent one instance
    Returns: Aggregated Jaccard index for GT & mask
    """
    def compute_iou(m, pred, pred_mark_isused, idx_pred):
        # check the prediction has been used or not
        if pred_mark_isused[idx_pred]:
            intersect = 0
            union = np.count_nonzero(m)
        else:
            p = (pred == idx_pred)
            # replace multiply with bool operation
            s = ne.evaluate("m&p")
            intersect = np.count_nonzero(s)
            union = np.count_nonzero(m) + np.count_nonzero(p) - intersect
        return (intersect, union)

    mask = mask.astype(np.bool)
    c = 0 # count intersection
    u = 0 # count union
    pred_instance = pred.max() # predcition instance number
    pred_mark_used = [] # mask used
    pred_mark_isused = np.zeros((pred_instance+1), dtype=bool)

    for idx_m in range(len(mask[0,0,:])):
        m = np.take(mask, idx_m, axis=2)

        intersect_list, union_list = zip(*[compute_iou(m, pred, pred_mark_isused, idx_pred) for idx_pred in range(1, pred_instance+1)])

        iou_list = np.array(intersect_list) / np.array(union_list)
        hit_idx = np.argmax(iou_list)
        c += intersect_list[hit_idx]
        u += union_list[hit_idx]
        pred_mark_used.append(hit_idx)
        pred_mark_isused[hit_idx+1] = True

    pred_mark_used = [x+1 for x in pred_mark_used]
    pred_fp = set(np.unique(pred)) - {0} - set(pred_mark_used)
    pred_fp_pixel = np.sum([np.sum(pred==i) for i in pred_fp])

    u += pred_fp_pixel
    return (c / u)

def evaluate(model, dataset_dir, model_path, config):
    print("EVALUATING...")
    dataset_val = MonusegDataset()
    dataset_val.load_monuseg(dataset_dir, "val")
    dataset_val.prepare()
    dataset_val.read_data_and_mask_arr()

    model.load_weights(model_path, by_name=True)

    ajx = []
    for i in dataset_val.image_ids:
        image, image_meta, gt_class_id, gt_bbox, gt_mask =\
        modellib.load_image_gt(dataset_val, config, image_id=i, use_mini_mask=False, load_from_array=True)

        # Run object detection
        results = model.detect_molded(np.expand_dims(image, 0), np.expand_dims(image_meta, 0), verbose=1)
        r = results[0]

        # get resized mask & prediction from (1024, 1024) to (1000, 1000)
        y_true, y_pred = revise_predict_mask(gt_mask, r['masks'])

        # calculate the jc index
        aj_index = agg_jc_index(y_true, y_pred)
        ajx.append(aj_index)
    return np.array(ajx)

############################################################
#  Command Line
############################################################

if __name__ == '__main__':
    import argparse

    # Parse command line arguments
    parser = argparse.ArgumentParser(
        description='Mask R-CNN for nuclei counting and segmentation')

    parser.add_argument('--dataset', required=False,
                        default="dataset/",
                        metavar="/path/to/dataset/",
                        help='Root directory of the dataset')
    parser.add_argument('--weights', required=True,
                        metavar="/path/to/weights.h5",
                        help="Path to weights .h5 file or 'coco'")
    parser.add_argument('--logs', required=False,
                        default=DEFAULT_LOGS_DIR,
                        metavar="/path/to/logs/",
                        help='Logs and checkpoints directory (default=logs/)')
    parser.add_argument('--subset', required=False,
                        default="train",
                        metavar="Dataset sub-directory",
                        help="Subset of dataset to run prediction on")
#     parser.add_argument('--model_use', required=False,
#                         default="panet"
#                         metavar="Dataset sub-directory",
#                         help="Model structure used, choose 'panet' or 'maskrcnn' ")
    args = parser.parse_args()

    print("Weights: ", args.weights)
    print("Dataset: ", args.dataset)
    if args.subset:
        print("Subset: ", args.subset)
    print("Logs: ", args.logs)

    # Check config setting
    config.display()

    # Create model
    model = modellib.MaskRCNN(mode="training", config=config,
                                  model_dir=args.logs)

    # Select weights file to load
    if args.weights.lower() == "coco":
        weights_path = COCO_WEIGHTS_PATH
        # Download weights file
        if not os.path.exists(weights_path):
            utils.download_trained_weights(weights_path)
    elif args.weights.lower() == "last":
        # Find last trained weights
        weights_path = model.find_last()
    elif args.weights.lower() == "imagenet":
        # Start from ImageNet trained weights
        weights_path = model.get_imagenet_weights()
    else:
        weights_path = args.weights

    # Load weights
    print("Loading weights ", weights_path)
    if args.weights.lower() == "coco":
        # Exclude the last layers because they require a matching
        # number of classes
        model.load_weights(weights_path, by_name=True, exclude=[
            "mrcnn_class_logits", "mrcnn_bbox_fc",
            "mrcnn_bbox", "mrcnn_mask"])
    else:
        model.load_weights(weights_path, by_name=True)

    # Train or evaluate
    T = time.time()
    train(model, args.dataset, args.subset)

    ## after training, run AJI evaluation
    model_infer = modellib.MaskRCNN(mode="inference", config=infer_config,
                                  model_dir=args.logs)

    trained_weights_path = model.checkpoint_path
    config.model_path = model.checkpoint_path
    print("LOADING WEIGHTS FROM {}".format(trained_weights_path))

    ajx_result = evaluate(model_infer, args.dataset, trained_weights_path, infer_config)
    print("MEAN AJI is {}".format(np.mean(ajx_result)))

    config.AJI_score = list(ajx_result)
    T2 = time.time()
    print("TOTAL TRAINING TIME: {} hours".format((T2-T)/3600))

    ## writing training parameter into google sheets

    path = 'miccai_gsheet.json'
    gc = pygsheets.authorize(service_file=path)
    sh = gc.open("miccai_maskrcnn")
    # get first sheet
    test = sh[0]
    # add columns
    test.add_cols(1)
    # get number of cols
    n_cols = test.cols

    # build config DataFrame
    config_list = []
    for a in dir(config):
        if not a.startswith("__") and not callable(getattr(config, a)):
            config_list.append([getattr(config, a)])

    df = pd.DataFrame(config_list)
    df[0] = df[0].apply(str)

    import string
    idx_dict = {}
    for i in range(1, 27):
        idx_dict[i] = string.ascii_uppercase[i-1]

    # append dataframe in new columns
    test.set_dataframe(df, "{}1".format(idx_dict[n_cols]))

    import pickle
    # save inference config for replication of performance
    with open('logs/{}/inference_config_{}.pickle'.format(config.NAME, config.NAME), 'wb') as f:
        pickle.dump(infer_config, f, -1)

reason: import numpy,import numexpr


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 12 11:17:27 2020

@author: hiroyasu
"""

import pandas as pd
import torch
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
import random
from sklearn.ensemble import AdaBoostClassifier
from sklearn.decomposition import PCA

from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D

df_train = pd.read_csv('data/train.csv', index_col=0)
df_test = pd.read_csv('data/test.csv', index_col=0)
'''
x_cols = {
    'series_id': np.uint32,
    'last_price': np.float32,
    'mid': np.float32,
    'opened_position_qty': np.float32,
    'closed_position_qty': np.float32,
    'transacted_qty': np.float32,
    'bid1': np.float32,
    'bid2': np.float32,
    'bid3': np.float32,
    'bid4': np.float32,
    'bid5': np.float32,
    'ask1': np.float32,
    'ask2': np.float32,
    'ask3': np.float32,
    'ask4': np.float32,
    'ask5': np.float32,
    'bid1vol': np.float32,
    'bid2vol': np.float32,
    'bid3vol': np.float32,
    'bid4vol': np.float32,
    'bid5vol': np.float32,
    'ask1vol': np.float32,
    'ask2vol': np.float32,
    'ask3vol': np.float32,
    'ask4vol': np.float32,
    'ask5vol': np.float32
}
'''
df_train = df_train.fillna(method='ffill')
df_train = df_train.fillna(method='bfill')
df_test = df_test.fillna(method='ffill')
df_test = df_test.fillna(method='bfill')
n = (df_train.values).shape[1]-1
n = 4
nt = (df_test.values).shape[1]-1
Xall = df_train.values[:,0:26]
Yall = df_train.values[:,26]
Xtest = df_test.values[:,0:26]
n_input = n # rnn input size
n_hidden = 128 # number of rnn hidden units
n_output = 1 # rnn input size
n_layers = 2 # rnn number of layers

# truncate and pad input sequences

class RNN(nn.Module):
    def __init__(self,n_input,n_hidden,n_output,n_layers):
        super(RNN, self).__init__()
        self.rnn = nn.LSTM(
                input_size=n_input,
                hidden_size=n_hidden, # rnn hidden unit
                num_layers=n_layers, # number of rnn layer
                batch_first=True, # input & output will has batch size as 1s dimension. e.g. (batch, time_step, input_size)
                )
        self.out1 = nn.Linear(n_hidden,n_output)
        #self.out2 = nn.Sigmoid()
        self.n_layers = n_layers
        self.n_hidden = n_hidden

    def forward(self,x):
        # x (batch, time_step, input_size)
        # h_state (n_layers, batch, hidden_size)
        # r_out (batch, time_step, hidden_size)
        r_out,_ = self.rnn(x)
        out = self.out1(r_out)
        #out = self.out2(out)
        return out

def Np2Var(X):
    X = X.astype(np.float32)
    X = torch.from_numpy(X)
    return X

def RandomizeTrainData(Xtrain,Ytrain,time_step):
    # Note that training data = original data - test data
    N = Ytrain.size
    Ntime = N/time_step
    Ntime = int(Ntime)
    XtrainR = np.zeros((Ntime,time_step,26))
    YtrainR = np.zeros((Ntime,time_step,1))
    for j in range(Ntime):
        XtrainR[j,:,:] = Xtrain[(time_step)*j:(time_step)*(j+1),:]
        YtrainR[j,:,:] = np.array([Ytrain[(time_step)*j:(time_step)*(j+1)]]).T
    idx = np.random.permutation(Ntime)
    XtrainR = XtrainR[idx,:,:]
    YtrainR = YtrainR[idx,:,:]
    return XtrainR,YtrainR

def preprocessX(X):
    n = X.shape[1]
    Xm = np.mean(X,axis=0)
    Xsig = np.std(X,axis=0)
    Xnor = (X-Xm)/Xsig
    return Xnor,Xm

def get_model(dropout_prob):
    model = nn.Sequential(
    nn.Linear(n,333),
    nn.ReLU(),
    nn.Dropout(dropout_prob),
    nn.Linear(333,333),
    nn.ReLU(),
    nn.Dropout(dropout_prob),
    nn.Linear(333,333),
    nn.ReLU(),
    nn.Dropout(dropout_prob),
    nn.Linear(333,1)
    )
    return model

def train_model(model,Xtrain,Ytrain,Xval,Yval,n_epochs,BatchSize):
    # For a multi-class classification problem
    criterion = nn.BCEWithLogitsLoss()
    optimizer = torch.optim.RMSprop(model.parameters())
    # Train the model for 1 epoch, iterating on the data in batches
    N = Ytrain.size/BatchSize
    N = int(N)
    Xval = Np2Var(Xval)
    Yval = Np2Var(np.array([Yval])).T
    for epoch in range(n_epochs):
        print(f'Epoch {epoch+1}/10:', end='')
        # train
        model.train()
        for epoch in range(n_epochs):
            for i in range(N):
                X = Xtrain[BatchSize*i:BatchSize*(i+1),:]
                Y = Ytrain[BatchSize*i:BatchSize*(i+1)]
                Y = np.array([Y]).T
                X = Np2Var(X)
                Y = Np2Var(Y)
                # forward pass
                output = model(X)
                # calculate categorical cross entropy loss
                loss = criterion(output,Y)
                optimizer.zero_grad() # clear gradients for this training step
                loss.backward() # backpropagation, compute gradients
                optimizer.step() # apply gradients
            Ypred = model(Xval)
            loss_test = criterion(Ypred,Yval)
            print(loss_test)
        return model,loss_test

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

if __name__ == "__main__":
    Xall,XC = preprocessX(Xall)
    Xtest,XCt = preprocessX(Xtest)
    pca = PCA(n_components=4)
    pca.fit(Xall)
    comp = pca.components_
    Xall = pca.transform(Xall)
    Xtest = pca.transform(Xtest)
    lam = pca.explained_variance_ratio_
    print(lam)
    #importances = model.feature_importances_
    #model = AdaBoostClassifier(n_estimators=1000).fit(Xall,Yall)
    #np.save('data/params/importances1000.npy',importances)
    #importances = np.load('data/params/importances1000.npy')
    time_step = 10 # rnn time step
    rnn = RNN(n_input,n_hidden,n_output,n_layers)
    LR = 0.2 # learning rate
    N = np.size(Xall,0)
    BatchSize = np.floor(N/100)
    BatchSize = BatchSize.astype(np.int32)
    ValSize = 1000-1
    Xval = Xall[0:ValSize+1,:]
    Yval = Yall[0:ValSize+1]
    Xtrain0 = Xall[ValSize+1:,:]
    Ytrain0 = Yall[ValSize+1:]
    model = get_model(0.1)
    model,loss = train_model(model,Xtrain0,Ytrain0,Xval,Yval,100,BatchSize)
    df_test['Predicted'] = sigmoid(model(Np2Var(Xtest)).data.numpy())
    df_test[['Predicted']].to_csv('data/submission.csv')


text:
#!/usr/bin/env python3

# Copyright (c) 2021 CINN Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
import math
import numpy as np
import cinn
from cinn import frontend
from cinn import runtime
from cinn import lang
from cinn import framework
from cinn import ir
from cinn import common
from cinn.poly import create_stages
import logging
from test_utils import SingleOpTester

class OpTest_add_0(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X, Y] = inputs_data
        return X + Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axis", 0)
        self.to_test_op([[100, 32], [100, 32]], [[100, 32]], "elementwise_add",
                        attrs)

class OpTest_add_1(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X, Y] = inputs_data
        return X + Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axis", 1)
        self.to_test_op([[3, 2], [2]], [[3, 2]], "elementwise_add", attrs)

class OpTest_mul_0(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X, Y] = inputs_data
        return X * Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axis", 0)
        self.to_test_op([[100, 32], [100, 32]], [[100, 32]], "elementwise_mul",
                        attrs)

class OpTest_mul_1(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X, Y] = inputs_data
        return X * Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axis", 1)
        self.to_test_op([[3, 2], [2]], [[3, 2]], "elementwise_mul", attrs)

class OpTest_scale_0(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        return X * attrs.attr_store["scale"] + attrs.attr_store["bias"]

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("scale", 0.7)
        attrs.set_attr("bias", 0.3)
        self.to_test_op([[100, 32]], [[100, 32]], "scale", attrs)

class OpTest_scale_1(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        return (X + attrs.attr_store["bias"]) * attrs.attr_store["scale"]

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("scale", 0.6)
        attrs.set_attr("bias", 0.4)
        attrs.set_attr("bias_after_scale", False)
        self.to_test_op([[100, 32]], [[100, 32]], "scale", attrs)

if __name__ == "__main__":
    unittest.main()