text:
"""Unit tests for corpus functionality in Annif"""

import gzip
import numpy as np
import annif.corpus
from annif.corpus import TransformingDocumentCorpus

def test_subjectset_uris():
    data = """<http://example.org/dummy>\tdummy
    <http://example.org/another>\tanother
    """

    sset = annif.corpus.SubjectSet.from_string(data)
    assert sset.has_uris()
    assert len(sset.subject_uris) == 2
    assert "http://example.org/dummy" in sset.subject_uris
    assert "http://example.org/another" in sset.subject_uris

def test_subjectset_labels():
    data = """dummy
    another
    """

    sset = annif.corpus.SubjectSet.from_string(data)
    assert not sset.has_uris()
    assert len(sset.subject_labels) == 2
    assert "dummy" in sset.subject_labels
    assert "another" in sset.subject_labels

def test_subjectset_from_tuple():
    uris = ['http://www.yso.fi/onto/yso/p10849',
            'http://www.yso.fi/onto/yso/p19740']
    labels = ['arkeologit', 'obeliskit']
    sset = annif.corpus.SubjectSet((uris, labels))
    assert sset.has_uris()
    assert len(sset.subject_uris) == 2
    assert 'http://www.yso.fi/onto/yso/p10849' in sset.subject_uris
    assert 'http://www.yso.fi/onto/yso/p19740' in sset.subject_uris

def test_subjectset_as_vector(subject_index):
    uris = ['http://www.yso.fi/onto/yso/p10849', 'http://example.org/unknown']
    labels = ['arkeologit', 'unknown-subject']
    sset = annif.corpus.SubjectSet((uris, labels))
    vector = sset.as_vector(subject_index)
    assert vector.sum() == 1  # only one known subject

def test_subjectset_as_vector_destination(subject_index):
    uris = ['http://www.yso.fi/onto/yso/p10849', 'http://example.org/unknown']
    labels = ['arkeologit', 'unknown-subject']
    sset = annif.corpus.SubjectSet((uris, labels))
    destination = np.zeros(len(subject_index), dtype=np.float32)
    vector = sset.as_vector(subject_index, destination=destination)
    assert vector.sum() == 1  # only one known subject
    assert vector is destination

def test_docdir_key(tmpdir):
    tmpdir.join('doc1.txt').write('doc1')
    tmpdir.join('doc1.key').write('key1')
    tmpdir.join('doc2.txt').write('doc2')
    tmpdir.join('doc2.key').write('key2')
    tmpdir.join('doc3.txt').write('doc3')

    docdir = annif.corpus.DocumentDirectory(str(tmpdir))
    files = sorted(list(docdir))
    assert len(files) == 3
    assert files[0][0] == str(tmpdir.join('doc1.txt'))
    assert files[0][1] == str(tmpdir.join('doc1.key'))
    assert files[1][0] == str(tmpdir.join('doc2.txt'))
    assert files[1][1] == str(tmpdir.join('doc2.key'))
    assert files[2][0] == str(tmpdir.join('doc3.txt'))
    assert files[2][1] is None

def test_docdir_tsv(tmpdir):
    tmpdir.join('doc1.txt').write('doc1')
    tmpdir.join('doc1.tsv').write('<http://example.org/key1>\tkey1')
    tmpdir.join('doc2.txt').write('doc2')
    tmpdir.join('doc2.tsv').write('<http://example.org/key2>\tkey2')
    tmpdir.join('doc3.txt').write('doc3')

    docdir = annif.corpus.DocumentDirectory(str(tmpdir))
    files = sorted(list(docdir))
    assert len(files) == 3
    assert files[0][0] == str(tmpdir.join('doc1.txt'))
    assert files[0][1] == str(tmpdir.join('doc1.tsv'))
    assert files[1][0] == str(tmpdir.join('doc2.txt'))
    assert files[1][1] == str(tmpdir.join('doc2.tsv'))
    assert files[2][0] == str(tmpdir.join('doc3.txt'))
    assert files[2][1] is None

def test_docdir_tsv_bom(tmpdir):
    tmpdir.join('doc1.txt').write('doc1'.encode('utf-8-sig'))
    tmpdir.join('doc1.tsv').write(
        '<http://example.org/key1>\tkey1'.encode('utf-8-sig'))
    tmpdir.join('doc2.txt').write('doc2'.encode('utf-8-sig'))
    tmpdir.join('doc2.tsv').write(
        '<http://example.org/key2>\tkey2'.encode('utf-8-sig'))

    docdir = annif.corpus.DocumentDirectory(str(tmpdir))
    docs = list(docdir.documents)
    assert docs[0].text == 'doc1'
    assert list(docs[0].uris)[0] == 'http://example.org/key1'
    assert list(docs[0].labels)[0] == 'key1'
    assert docs[1].text == 'doc2'
    assert list(docs[1].uris)[0] == 'http://example.org/key2'
    assert list(docs[1].labels)[0] == 'key2'

def test_docdir_key_require_subjects(tmpdir):
    tmpdir.join('doc1.txt').write('doc1')
    tmpdir.join('doc1.key').write('<http://example.org/key1>\tkey1')
    tmpdir.join('doc2.txt').write('doc2')
    tmpdir.join('doc2.key').write('<http://example.org/key2>\tkey2')
    tmpdir.join('doc3.txt').write('doc3')

    docdir = annif.corpus.DocumentDirectory(str(tmpdir), require_subjects=True)
    files = sorted(list(docdir))
    assert len(files) == 2
    assert files[0][0] == str(tmpdir.join('doc1.txt'))
    assert files[0][1] == str(tmpdir.join('doc1.key'))
    assert files[1][0] == str(tmpdir.join('doc2.txt'))
    assert files[1][1] == str(tmpdir.join('doc2.key'))

def test_docdir_tsv_require_subjects(tmpdir):
    tmpdir.join('doc1.txt').write('doc1')
    tmpdir.join('doc1.tsv').write('key1')
    tmpdir.join('doc2.txt').write('doc2')
    tmpdir.join('doc2.tsv').write('key2')
    tmpdir.join('doc3.txt').write('doc3')

    docdir = annif.corpus.DocumentDirectory(str(tmpdir), require_subjects=True)
    files = sorted(list(docdir))
    assert len(files) == 2
    assert files[0][0] == str(tmpdir.join('doc1.txt'))
    assert files[0][1] == str(tmpdir.join('doc1.tsv'))
    assert files[1][0] == str(tmpdir.join('doc2.txt'))
    assert files[1][1] == str(tmpdir.join('doc2.tsv'))

def test_docdir_tsv_as_doccorpus(tmpdir):
    tmpdir.join('doc1.txt').write('doc1')
    tmpdir.join('doc1.tsv').write('<http://example.org/subj1>\tsubj1')
    tmpdir.join('doc2.txt').write('doc2')
    tmpdir.join('doc2.tsv').write('<http://example.org/subj2>\tsubj2')
    tmpdir.join('doc3.txt').write('doc3')

    docdir = annif.corpus.DocumentDirectory(str(tmpdir), require_subjects=True)
    docs = list(docdir.documents)
    assert len(docs) == 2
    assert docs[0].text == 'doc1'
    assert docs[0].uris == {'http://example.org/subj1'}
    assert docs[1].text == 'doc2'
    assert docs[1].uris == {'http://example.org/subj2'}

def test_docdir_key_as_doccorpus(tmpdir, subject_index):
    tmpdir.join('doc1.txt').write('doc1')
    tmpdir.join('doc1.key').write('arkeologit')
    tmpdir.join('doc2.txt').write('doc2')
    tmpdir.join('doc2.key').write('kalliotaide')
    tmpdir.join('doc3.txt').write('doc3')

    docdir = annif.corpus.DocumentDirectory(str(tmpdir), require_subjects=True)
    docdir.set_subject_index(subject_index)
    docs = list(docdir.documents)
    assert len(docs) == 2
    assert docs[0].text == 'doc1'
    assert docs[0].uris == {'http://www.yso.fi/onto/yso/p10849'}
    assert docs[1].text == 'doc2'
    assert docs[1].uris == {'http://www.yso.fi/onto/yso/p13027'}

def test_subject_by_uri(subject_index):
    subj_id = subject_index.by_uri('http://www.yso.fi/onto/yso/p7141')
    assert subject_index[subj_id][1] == 'sinetit'

def test_subject_by_uri_missing(subject_index):
    subj_id = subject_index.by_uri('http://nonexistent')
    assert subj_id is None

def test_subject_by_label(subject_index):
    subj_id = subject_index.by_label('sinetit')
    assert subject_index[subj_id][0] == 'http://www.yso.fi/onto/yso/p7141'

def test_subject_by_label_missing(subject_index):
    subj_id = subject_index.by_label('nonexistent')
    assert subj_id is None

def test_docfile_plain(tmpdir):
    docfile = tmpdir.join('documents.tsv')
    docfile.write("""Läntinen\t<http://www.yso.fi/onto/yso/p2557>
        Oulunlinnan\t<http://www.yso.fi/onto/yso/p7346>
        Harald Hirmuinen\t<http://www.yso.fi/onto/yso/p6479>""")

    docs = annif.corpus.DocumentFile(str(docfile))
    assert len(list(docs.documents)) == 3

def test_docfile_bom(tmpdir):
    docfile = tmpdir.join('documents_bom.tsv')
    data = """Läntinen\t<http://www.yso.fi/onto/yso/p2557>
        Oulunlinnan\t<http://www.yso.fi/onto/yso/p7346>
        Harald Hirmuinen\t<http://www.yso.fi/onto/yso/p6479>"""
    docfile.write(data.encode('utf-8-sig'))

    docs = annif.corpus.DocumentFile(str(docfile))
    firstdoc = next(docs.documents)
    assert firstdoc.text.startswith("Läntinen")

def test_docfile_plain_invalid_lines(tmpdir, caplog):
    logger = annif.logger
    logger.propagate = True
    docfile = tmpdir.join('documents_invalid.tsv')
    docfile.write("""Läntinen\t<http://www.yso.fi/onto/yso/p2557>

        Oulunlinnan\t<http://www.yso.fi/onto/yso/p7346>
        A line with no tabs
        Harald Hirmuinen\t<http://www.yso.fi/onto/yso/p6479>""")
    docs = annif.corpus.DocumentFile(str(docfile))
    assert len(list(docs.documents)) == 3
    assert len(caplog.records) == 2
    expected_msg = 'Skipping invalid line (missing tab):'
    for record in caplog.records:
        assert expected_msg in record.message

def test_docfile_gzipped(tmpdir):
    docfile = tmpdir.join('documents.tsv.gz')
    with gzip.open(str(docfile), 'wt') as gzf:
        gzf.write("""Pohjoinen\t<http://www.yso.fi/onto/yso/p2557>
            Oulunlinnan\t<http://www.yso.fi/onto/yso/p7346>
            Harald Hirmuinen\t<http://www.yso.fi/onto/yso/p6479>""")

    docs = annif.corpus.DocumentFile(str(docfile))
    assert len(list(docs.documents)) == 3

def test_docfile_is_empty(tmpdir):
    empty_file = tmpdir.ensure('empty.tsv')
    docs = annif.corpus.DocumentFile(str(empty_file))
    assert docs.is_empty()

def test_combinedcorpus(tmpdir):
    docfile = tmpdir.join('documents.tsv')
    docfile.write("""Läntinen\t<http://www.yso.fi/onto/yso/p2557>
        Oulunlinnan\t<http://www.yso.fi/onto/yso/p7346>
        Harald Hirmuinen\t<http://www.yso.fi/onto/yso/p6479>""")

    corpus1 = annif.corpus.DocumentFile(str(docfile))
    corpus2 = annif.corpus.DocumentFile(str(docfile))

    combined = annif.corpus.CombinedCorpus([corpus1, corpus2])

    assert len(list(combined.documents)) == 6

def test_transformingcorpus(document_corpus):
    def double(x): return x + x

    transformed_corpus = TransformingDocumentCorpus(document_corpus, double)
    for transf_doc, doc in zip(transformed_corpus.documents,
                               document_corpus.documents):
        assert transf_doc.text == doc.text + doc.text
        assert transf_doc.uris == doc.uris
        assert transf_doc.labels == doc.labels
    # Ensure docs are still available after iterating
    assert len(list(transformed_corpus.documents)) \
        == len(list(document_corpus.documents))

def test_limitingcorpus(tmpdir):
    docfile = tmpdir.join('documents_invalid.tsv')
    docfile.write("""Läntinen\t<http://www.yso.fi/onto/yso/p2557>

        Oulunlinnan\t<http://www.yso.fi/onto/yso/p7346>
        A line with no tabs
        Harald Hirmuinen\t<http://www.yso.fi/onto/yso/p6479>""")

    document_corpus = annif.corpus.DocumentFile(str(docfile))
    limiting_corpus = annif.corpus.LimitingDocumentCorpus(document_corpus, 2)

    assert len(list(limiting_corpus.documents)) == 2
    for limited_doc, doc in zip(limiting_corpus.documents,
                                document_corpus.documents):
        assert limited_doc.text == doc.text

max_stars_repo_name: jimfhahn/Annif

max_issues_repo_name: jimfhahn/Annif

max_forks_repo_name: jimfhahn/Annif


text:
import numpy as np
import json
import os
import sys

import eval_helpers
from eval_helpers import Joint

def computeMetrics(scores_all, labels_all, num_gt_all):
    ap_all = np.zeros((num_gt_all.shape[0] + 1, 1))
    rec_all = np.zeros((num_gt_all.shape[0] + 1, 1))
    pre_all = np.zeros((num_gt_all.shape[0] + 1, 1))
    # iterate over joints
    for j in range(num_gt_all.shape[0]):
        scores = np.zeros([0, 0], dtype=np.float32)
        labels = np.zeros([0, 0], dtype=np.int8)
        # iterate over images
        for imgidx in range(num_gt_all.shape[1]):
            scores = np.append(scores, scores_all[j][imgidx])
            labels = np.append(labels, labels_all[j][imgidx])
        # compute recall/precision values
        n_gt = sum(num_gt_all[j, :])
        precision, recall, scores_sorted_idxs = eval_helpers.computeRPC(scores, labels, n_gt)
        if len(precision) > 0:
            ap_all[j] = eval_helpers.VOCap(recall, precision) * 100
            pre_all[j] = precision[len(precision) - 1] * 100
            rec_all[j] = recall[len(recall) - 1] * 100

    # mean cuối cùng ở phần từ cuối cùng
    idxs = np.argwhere(~np.isnan(ap_all[:num_gt_all.shape[0], 0]))
    ap_all[num_gt_all.shape[0]] = ap_all[idxs, 0].mean()
    idxs = np.argwhere(~np.isnan(rec_all[:num_gt_all.shape[0], 0]))
    rec_all[num_gt_all.shape[0]] = rec_all[idxs, 0].mean()
    idxs = np.argwhere(~np.isnan(pre_all[:num_gt_all.shape[0], 0]))
    pre_all[num_gt_all.shape[0]] = pre_all[idxs, 0].mean()

    return ap_all, pre_all, rec_all

def evaluateAP(gt_frames_all, pred_frames_all, output_dir, bSaveAll=True, bSaveSeq=False):
    dist_thresh = 0.5

    seqidxs = []
    for imgidx in range(len(gt_frames_all)):
        seqidxs += [gt_frames_all[imgidx]["seq_id"]]
    seqidxs = np.array(seqidxs)

    seqidxsUniq = np.unique(seqidxs)
    nSeq = len(seqidxsUniq)

    names = Joint().name
    names['21'] = 'total'

    if bSaveSeq:
        for si in range(nSeq):
            print("seqidx: %d/%d" % (si + 1, nSeq))

            # extract frames IDs for the sequence
            imgidxs = np.argwhere(seqidxs == seqidxsUniq[si])
            seq_name = gt_frames_all[imgidxs[0, 0]]["seq_name"]

            gt_frames = [gt_frames_all[imgidx] for imgidx in imgidxs.flatten().tolist()]
            pred_frames = [pred_frames_all[imgidx] for imgidx in imgidxs.flatten().tolist()]

            # assign predicted poses to GT poses
            # scores: part detections scores
            # labels: positive/negative labels
            # nGT: number of annotated joints per image
            scores, labels, num_gt, _ = eval_helpers.assignGTmulti(gt_frames, pred_frames, dist_thresh)

            # compute average precision (AP), precision and recall per part
            ap, precision, recall = computeMetrics(scores, labels, num_gt)
            metricsSeq = {'ap': ap.flatten().tolist(), 'pre': precision.flatten().tolist(), 'rec': recall.flatten().tolist(), 'names': names}

            filename = output_dir + '/' + seq_name + '_AP_metrics.json'
            print('saving results to', filename)
            eval_helpers.writeJson(metricsSeq, filename)

    # assign predicted poses to GT poses
    scoresAll, labelsAll, nGTall, _ = eval_helpers.assignGTmulti(gt_frames_all, pred_frames_all, dist_thresh)

    # compute average precision (AP), precision and recall per part
    apAll, preAll, recAll = computeMetrics(scoresAll, labelsAll, nGTall)
    if bSaveAll:
        metrics = {'ap': apAll.flatten().tolist(), 'pre': preAll.flatten().tolist(), 'rec': recAll.flatten().tolist(), 'names': names}
        filename = output_dir + '/total_AP_metrics.json'
        print('saving results to', filename)
        eval_helpers.writeJson(metrics, filename)

    return apAll, preAll, recAll

ext: py

reason: import numpy


text:
"""

"""
import time
import numpy as np
from threading import Thread
from TPD.MassSpec_Functions import thread_read_masses
from TPD.Eurotherm_Functions import thread_send_mV_setpoint,read_Temp_value
# from TPD.Gui import emergency_stop
from TPD.Header_info import write_data, header_info
import pyqtgraph as pg
from queue import Queue
import sys
import os
from PyQt5.QtWidgets import QMainWindow, QPushButton, QApplication
from pyqtgraph.Qt import QtCore, QtGui

def runme(iter_freq, mv_setpoints, time_array,list_of_masses, list_of_sensitivities, sendFreq, alpha, alpha_iter,
          n, temp_plot_arr, T_array, mass_signals, setpoint_sending, temp_setpoints, curves, T_set_curve, T_read_curve,
          controlObj, controlObj2, uti1, app, win1, win2, button, button_hold, button_hold_off, stop_button, hold_button, hold_off_button, save_path,
          project_folder, experiment_name, hold_time=30):

    def emergency_stop():
        # stop increasing the temp and send a safe setpoint
        # controlObj.write_sp(0.0)
        # TODO add save data

        # from TPD.Main import cum_time, T_array, mass_signals

        controlObj.write_sp(-2.9)
        controlObj.close_me()
        controlObj2.close_me()
        time.sleep(1)
        # quit()

        os.chdir(save_path)
        os.makedirs(project_folder, exist_ok=True)
        write_data(project_folder, experiment_name, cum_time, T_array, mass_signals)

        sys.exit()

    def hold_off():
        time.sleep(hold_time)
        controlObj.write_sp(-2.9)
        controlObj.close_me()
        controlObj2.close_me()
        time.sleep(1)
        # quit()

        os.chdir(save_path)
        os.makedirs(project_folder, exist_ok=True)
        write_data(project_folder, experiment_name, cum_time, T_array, mass_signals)

        sys.exit()

    def hold_temperature():
        time.sleep(hold_time)

    button.clicked.connect(emergency_stop)
    button_hold.clicked.connect(hold_temperature)
    button_hold_off.clicked.connect(hold_off)

    cum_time = 0

    # proxy = QtGui.QGraphicsProxyWidget()
    # button = QtGui.QPushButton('STOP')
    # proxy.setWidget(button)
    # stop_button = win2.addItem(proxy)
    # button.clicked.connect(emergency_stop(controlObj))

    queue = Queue()
    # while iter_freq < len(mv_setpoints):
    while alpha_iter < len(setpoint_sending):
        start_alpha = time.time()
        # append current time to the array.
        time_array = np.append(time_array, time.time())
        # Read time and temperature data
        cum_time = np.cumsum(np.concatenate(([0], np.diff(time_array))))

        t = Thread(target=thread_read_masses, args=(uti1, list_of_masses, list_of_sensitivities, queue))
        # t = Thread(target=thread_read_masses, args=(list_of_masses, list_of_sensitivities, mass_signals))
        # cProfile.run('t.start()')
        t.start()
        # t = cProfile.run('thread_profile(list_of_masses, list_of_sensitivities, queue)')
        # print(setpoint_sending[alpha_iter])
        iter_freq = thread_send_mV_setpoint(iter_freq, mv_setpoints, setpoint_sending[alpha_iter], sendFreq, alpha,
                                            controlObj, t)
        t.join()
        response = queue.get()
        # print(alpha_iter)
        alpha_iter += 1

        try:
            # controlObj.write_sp(mv_setpoints[iter_freq])
            if alpha_iter*n <= len(temp_setpoints) and alpha_iter != 0:
                # temp_plot_arr = np.append(temp_plot_arr, temp_setpoints[multiple*i])
                temp_plot_arr = np.append(temp_plot_arr, temp_setpoints[iter_freq])
            else:
                temp_plot_arr = np.append(temp_plot_arr, temp_setpoints[0])
        except IndexError as e:
            print(e)
            controlObj.write_sp(mv_setpoints[0])

        try:
            T_set_curve.setData(x=cum_time[:len(temp_plot_arr)], y=temp_plot_arr, pen='r', name='setpoint')
        except:
            T_set_curve.setData(x=cum_time, y=temp_plot_arr[:len(cum_time)], pen='r', name='setpoint')
        # TODO read_Temp_value is a duplicate of Eurotherm.read_temp() fcn
        T_array = np.append(T_array, read_Temp_value(controlObj, controlObj2))
        T_read_curve.setData(x=cum_time, y=T_array, pen='g', name='readout')

        # TODO add masses

        if len(list_of_masses) > 0 and not None:
            for i, j, k in zip(response, list_of_masses, curves):
                mass_signals['mass' + str(j)] = np.append(mass_signals['mass' + str(j)], i)
                if alpha_iter % 4 == 0:
                    k.setData(x=T_array, y=mass_signals['mass{0}'.format(str(j))], pen='g')
                    # pg.QtGui.QApplication.processEvents()
        # for mass, mass_plot in zip(list_of_masses, curves):
        #     mass_plot.setData(x=T_array, y=mass_signals['mass{0}'.format(mass)], pen='g')

        # T_set_curve.setData(x=cum_time,y=temp_plot_arr[:len(cum_time)], pen='r',name='setpoint')
        if alpha_iter % 4 == 0:
            pg.QtGui.QApplication.processEvents()
        # uncomment if you want to see some cooldown
        if iter_freq == len(mv_setpoints):
            break

        print('total iter time = ' + str(time.time() - start_alpha))

    return T_array, cum_time, start_alpha, time_array, mass_signals

max_stars_repo_name: mannyfin/PythonLabView

max_issues_repo_name: mannyfin/PythonLabView

max_forks_repo_name: mannyfin/PythonLabView


text:
import numpy as np
import time
from algorith.arrays.binary_srch import binary_search, binary_search_oprns

def critical_events(ts1,ts2,w):
    """
    ts1 rains down upon ts2. The window size can be
    positive or negative.
    """
    if len(ts2)==0 or len(ts1)==0:
        return 0
    j=0; critical=0
    for t in ts1:
        #First time an entry in ts2 crosses
        #current entry of ts1
        # ts2[j-1].....t......ts2[j]
        while j<len(ts2) and t>ts2[j]:
            j+=1
        if w>0 and j>0:
            critical+=(ts2[j-1]+w>t)
        elif j<len(ts2) and w<0:
            critical+=(ts2[j]+w<t)
    return critical

def critical_events_v2(ts1,ts2,w):
    if w>0:
        return critical_events_v2_pos_w(ts1,ts2,w)
    elif w<0:
        return critical_events_v2_neg_w(ts1,ts2,w)
    else:
        return 0

def critical_events_v2_neg_w(ts1,ts2,w):
    """
    Designed for when ts1 is much larger than ts2.
    Leverages binary search to find critical events
    from ts1 lying within w-window of some event in ts2
    in O(mlog(n)) time where m=len(ts2), n=len(ts1)
    params:
        ts1: The first array, raining down on ts2.
        ts2: The second array around which we find intervals.
        w: The window size. Must be negative for this method.
    """
    critical=0; ix=len(ts1)-1; seen_until=len(ts1)
    for ii in range(len(ts2)-1,-1,-1):
        t=ts2[ii]
        ix=binary_search(ts1,t,0,ix)
        seen_until=min(seen_until,ix+1)
        while ix>-1 and t+w<ts1[ix]:
            # The binary search has a propensity to
            # go forward. So, we avoid double counting.
            critical+=(seen_until>ix)
            ix-=1
            seen_until=min(seen_until,ix+1)
    return critical

def critical_events_v2_pos_w(ts1,ts2,w):
    """
    Designed for when ts1 is much larger than ts2.
    Leverages binary search to find critical events
    from ts1 lying within w-window of some event in ts2
    in O(mlog(n)) time where m=len(ts2), n=len(ts1)
    params:
        ts1: The first array, raining down on ts2.
        ts2: The second array around which we find intervals.
        w: The window size. Must be positive for this method.
    """
    critical=0; ix=0; seen_until=-1
    for t in ts2:
        ix=binary_search(ts1,t,ix,len(ts1)-1)
        seen_until=max(seen_until,ix)
        while ix+1<len(ts1) and t+w>ts1[ix+1]:
            # The binary search has a propensity to
            # go backwards. So, we avoid double counting.
            critical+=(seen_until<(ix+1))
            ix+=1
            seen_until=max(seen_until,ix)
    return critical

def critical_events_v3_pos_w(ts1,ts2,w):
    eta=0; start=-1; end=len(ts1)-1
    for t in ts2:
        i1 = binary_search(ts1,t,max(0,start),end)
        i1=max(i1,start)
        i2 = binary_search(ts1,t+w,max(0,i1),end)
        i2=max(i2,start)
        eta+=(i2-i1)
        start=max(start,i2)
        if start>=len(ts1)-1:
            return eta
    return eta

def critical_events_v3_pos_w_oprns(ts1,ts2,w,oprns=[0]):
    eta=0; start=-1; end=len(ts1)-1
    for t in ts2:
        i1 = binary_search_oprns(ts1,t,max(0,start),end,oprns=oprns)
        i1=max(i1,start)
        oprns[0]+=1
        i2 = binary_search_oprns(ts1,t+w,max(0,i1),end,oprns=oprns)
        i2=max(i2,start)
        eta+=(i2-i1)
        start=max(start,i2)
        oprns[0]+=3
        if start>=len(ts1)-1:
            oprns[0]+=1
            return eta
    return eta

def critical_events_v3(ts1,ts2,w):
    if w>0:
        return critical_events_v3_pos_w(ts1,ts2,w)
    else:
        ## TODO: Implement v3 version for negative w.
        return critical_events_v2_neg_w(ts1,ts2,w)

def critical_events_v4_pos_w(ts1,ts2,w):
    st1=-1; st2=0
    eta=0
    while (st1<len(ts1)-1) and (st2<len(ts2)):
        i1 = binary_search(ts1,ts2[st2],max(st1,0),len(ts1)-1)
        i1=max(st1,i1)
        i2 = binary_search(ts1,ts2[st2]+w,max(i1,0),len(ts1)-1)
        i2=max(st1,i2)
        eta+=(i2-i1)
        st1 = max(st1,i2)
        if st1<len(ts1)-1:
            st2_pr=binary_search(ts2,ts1[st1+1],st2+1,len(ts2)-1)
            st2=max(st2_pr,st2+1)
    return eta

def critical_events_v4(ts1,ts2,w):
    if w>0:
        return critical_events_v4_pos_w(ts1,ts2,w)
    else:
        ## TODO: Implement v4 version for negative w.
        return critical_events_v2_neg_w(ts1,ts2,w)

def functional_tst():
    ## Test cases.
    ev = critical_events_v2([.5,1.5,2.5],[1,2,3],.45)
    print(ev==0)
    ev = critical_events_v2([1.4,1.4,2.7],[1,2,3],.5)
    print(ev==2)
    ev = critical_events_v2([1.4,2.4,3.4],[1,2,3],.5)
    print(ev==3)
    ev = critical_events([1,2,3],[.5,1.5,2.5],-1)
    print(ev==3)
    ev = critical_events([1,2,3],[.5,1.5,2.5],1)
    print(ev==3)
    ## From a simulation.
    ts1=np.array([0.02364078, 0.05228936, 0.3201887 ,
        0.41696722, 0.50680057, 0.58062848, 0.6491584 , 0.76480126,
        0.79478828, 0.9453893 ])
    ts2=np.array([0.24825007, 0.70216959, 0.73216372, 0.89533743, 0.97304893])
    ev=critical_events_v2(ts1,ts2,0.1)
    print(ev==4)
    ev=critical_events_v2(ts1,ts2,-0.1)
    print(ev==2)

def scale_tst():
    ts1=np.sort(np.random.uniform(size=500000))
    ts2=np.sort(np.random.uniform(size=100))
    start = time.time()
    crit1 = critical_events(ts1,ts2,0.0001)
    end = time.time()
    print(crit1)
    print(end - start)
    start = time.time()
    crit2 = critical_events_v2(ts1,ts2,0.0001)
    end = time.time()
    print(crit2)
    print(end - start)
    print("Functional test result: " + str(crit2==crit1))


text:
import pytest
import util

@pytest.fixture
def train_skl():
    # Example BDT creation from: https://scikit-learn.org/stable/modules/ensemble.html
    from sklearn.datasets import make_hastie_10_2
    from sklearn.ensemble import GradientBoostingClassifier

    # Make a random dataset from sklearn 'hastie'
    X, y = make_hastie_10_2(random_state=0)
    X_train, X_test = X[:2000], X[2000:]
    y_train, y_test = y[:2000], y[2000:]

    # Train a BDT
    clf = GradientBoostingClassifier(n_estimators=20, learning_rate=1.0,
        max_depth=3, random_state=0).fit(X_train, y_train)

    return clf, X_test, y_test

@pytest.fixture
def hls_convert(train_skl):
    import conifer
    import datetime

    clf, X, y = train_skl

    # Create a conifer config
    cfg = conifer.backends.vivadohls.auto_config()
    cfg['Precision'] = 'ap_fixed<32,16,AP_RND,AP_SAT>'
    # Set the output directory to something unique
    cfg['OutputDir'] = 'prj_{}'.format(int(datetime.datetime.now().timestamp()))
    cfg['XilinxPart'] = 'xcu250-figd2104-2L-e'

    # Create and compile the model
    model = conifer.model(clf, conifer.converters.sklearn, conifer.backends.vivadohls, cfg)
    model.compile()
    return model

@pytest.fixture
def vhdl_convert(train_skl):
    import conifer
    import datetime

    clf, X, y = train_skl

    # Create a conifer config
    cfg = conifer.backends.vivadohls.auto_config()
    cfg['Precision'] = 'ap_fixed<32,16>'
    # Set the output directory to something unique
    cfg['OutputDir'] = 'prj_{}'.format(int(datetime.datetime.now().timestamp()))
    cfg['XilinxPart'] = 'xcu250-figd2104-2L-e'

    # Create and compile the model
    model = conifer.model(clf, conifer.converters.sklearn, conifer.backends.vhdl, cfg)
    model.compile()
    return model

@pytest.fixture
def predict(train_skl, hls_convert):
    clf, X, y = train_skl
    model = hls_convert
    return util.predict(clf, X, y, model)

def test_skl_hls_predict(predict):
    import numpy as np
    y_hls, y_skl = predict
    assert np.all(np.isclose(y_hls, y_skl, rtol=1e-2, atol=1e-2))

def test_skl_build(hls_convert):
    model = hls_convert
    model.build()
    assert True

def test_hdl_predict(train_skl, vhdl_convert):
    clf, X, y = train_skl
    model = vhdl_convert
    return util.predict(clf, X, y, model)

def test_hdl_build(vhdl_convert):
    model = vhdl_convert
    model.build()
    assert True

max_stars_repo_name: ConsVin/conifer

max_issues_repo_name: ConsVin/conifer

max_forks_repo_name: ConsVin/conifer


text:
#  extract_cnn_vgg16_keras.py
# -*- coding: utf-8 -*-

# keras/tensorflow 使用flask部署服务的常见错误的解决办法
# http://www.luyixian.cn/news_show_381268.aspx
# 非常重要，需要如此才能嵌入flask中使用
# 当前仅能在非dev模式下使用，
# dev会二次load，会有找不到模块的错误。当前没办法解决。

import os

import numpy as np
import tensorflow.compat.v1 as tf  # 这个是需要的
from keras.applications.vgg16 import VGG16
from keras.applications.vgg16 import preprocess_input
from keras.backend import set_session
from keras.preprocessing import image
from numpy import linalg

# 下面这些方法是无法运行的
# from tensorflow_core.python.keras.api.keras.preprocessing import image
# from tensorflow_core.python.keras.applications.vgg16 import VGG16, preprocess_input
# from tensorflow_core.python.keras.backend import set_session

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

sess = tf.Session()
graph = tf.get_default_graph()

__all__ = [
    "vgg16_model"
]

class VGGNet:
    def __init__(self):
        set_session(sess)
        # weights: 'imagenet'
        # pooling: 'max' or 'avg'
        # input_shape: (width, height, 3), width and height should >= 48
        self.input_shape = (224, 224, 3)
        self.weight = 'imagenet'
        self.pooling = 'max'
        # include_top：是否保留顶层的3个全连接网络
        # weights：None代表随机初始化，即不加载预训练权重。'imagenet'代表加载预训练权重
        # input_tensor：可填入Keras tensor作为模型的图像输出tensor
        # input_shape：可选，仅当include_top=False有效，应为长为3的tuple，指明输入图片的shape，图片的宽高必须大于48，如(200,200,3)
        # pooling：当include_top = False时，该参数指定了池化方式。None代表不池化，最后一个卷积层的输出为4D张量。‘avg’代表全局平均池化，‘max’代表全局最大值池化。
        # classes：可选，图片分类的类别数，仅当include_top = True并且不加载预训练权重时可用。
        self.model_vgg = VGG16(weights=self.weight,
                               input_shape=(self.input_shape[0], self.input_shape[1], self.input_shape[2]),
                               pooling=self.pooling, include_top=False)
        # with graph.as_default():
        self.model_vgg.predict(np.zeros((1, 224, 224, 3)))

    '''
    Use vgg16/Resnet model to extract features
    Output normalized feature vector
    '''

    # 提取vgg16最后一层卷积特征
    def vgg_extract_feat(self, img_path):
        global sess, graph
        with graph.as_default():
            img = image.load_img(img_path, target_size=(self.input_shape[0], self.input_shape[1]))
            img = image.img_to_array(img)
            img = np.expand_dims(img, axis=0)
            set_session(sess)
            img = preprocess_input(img)
            feat = self.model_vgg.predict(img)
        # print(feat.shape)
        # print(feat[0])
        # print(LA.norm(feat[0]))
        norm_feat = feat[0] / linalg.norm(feat[0])
        return norm_feat

    # 提取vgg16最后一层卷积特征
    def vgg_extract_feat_batch(self, img_path_arr):
        global sess, graph
        set_session(sess)
        imgs = []
        for img_path in img_path_arr:
            img = image.load_img(img_path, target_size=(self.input_shape[0], self.input_shape[1]))
            img = image.img_to_array(img)
            img = np.expand_dims(img, axis=0)
            imgs.append(img)
        with graph.as_default():
            set_session(sess)
            img = np.concatenate([x for x in imgs])
            img = preprocess_input(img)
            # with graph.as_default():
            feat = self.model_vgg.predict(img)
        # print(feat.shape)
        # print(feat[0])
        # print(LA.norm(feat[0]))
        # norm_feat_arr = []
        # for i in xrange(0, feat.shape[0]):
        #     norm_feat = feat[i] / LA.norm(feat[i])
        #     norm_feat_arr.append(norm_feat)
        # return norm_feat_arr
        return feat / linalg.norm(feat)

vgg16_model = VGGNet()

max_stars_repo_name: hontlong/vue-admin-template

max_issues_repo_name: hontlong/vue-admin-template

max_forks_repo_name: hontlong/vue-admin-template

reason: import numpy,from numpy


text:
# coding: utf-8

from sklearn import tree
import numpy as np
import csv

# Read Data Set
data = []
with open('../dataSets/sampleData.csv', 'rU') as csvfile:
    reader  = csv.reader(csvfile)
    for line in reader:
        data.append([float(line[0]), float(line[1])])
x_train = np.array(data)
y_train = [2,1,0,2,0,1,2,0,0,0,0,0,0,1,0,0,0,0,0]
x_test = np.array([[26,17] , [6.74, 10.45], [25., 15.]])

model = tree.DecisionTreeClassifier(criterion='gini')
model.fit(x_train, y_train)
model.score(x_train, y_train)
predicted= model.predict(x_test)
print predicted

max_stars_repo_name: Insiyaa/pyrobo

max_issues_repo_name: Insiyaa/pyrobo


text:
import json
import os
from math import *
from numpy import *

def ReadJson(fileName):
   fp = open(fileName, 'r')
   jsonDict = json.load(fp)
   fp.close()
   return jsonDict

def writeDataFile(realVectorX, complexVectorYList,fileName):
   if os.path.isfile(fileName):
      print 'warning, overwriting %s' % fileName
   fp = open(fileName, 'w')
   #fp.write(header)  #header
   isCompl = True #any(iscomplex(complexVectorYList))
   for ii in range(0,len(realVectorX)):
      lineToPrint = '%  10.7f' % realVectorX[ii]
      lineToPrint += ' % 9.7f' % real(complexVectorYList[ii])
      lineToPrint += ' % 9.7f' % imag(complexVectorYList[ii])
      fp.write(lineToPrint + '\n')
   fp.close()

def main1():
   hyb = ReadJson('Hyb99.json')
   #################################################
   #WRITE YOUR CODE HERE

   omega_n=[]
   complexVectorYList=[]
   for nn in range(0,len(hyb['A']['imag'])):
      omega_n.append((2*nn+1)*3.1416/hyb['A']['beta'])
      complexVectorYList.append(hyb['A']['real'][nn] + 1.0j* hyb['A']['imag'][nn])

   print omega_n
   print complexVectorYList
   #################################################
   writeDataFile(omega_n, complexVectorYList,'hyb99.dat')

main1()

max_stars_repo_name: mnmpdadish/ctint

max_issues_repo_name: mnmpdadish/ctint

max_forks_repo_name: mnmpdadish/ctint

reason: from numpy


text:
import tensorflow as tf
import numpy as np
import sys
from src.convolutional import set_convolutional
from src.crops_sr import extract_crops_z_sr, extract_crops_x_sr, pad_frame_sr, resize_images
import src.ConvNetValue_Class as cnvc

sys.path.append('../')

# the follow parameters *have to* reflect the design of the network to be imported
# TODO: <> instance version
def build_tracking_graph_from_instance(final_score_sz, design):
    pos_x_ph = tf.placeholder(tf.float64)
    pos_y_ph = tf.placeholder(tf.float64)

    z_sz_ph_w = tf.placeholder(tf.float64)
    z_sz_ph_h = tf.placeholder(tf.float64)

    # TODO: <> add placeholders for scale-ratio here
    # x_sz0_ph = tf.placeholder(tf.float64)
    # x_sz1_ph = tf.placeholder(tf.float64)
    # x_sz2_ph = tf.placeholder(tf.float64)

    x_s0r0_w = tf.placeholder(tf.float64)
    x_s0r0_h = tf.placeholder(tf.float64)
    x_s0r1_w = tf.placeholder(tf.float64)
    x_s0r1_h = tf.placeholder(tf.float64)
    x_s0r2_w = tf.placeholder(tf.float64)
    x_s0r2_h = tf.placeholder(tf.float64)

    x_s1r0_w = tf.placeholder(tf.float64)
    x_s1r0_h = tf.placeholder(tf.float64)
    x_s1r1_w = tf.placeholder(tf.float64)
    x_s1r1_h = tf.placeholder(tf.float64)
    x_s1r2_w = tf.placeholder(tf.float64)
    x_s1r2_h = tf.placeholder(tf.float64)

    x_s2r0_w = tf.placeholder(tf.float64)
    x_s2r0_h = tf.placeholder(tf.float64)
    x_s2r1_w = tf.placeholder(tf.float64)
    x_s2r1_h = tf.placeholder(tf.float64)
    x_s2r2_w = tf.placeholder(tf.float64)
    x_s2r2_h = tf.placeholder(tf.float64)

    # TODO <> set max scale
    x_max_ph_w = tf.placeholder(tf.float64)
    x_max_ph_h = tf.placeholder(tf.float64)

    # TODO: <> generate a instance of class ConvNetValue who has the latest value of params of model
    # matconvnet_value = cnvc.ConvNetValue()
    # TODO: <> OR I just import the value from file ConvNetValue_Class.py
    matconvnet_value = cnvc.ConvNetValue().update_value()

    filename = tf.placeholder(tf.string, [], name='filename')
    image_file = tf.read_file(filename)
    # Decode the image as a JPEG file, this will turn it into a Tensor
    image = tf.image.decode_jpeg(image_file)
    image = 255.0 * tf.image.convert_image_dtype(image, tf.float32)

    frame_sz = tf.shape(image)
    # used to pad the crops
    if design.pad_with_image_mean:
        avg_chan = tf.reduce_mean(image, axis=(0,1), name='avg_chan')
    else:
        avg_chan = None
    # pad with if necessary
    # TODO: <>
    #frame_padded_z, npad_z = pad_frame(image, frame_sz, pos_x_ph, pos_y_ph, z_sz_ph, avg_chan)
    frame_padded_z, npad_z = pad_frame_sr(image, frame_sz, pos_x_ph, pos_y_ph, z_sz_ph_w, z_sz_ph_h, avg_chan)
    frame_padded_z = tf.cast(frame_padded_z, tf.float32)
    # extract tensor of z_crops
    # TODO: <> introduce width and height here
    #z_crops = extract_crops_z(frame_padded_z, npad_z, pos_x_ph, pos_y_ph, z_sz_ph, design.exemplar_sz)
    z_crops = extract_crops_z_sr(frame_padded_z, npad_z, pos_x_ph, pos_y_ph, z_sz_ph_w, z_sz_ph_h, design.exemplar_sz)

    # TODO: <> set max scale
    #frame_padded_x, npad_x = pad_frame(image, frame_sz, pos_x_ph, pos_y_ph, x_sz2_ph, avg_chan)
    frame_padded_x, npad_x = pad_frame_sr(image, frame_sz, pos_x_ph, pos_y_ph, x_max_ph_w, x_max_ph_h, avg_chan)
    frame_padded_x = tf.cast(frame_padded_x, tf.float32)
    # extract tensor of x_crops (3 scales)
    # TODO: <> introduce 9 scale and ratio
    #x_crops = extract_crops_x(frame_padded_x, npad_x, pos_x_ph, pos_y_ph, x_sz0_ph, x_sz1_ph, x_sz2_ph, design.search_sz)
    x_crops = extract_crops_x_sr(frame_padded_x, npad_x, pos_x_ph, pos_y_ph,
                              x_s0r0_w, x_s0r0_h, x_s0r1_w, x_s0r1_h, x_s0r2_w, x_s0r2_h,
                              x_s1r0_w, x_s1r0_h, x_s1r1_w, x_s1r1_h, x_s1r2_w, x_s1r2_h,
                              x_s2r0_w, x_s2r0_h, x_s2r1_w, x_s2r1_h, x_s2r2_w, x_s2r2_h,
                              x_max_ph_w, x_max_ph_h,
                              design.search_sz)

    # TODO: <> create siamese from instance
    # use crops as input of (MatConvnet imported) pre-trained fully-convolutional Siamese net
    # template_z, templates_x, p_names_list, p_val_list = _create_siamese(os.path.join(env.root_pretrained,design.net), x_crops, z_crops)
    template_z, templates_x= _create_siamese_from_instance(z_crops, x_crops, matconvnet_value)
    template_z = tf.squeeze(template_z)
    # TODO: <>
    #templates_z = tf.stack([template_z, template_z, template_z])
    templates_z = tf.stack([template_z, template_z, template_z, template_z, template_z, template_z,
                            template_z, template_z, template_z])

    # TODO: <> match templates from instance
    # compare templates via cross-correlation
    #scores = _match_templates(templates_z, templates_x, p_names_list, p_val_list)
    scores = _match_templates_from_instance(templates_z, templates_x, matconvnet_value)

    # upsample the score maps
    scores_up = tf.image.resize_images(scores, [final_score_sz, final_score_sz],
        method=tf.image.ResizeMethod.BICUBIC, align_corners=True)
    return filename, image, templates_z, scores_up

# TODO: <> load the model from instance of class convnetvalue
def _create_siamese_from_instance(net_z, net_x, instance):
    # read mat file from net_path and start TF Siamese graph from placeholders X and Z
    # params_names_list, params_values_list = _import_from_matconvnet(net_path)

    # loop through the flag arrays and re-construct network, reading parameters of conv and bnorm layers
    for i in range(instance.num_layers):

        # set up conv "block" with bnorm and activation
        net_x = set_convolutional(net_x, instance.W_d[i], np.swapaxes(instance.b_d[i], 0, 1), instance.conv_stride_n[i],
                                  instance.bn_beta_d[i], instance.bn_gamma_d[i], instance.bn_moving_mean_d[i],
                                  instance.bn_moving_variance_d[i], filtergroup=instance.filtergroup_yn_n[i],
                                  batchnorm=instance.bnorm_yn_n[i], activation=instance.relu_yn_n[i],
                                  scope='conv' + str(i + 1), trainable=instance.trainable_n[i], reuse=tf.AUTO_REUSE)

        # notice reuse=True for Siamese parameters sharing
        net_z = set_convolutional(net_z, instance.W_d[i], np.swapaxes(instance.b_d[i], 0, 1), instance.conv_stride_n[i],
                                  instance.bn_beta_d[i], instance.bn_gamma_d[i], instance.bn_moving_mean_d[i],
                                  instance.bn_moving_variance_d[i], filtergroup=instance.filtergroup_yn_n[i],
                                  batchnorm=instance.bnorm_yn_n[i], activation=instance.relu_yn_n[i],
                                  scope='conv' + str(i + 1), trainable=instance.trainable_n[i], reuse=True)

        # add max pool if required
        if instance.pool_stride_n[i] > 0:
            print('\t\tMAX-POOL: size ' + str(instance.pool_sz) + ' and stride ' + str(instance.pool_stride_n[i]))
            net_x = tf.nn.max_pool(net_x, [1, instance.pool_sz, instance.pool_sz, 1],
                                   strides=[1, instance.pool_stride_n[i], instance.pool_stride_n[i], 1],
                                   padding='VALID', name='pool' + str(i + 1))
            net_z = tf.nn.max_pool(net_z, [1, instance.pool_sz, instance.pool_sz, 1],
                                   strides=[1, instance.pool_stride_n[i], instance.pool_stride_n[i], 1],
                                   padding='VALID', name='pool' + str(i + 1))

    return net_z, net_x

# TODO: <> instance version
def _match_templates_from_instance(net_z, net_x, instance):
    # finalize network
    # z, x are [B, H, W, C]
    net_z = tf.transpose(net_z, perm=[1,2,0,3])
    net_x = tf.transpose(net_x, perm=[1,2,0,3])
    # z, x are [H, W, B, C]
    Hz, Wz, B, C = tf.unstack(tf.shape(net_z))
    Hx, Wx, Bx, Cx = tf.unstack(tf.shape(net_x))
    # assert B==Bx, ('Z and X should have same Batch size')
    # assert C==Cx, ('Z and X should have same Channels number')
    net_z = tf.reshape(net_z, (Hz, Wz, B*C, 1))
    net_x = tf.reshape(net_x, (1, Hx, Wx, B*C))
    net_final = tf.nn.depthwise_conv2d(net_x, net_z, strides=[1,1,1,1], padding='VALID')
    # final is [1, Hf, Wf, BC]
    # TODO: <>
    # net_final = tf.concat(tf.split(net_final, 3, axis=3), axis=0)
    net_final = tf.concat(tf.split(net_final, 9, axis=3), axis=0)
    # final is [B, Hf, Wf, C]
    net_final = tf.expand_dims(tf.reduce_sum(net_final, axis=3), axis=3)
    # final is [B, Hf, Wf, 1]
    if instance.bnorm_adjust:
        bn_beta = instance.bn_beta_adj
        bn_gamma = instance.bn_gamma_adj
        bn_moments = instance.bn_moments_adj
        bn_moving_mean = instance.bn_moving_mean_adj
        bn_moving_variance = instance.bn_moving_variance_adj
        net_final = tf.layers.batch_normalization(net_final, beta_initializer=tf.constant_initializer(bn_beta),
                                                gamma_initializer=tf.constant_initializer(bn_gamma),
                                                moving_mean_initializer=tf.constant_initializer(bn_moving_mean),
                                                moving_variance_initializer=tf.constant_initializer(bn_moving_variance),
                                                # TODO: <> params' setting
                                                training=False, trainable=False)

    return net_final


text:
import numpy as np; from numba import jit; from PIL import Image; import time
Iterations = 10000
Height, Width = 1000,2000
XZoom,YZoom,Xb,Yb = 4, 2.2, 2.5, 1.1

@jit(nopython=True)
def calculation(a,b):
    zi = 0
    zr = 0
    for i in range(Iterations):
        zi, zr = 2 * zr * zi + b, zr**2 - zi**2 + a
        if (zr**2 + zi**2) > 4:
            if i < 256: return i
            else: return 255
    return 0

@jit(nopython=True, parallel=True)
def main():
    array = np.zeros((Height,Width,3), dtype=np.uint8)
    for h in range(Height):
        for w in range(Width):
            a,b = w * XZoom / Width  - Xb, h * YZoom / Height - Yb
            array[h,w,0] = calculation(a,b)
        print(h)
    return array

start_time = time.time()
im = Image.fromarray(main(), mode="RGB")
im.save("Mandelbrot_%s.png" % round(time.time()))
print("--- %s seconds ---" % (time.time() - start_time))

max_stars_repo_name: BoardTM/fractals

max_issues_repo_name: BoardTM/fractals

max_forks_repo_name: BoardTM/fractals

reason: import numpy,from numba


text:
from typing import Sequence, Dict
import numpy as np

from abm1559.config import rng

from abm1559.utils import (
    constants,
)

from abm1559.chain import Block
from abm1559.users import User, User1559

def spawn_poisson_demand(timestep: int, demand_lambda: float, UserClass, rng: np.random.Generator = rng) -> Sequence[User]:
    """
    One-step demand from homogeneous users, with demand size drawn from a Poisson distribution.

    Args:
        timestep (int): Current round
        demand_lambda (float): Rate of arrival, the :math:`lambda` parameter of a Poisson distribution
        UserClass (class): The user type

    Returns:
        Sequence[User]: An array of users
    """

    demand_size = rng.poisson(demand_lambda)
    new_users = [UserClass(timestep, rng=rng) for i in range(demand_size)]
    return new_users

def spawn_poisson_heterogeneous_demand(timestep: int, demand_lambda: float, shares: Dict[type, float], rng: np.random.Generator = rng) -> Sequence[User]:
    """
    One-step demand from heterogeneous users, with demand size drawn from a Poisson distribution.

    Args:
        timestep (int): Current round
        demand_lambda (float): Rate of arrival, the :math:`lambda` parameter of a Poisson distribution
        UserClass (Dict[type, float]): Keys are user classes (subclasses of :py:class:`abm1559.users.User`), values are the share of each user class to spawn this round. Shares are expected to sum to 1.

    Returns:
        Sequence[User]: An array of users
    """

    new_users = []
    demand_size = rng.poisson(demand_lambda)
    sizes = shares_to_sizes(shares, demand_size)
    for UserClass, size in sizes.items():
        new_users += [UserClass(timestep, rng=rng) for i in range(size)]
    return new_users

def shares_to_sizes(shares: Dict[type, float], demand_size: int) -> Dict[type, int]:
    new_sizes = {}
    for i, (UserClass, share) in enumerate(shares.items()):
        if i == len(shares.values()) - 1: # last item
            new_sizes[UserClass] = demand_size - sum(new_sizes.values())
        else:
            new_sizes[UserClass] = int(share * demand_size)
    return new_sizes

def update_basefee(block: Block, basefee: int) -> int:
    """
    Basefee update rule

    Args:
        block (Block): The previous block
        basefee (int): The current basefee

    Returns:
        int: The new basefee
    """

    gas_used = sum([tx.gas_used for tx in block.txs])
    if gas_used == constants["TARGET_GAS_USED"]:
        new_basefee = basefee
    elif gas_used > constants["TARGET_GAS_USED"]:
        gas_delta = gas_used - constants["TARGET_GAS_USED"]
        fee_delta = max(basefee * gas_delta // constants["TARGET_GAS_USED"] // constants["BASEFEE_MAX_CHANGE_DENOMINATOR"], 1)
        new_basefee = basefee + fee_delta
    else:
        gas_delta = constants["TARGET_GAS_USED"] - gas_used
        fee_delta = basefee * gas_delta // constants["TARGET_GAS_USED"] // constants["BASEFEE_MAX_CHANGE_DENOMINATOR"]
        new_basefee = basefee - fee_delta
    return new_basefee

def generate_seeds(seeds: int = 100, rng: np.random.Generator = rng):
    return rng.integers(low=0, high=seeds*1000, size=seeds)

def generate_gbm(lambda_0: float, T: int, paths: int = 1, mu: float = 0.5, sigma: float = 1, rng: np.random.Generator = rng):
    t = np.repeat(np.array([np.arange(1, T+1)]), paths, axis = 0)
    b = rng.normal(size=[paths, T])
    w = b.cumsum(axis = 1)
    drift = (mu - 0.5 * sigma**2) * t
    diffusion = sigma * w
    S = lambda_0 * np.exp(drift + diffusion)
    return S


text:
import numpy as np

from brick.azr import AZR

azr = AZR('12C+p.azr')
azr.use_gsl = True
azr.ext_capture_file = 'output/intEC.dat'

theta = azr.config.get_input_values()

mu = azr.predict(theta, dress_up=False)[0]
azure2_output = np.loadtxt('output/AZUREOut_aa=1_R=2.out')

_, n = mu.shape

for i in range(n):
    print(i, np.linalg.norm(mu[:, i] - azure2_output[:, i]))

print(f'{np.linalg.norm(mu - azure2_output):.8e}')

max_stars_repo_name: odell/brick

max_issues_repo_name: odell/brick

max_forks_repo_name: odell/brick


text:
#!/usr/bin/env python3
#
# Copyright 2021.
# ozora-ogino

from typing import Tuple

import cv2
import numpy as np
from tflite_runtime.interpreter import Interpreter

class Detect(object):
    """YOLOv5 tflite detect model."""

    def __init__(
        self,
        model_file: str,
        conf_thr: float,
    ):
        # Load model to memory.
        self.interpreter = Interpreter(model_file)
        self.interpreter.allocate_tensors()

        self.input_details = self.interpreter
        _, self.width, self.height, _ = self.interpreter.get_input_details()[0]["shape"]
        self.output_details = self.interpreter.get_output_details()
        self.conf_thr = conf_thr

    def detect(self, img: np.ndarray, box_type="xywh") -> Tuple[np.ndarray]:
        """Detect objects.
        Returns:
           Tuple[np.ndarray]: The shape of each element is (25500, 4) (25500,) (25500,).
        """
        img = self.preprocess(img)
        output_data = self._detect(img)
        boxes, scores, class_idx = self.postprocess(output_data, box_type)
        return boxes, scores, class_idx

    def _detect(self, img: np.ndarray):
        """Inference."""
        self.interpreter.set_tensor(0, img)
        self.interpreter.invoke()
        output_data = self.interpreter.get_tensor(self.output_details[0]["index"])  # get tensor  x(1, 25200, 7)
        return output_data

    def preprocess(self, img: np.ndarray) -> np.ndarray:
        """Preprocess."""
        # Resize
        img = cv2.resize(img, (self.height, self.width))
        # BGR -> RGB
        img = img[:, :, [2, 1, 0]]
        # Normalize
        img = img / 255.0
        img = np.expand_dims(img, axis=0)
        return img.astype(np.float32)

    def postprocess(self, output_data, box_type: str) -> Tuple[np.ndarray]:
        """Postprocess."""
        output_data = output_data[0]
        # xywh
        boxes = output_data[..., :4]
        conf = output_data[..., 4:5]
        cls = np.argmax(output_data[..., 5:], axis=1).astype(np.float32).reshape(-1, 1)

        conf = np.squeeze(conf, axis=1)
        cls = np.squeeze(cls, axis=1)

        if box_type == "xyxy":
            # xywh -> xyxyx
            boxes = self.to_xyxy(boxes)

        # Filter by confidence threshold.
        idxs = np.where(conf > self.conf_thr)
        boxes = boxes[idxs]
        cls = cls[idxs]
        conf = conf[idxs]

        return boxes, conf, cls

    def to_xyxy(self, boxes: np.ndarray) -> np.ndarray:
        """Covert xywh to xyxy."""
        # (x, y) is cordinate fo the center of the box.
        x, y, w, h = boxes[..., 0], boxes[..., 1], boxes[..., 2], boxes[..., 3]
        boxes = np.array([x - w / 2, y - h / 2, x + w / 2, y + h / 2])
        # [4, n] -> [n, 4]
        boxes = boxes.transpose((1, 0))
        return boxes

max_stars_repo_name: ozora-ogino/tflite-human-tracking

max_issues_repo_name: ozora-ogino/tflite-human-tracking

max_forks_repo_name: ozora-ogino/tflite-human-tracking


text:
# Import networkx for graphs
import networkx as nx

# The interactive story is a directional graph
G = nx.DiGraph()

# Prorgram your story:

G.add_node("begin", narrative=open("res/begin.txt").read(), start=True)
G.add_node("left1", narrative=open("res/left1.txt").read())
G.add_node("center1", narrative=open("res/center1.txt").read())
G.add_node("right1", narrative=open("res/right1.txt").read())
G.add_node("fight", narrative=open("res/fight.txt").read())
G.add_node("runRight", narrative=open("res/runRight.txt").read())
G.add_node("runLeft", narrative=open("res/runLeft.txt").read())
G.add_node("tryFight", narrative=open("res/tryFight.txt").read())
G.add_node("tryRun", narrative=open("res/tryRun.txt").read())
G.add_node("riddle", narrative=open("res/riddle.txt").read())
G.add_node("wrong", narrative=open("res/wrong.txt").read())
G.add_node("noBeans", narrative=open("res/noBeans.txt").read())
G.add_node("turnRight", narrative=open("res/turnRight.txt").read())
G.add_node("exit", narrative=open("res/exit.txt").read())
G.add_node("exit2", narrative=open("res/exit2.txt").read())
G.add_node("backToBegin", narrative=open("res/backToBegin.txt").read())

G.add_edge("begin", "left1", text="Jimmy decides to take the left path. ")
G.add_edge("begin", "center1", text="Jimmy decides to go down the center path. ")
G.add_edge("begin", "right1", text="Jimmy decided to take the right path. ")
G.add_edge("left1", "fight", text="Jimmy musters up his courage and decides to fight the man! ")
G.add_edge("left1", "runRight", text="Jimmy decides to take the right path. ")
G.add_edge("left1", "runLeft", text="Jimmy takes the left path! ")
G.add_edge("runLeft", "tryFight", text="Jimmy turns around: with a sword in hand, there's no way he can lose! ")
G.add_edge("runLeft", "tryRun", text="Jimmy decides to keep running: after all, he doesn't even know how to use a sword! ")
G.add_edge("tryRun", "exit", text="Jimmy decides to walk towards the light. ")
G.add_edge("center1", "begin", text="Suddenly, Jimmy begins to feel sleepy. ")
G.add_edge("right1", "riddle", text="Jimmy decides to take the strange man up on his offer and gives up his shoes for the beans. ")
G.add_edge("right1", "noBeans", text="Jimmy decides to turn down the man's offer for a trade. ")
G.add_edge("riddle", "exit2", text="Jimmy decides to answer the riddle. ")
G.add_edge("riddle", "wrong", text="Jimmy decided not to answer the man's riddle. ")
G.add_edge("noBeans", "exit", text="Jimmy decides to take the left path.")
G.add_edge("noBeans", "backToBegin", text="Jimmy decides to turn right. ")
G.add_edge("backToBegin", "begin", text="Restart")

#
# Save the graph into the res/if.json
# (This code is required for your story to run in the workbook.)
#
with open("res/if.json", "w") as f:
    # Write the graph in a JSON-writable way:
    # See: https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.readwrite.json_graph.node_link_data.html#networkx.readwrite.json_graph.node_link_data
    from networkx.readwrite import json_graph
    data = json_graph.node_link_data(G)

    # Save the JSON-encoded graph to the file
    import json
    json.dump(data, f, indent=2)

reason: import networkx,from networkx


text:
# -*- coding: utf-8 -*-

import logging
import tempfile

import keras
import numpy as np

from mlprimitives.utils import import_object

LOGGER = logging.getLogger(__name__)

class Sequential(object):
    """A Wrapper around Sequential Keras models with a simpler interface."""

    def __getstate__(self):
        state = self.__dict__.copy()

        with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:
            keras.models.save_model(state.pop('model'), fd.name, overwrite=True)
            state['model_str'] = fd.read()

        return state

    def __setstate__(self, state):
        with tempfile.NamedTemporaryFile(suffix='.hdf5', delete=True) as fd:
            fd.write(state.pop('model_str'))
            fd.flush()

            state['model'] = keras.models.load_model(fd.name)

        self.__dict__ = state

    def _build_model(self, **kwargs):
        model = keras.models.Sequential()

        hyperparameters = self.hyperparameters.copy()
        hyperparameters.update(kwargs)

        for layer in self.layers:
            layer_kwargs = layer['parameters'].copy()
            for key, value in layer_kwargs.items():
                if isinstance(value, str):
                    layer_kwargs[key] = hyperparameters.get(value, value)

            model.add(layer['class'](**layer_kwargs))

        model.compile(loss=self.loss, optimizer=self.optimizer(), metrics=self.metrics)

        return model

    def __init__(self, layers, loss, optimizer, classification,
                 metrics=None, epochs=10, verbose=False, **hyperparameters):

        self.layers = list()
        for layer in layers:
            layer = layer.copy()
            layer['class'] = import_object(layer['class'])
            self.layers.append(layer)

        self.optimizer = import_object(optimizer)
        self.loss = import_object(loss)
        self.metrics = metrics

        self.epochs = epochs
        self.verbose = verbose
        self.classification = classification
        self.hyperparameters = hyperparameters

    def fit(self, X, y, **kwargs):
        self.model = self._build_model(**kwargs)

        if self.classification:
            y = keras.utils.to_categorical(y)

        self.model.fit(X, y, epochs=self.epochs, verbose=self.verbose)

    def predict(self, X):
        y = self.model.predict(X)

        if self.classification:
            y = np.argmax(y, axis=1)

        return y


text:
# -*- coding: utf-8 -*-
from flask import render_template, Flask, request, jsonify, Response

from threading import Thread

import tempfile
import time
import json
import re
import os

from time import gmtime, strftime

import pandas as pd
import numpy as np

from api import worker, scheduler, get_config, scale, NAME_TO_MODEL, NAME_TO_STATMODEL

app = Flask(__name__)

shed = scheduler()

@app.route('/')
@app.route('/check')
def check():
    config = get_config()
    return render_template('check.html', config=config)

@app.route('/get_result/<int:id>')
def get_result(id):
    model = shed.get_job(id)
    if model is None:
        response = jsonify({})
        response.status_code = 400

    elif model.result is None:
        response = jsonify({})
        response.status_code = 400
    else:
        tabledict = dict()
        for key in model.result:
            tabledict[key] = {'m*': model.result[key]['m*']}

        dataframe = pd.DataFrame(tabledict)
        table = dataframe.to_html(classes='data')

        response = jsonify(table=table)
        response.status_code = 200
    return response

@app.route('/result/<int:id>')
def result(id):
    model = shed.get_job(id)
    if model is None:
        status = 'Id not recognise.'
        tables = None
    elif model.result is None:
        status = 'Result not ready, please wait.'
        tables = None
    elif model.status is not None:
        status = model.status
        tables = None
    else:
        tabledict = dict()
        for key in model.result:
            tabledict[key] = {'m*': model.result[key]['m*']}

        dataframe = pd.DataFrame(tabledict).T
        tables = [dataframe.to_html(classes='data')]
        status = None

    return render_template('result.html', status=status, tables=tables, _id=id)

@app.route('/progress/<int:id>')
def progress(id):
    def generate():
        response = dict()
        response['persentage'] = 0.
        response['progress'] = dict()
        status = None
        while response['persentage'] < 100 and status is None:
            model = shed.get_job(id)
            if model is None:
                return 'data:{}\n\n'.format(json.dumps(response))
            elif model.progress is None:
                return 'data:{}\n\n'.format(json.dumps(response))
            else:
                status = model.status
                response['persentage'] = model.percentage()
                for key in model.progress:
                    if key not in response['progress']:
                        response['progress'][key] = dict()
                    response['progress'][key]['progress'] = 0.
                    response['progress'][key]['status'] = 'none'
                    response['progress'][key]['m*'] = 'none'

                    response['progress'][key]['status'] = model.progress[key]['status']
                    response['progress'][key]['progress'] = int(model.models[key].status())

                    if 'result' in model.progress[key]:
                        response['progress'][key]['m*'] = int(model.progress[key]['result']['m*'])

            text = 'data:{}\n\n'.format(json.dumps(response))
            yield text
            time.sleep(1)

    return Response(generate(), mimetype= 'text/event-stream')

@app.route('/checker', methods = ['GET', 'POST'])
def checker():
    status = 'Dataset is not downloaded.'
    _id = None
    tabledict = dict()
    info = []

    if request.method == 'POST':
        statmodel = NAME_TO_STATMODEL.get(request.form.get('statmodel', None), None)
        preprocess_type = request.form.get('preprocess', None)
        if statmodel is None:
            status = 'Linear model is not recognise: {}'.format(statmodel)
            return render_template('checker.html', status=status)

        default_config = get_config()
        config = dict()
        for key in default_config:
            if request.form.get('config[{}][use]'.format(key), 'off') == 'on':
                config[key] = dict()
                for param in default_config[key]:
                    default_config[key][param] = request.form.get('config[{}][{}]'.format(key, param), None)
                    if default_config[key][param]:
                        config[key][param] = float(default_config[key][param])

        status = 'Dataset is loaded but not processed, please try again.'

        f = request.files['file']
        uploaded_filename = f.filename
        if '.' not in uploaded_filename:
            status = 'Cant recognise file extension: "{}".'.format(uploaded_filename)
            return render_template('checker.html', status=status)

        extension = f.filename.rsplit('.')[-1]

        new_file, filename = tempfile.mkstemp('.{}'.format(extension))
        os.write(new_file, f.read())
        os.close(new_file)

        if extension == 'csv':
            status = 'Dataset is received but not processed, please try again.'

            dataset = pd.read_csv(filename)
            if 'y' not in dataset and 'answer' not in dataset:
                status = 'Target column "y" or column "answer" is not specified'
                return render_template('checker.html', status=status)
            else:
                target_column = 'y'
                if 'answer' in dataset:
                    target_column = 'answer'
                y = dataset[target_column].to_numpy()
                del dataset[target_column]
                X = dataset.to_numpy()

                if preprocess_type == 'scale':
                    X = scale(X)
                    if statmodel == statmodel = NAME_TO_STATMODEL.get('RegressionModel', None):
                    	y = scale(y)

                X = np.hstack((X, np.ones([len(X), 1])))

            job = worker(statmodel, config, X, y)
            if job.status is not None:
                return render_template('checker.html', status=job.status)
            _id = shed.add_job(job)

            thread = Thread(target=job.forward)
            thread.daemon = True
            thread.start()

            for key in job.progress:
                tabledict[key] = dict()
                info = ['progress', 'status', 'm*']
                for item in info:
                    tabledict[key][item] = 'none'
                tabledict[key]['progress'] = 0

            status = None
        else:
            status = 'Invalid file extension: "{}".'.format(extension)

        os.remove(filename)

    return render_template('checker.html', status=status, tabledict=tabledict, info=info, _id=_id)

max_stars_repo_name: andriygav/SampleSizeEstimation

max_issues_repo_name: andriygav/SampleSizeEstimation

max_forks_repo_name: andriygav/SampleSizeEstimation


text:
'''
Created on Nov 4, 2017

@author: kashefy
'''
import numpy as np
import tensorflow as tf

def rotation_rad(min_deg, max_deg, delta_deg):
    return np.deg2rad(np.arange(min_deg, max_deg+delta_deg, delta_deg)).tolist()

def rotation_ops(x,
                 min_deg, max_deg, delta_deg,
                 batch_sz,
                 name,
                 target_shape=[-1, 28, 28, 1]):
    reshape_op = tf.reshape(x, target_shape)
    rotations_rad = rotation_rad(min_deg, max_deg, delta_deg)
    rots_cur = np.random.choice(rotations_rad, batch_sz)
    rot_op = tf.contrib.image.rotate(reshape_op, rots_cur)
    flatten_op = tf.reshape(rot_op, [-1, x.get_shape()[-1].value],
                            name=name+'/flatten_rotation')
    return flatten_op, rots_cur

def scaled_dims(height, width, min_scale, max_scale, delta_scale):
    scales = np.arange(min_scale, max_scale+delta_scale, delta_scale)
    new_dims = [[max(1, int(s*height)), max(1, int(s*width))] for s in scales]
    return new_dims

def scale_translate_ops(x,
                 new_dims,
                 batch_sz,
                 name,
                 target_shape=[-1, 28, 28, 1]):
    reshape_op = tf.reshape(x, target_shape)
    _, height, width, num_channels = target_shape
    max_height, max_width = np.max(new_dims, axis=0)
    scales_idx_cur = np.random.choice(len(new_dims), batch_sz, replace=True)
    scale_ops = [tf.image.resize_images(tf.gather(reshape_op, tf.constant(img_idx)), new_dims[sc_idx]) for img_idx, sc_idx in enumerate(scales_idx_cur)]
    pad_ops, padding = pad_to_ops(scale_ops, max_height, max_width, [new_dims[sc_idx] for sc_idx in scales_idx_cur])
    flatten_op = tf.reshape(pad_ops, [-1, max_height*max_width*num_channels],
                            name=name+'/flatten_scale')
    bbox_top_left = [[ph[0], pw[0]] for ph, pw in padding]
    return flatten_op, [new_dims[s_idx] for s_idx in scales_idx_cur], bbox_top_left

def pad_to_ops(in_list, target_height, target_width, cur_dims):
    dim_diff = [[target_height-cur_h, target_width-cur_w] for cur_h, cur_w in cur_dims]
    padding = [[[int(np.floor(dh/2.)), int(np.ceil(dh/2.))], [int(np.floor(dw/2.)), int(np.ceil(dw/2.))]] for dh, dw in dim_diff]
    for idx, (ph, pw) in enumerate(padding):
        if ph[0] != ph[1] or pw[0] != pw[1]:
            flip_mode = np.random.randint(4)
            if flip_mode == 2: # vertical
                padding[idx][0] = padding[idx][0][::-1]
            elif flip_mode == 1: # horizontal
                padding[idx][1] = padding[idx][1][::-1]
            elif flip_mode == 2: # both
                padding[idx][0] = padding[idx][0][::-1]
                padding[idx][1] = padding[idx][1][::-1]
    pad_ops = [tf.pad(op, tf.constant([ph, pw, [0,0]]), mode="CONSTANT", constant_values=0) for op, (ph, pw) in zip(in_list, padding)]
    return pad_ops, padding

def gaussian_noise_op(in_, std):
    prefix = in_.name.replace(':', '-')
    noise = tf.random_normal(shape=tf.shape(in_),
                             mean=0.0, stddev=std,
                             dtype=tf.float32,
                             name='_'.join([prefix, 'noise']))
    return tf.add(in_, noise, name='_'.join([prefix, 'noise_add']))

max_stars_repo_name: kashefy/transform

max_issues_repo_name: kashefy/transform

max_forks_repo_name: kashefy/transform


text:
# -*- coding: utf-8 -*-
# Copyright 2018 The Blueoil Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================

__all__ = ["run_demo", "run_test"]

import os
import sys
import cv2
import time
import math
import threading
import collections
import numpy as np
from scipy import ndimage

sys.path.extend(["./lmnet", "/dlk/python/dlk"])

from lmnet.networks.optical_flow_estimation.flow_to_image import flow_to_image
from lmnet.datasets.optical_flow_estimation import (
    FlyingChairs, ChairsSDHom
)

def init_camera(camera_height, camera_width, device_id):
    cap = cv2.VideoCapture(device_id)
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, camera_width)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, camera_height)
    cap.set(cv2.CAP_PROP_FPS, 10)
    return cap

def rescale_frame(frame, ratio):
    height = int(frame.shape[0] * ratio)
    width = int(frame.shape[1] * ratio)
    shape = (width, height)
    return cv2.resize(frame, shape, interpolation=cv2.INTER_AREA)

def run_demo(func_inference, func_args=[], func_kwargs={},
             diff_step=5, window_name="output", full_screen=True,
             movie_path=None, demo_name="output", device_id=0):
    # initializing worker and variables
    store_num = diff_step + 10
    cap = init_camera(480, 640, device_id=device_id)
    time_list = collections.deque(maxlen=5)
    frame_list = collections.deque(maxlen=store_num)
    image_size = (384, 512, 3)
    input_image = np.zeros(
        (1, *image_size[:2], image_size[-1] * 2), dtype=np.uint8)
    output_image = np.zeros(image_size, dtype=np.uint8)
    display_image = np.zeros(
        (9 * 64, 16 * 64, 3), dtype=np.uint8)
    for _ in range(store_num):
        frame_list.append(np.zeros(image_size).astype(np.uint8))

    def _get_frame():
        while True:
            begin = time.time()
            res, frame = cap.read()
            assert res, "Something wrong occurs with camera!"
            frame_list.append(rescale_frame(frame[:, ::-1, :], 0.8))
            time.sleep(max(0.0, 1.0 / 30 - (time.time() - begin)))

    def _get_output():
        while True:
            try:
                begin = time.time()
                input_image[0, ..., :3] = frame_list[-diff_step]
                input_image[0, ..., 3:] = frame_list[-1]
                output_image[:] = func_inference(
                    input_image, *func_args, **func_kwargs)
                time_list.append(time.time() - begin)
            except ValueError:
                pass

    t1 = threading.Thread(target=_get_frame)
    t1.setDaemon(True)
    t1.start()
    t2 = threading.Thread(target=_get_output)
    t2.setDaemon(True)
    t2.start()

    size_h1 = int(output_image.shape[0] * 1.5)
    size_w1 = int(output_image.shape[1] * 1.5)
    size_w2 = int(size_w1 / 3)
    size_h2 = int(size_h1 / 3)
    color_map = np.dstack(
        np.meshgrid(
            np.linspace(-15, 15, size_w2),
            np.linspace(-10, 10, size_h2)))
    color_map = flow_to_image(-color_map[..., [1, 0]])
    display_image[-size_h2:, -size_w2:] = color_map

    if full_screen:
        cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
        cv2.setWindowProperty(
            window_name, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)

    if movie_path is not None:
        from skvideo.io import FFmpegWriter
        movie_shape = display_image.shape[:2]
        out = FFmpegWriter(
            movie_path,
            inputdict={
                '-r': str(10), '-s': '{}x{}'.format(*movie_shape[::-1])},
            outputdict={
                # '-r': str(10),
                # '-c:v': 'libx264',
                # '-crf': str(17),
                # '-preset': 'ultrafast',
                # '-pix_fmt': 'yuv444p'
            }
        )

    def add_text_info():
        # FPS infomation
        fps = 1 / np.mean(time_list)
        fps_text = "{} (FPS: {:.2f})".format(demo_name, fps)
        cv2.putText(
            display_image, fps_text, (10, 25),
            cv2.FONT_HERSHEY_PLAIN, 1.6, (0, 0, 0), 2)

        # title_1
        cv2.putText(
            display_image, "input1", (size_w1 + 10, 25),
            cv2.FONT_HERSHEY_PLAIN, 1.6, (255, 255, 255), 2)

        cv2.putText(
            display_image, "input2", (size_w1 + 10, size_h2 + 25),
            cv2.FONT_HERSHEY_PLAIN, 1.6, (255, 255, 255), 2)

        cv2.putText(
            display_image, "color coding", (size_w1 + 10, 2 * size_h2 + 25),
            cv2.FONT_HERSHEY_PLAIN, 1.6, (0, 0, 0), 2)

    while True:
        begin = time.time()
        _pre = frame_list[-diff_step].astype(np.float)
        _post = frame_list[-1].astype(np.float)
        display_image[:size_h1, :size_w1] = ndimage.zoom(
            np.array(output_image), [1.5, 1.5, 1], order=0)
        display_image[:size_h2, size_w1:] = _pre[::2, ::2]
        display_image[size_h2:2 * size_h2, size_w1:] = _post[::2, ::2]
        add_text_info()
        cv2.imshow(window_name, display_image)
        if movie_path is not None:
            out.writeFrame(display_image[..., ::-1])
            time.sleep(max(0.0, 0.1 - (time.time() - begin)))
            print(time.time() - begin)
        key = cv2.waitKey(2)
        if key == 27:
            break

def run_test(func_inference, func_args=[], func_kwargs={}):
    dataset = FlyingChairs(
        subset="validation", validation_rate=0.1, validation_seed=2019)

    def calc_epe(in_flow, out_flow):
        # assert in_flow.shape == out_flow.shape
        return np.mean(np.sqrt(
            np.sum((in_flow - out_flow) ** 2, axis=3))
        )

    epe_list = []
    for _i in range(100):
        input_image, target_flow = dataset[_i]
        output_flow = func_inference(
            input_image[np.newaxis], *func_args, **func_kwargs)
        epe_list.append(calc_epe(output_flow, target_flow))
        print("epe: {:.3f}".format(epe_list[-1]))

    print("mean: {:.3f}, std: {:.3f}".format(
        np.mean(epe_list), np.std(epe_list)))

reason: import numpy,from scipy


text:
# mlp-attention_03
# {'val_loss': [
# 0.8321127831733865, 0.8149616334763244, 0.8080661035819032, 0.8043228179784596, 0.8002183685173205,
# 0.7972857836676045, 0.793001569427853, 0.7924909000240787, 0.7921270519020359, 0.7874714549603654,
# 0.7854632683025837, 0.7836462063538413, 0.7816860973027518, 0.7811260843474584, 0.7826188791772042,
# 0.7800090469637393, 0.7793066135596607, 0.7784056678202643, 0.7783930039180803, 0.7800617894507442
# ],
# 'val_mean_squared_error': [
# 0.8321127831733865, 0.8149616334763244, 0.8080661035819032, 0.8043228179784596, 0.8002183685173205,
# 0.7972857836676045, 0.793001569427853, 0.7924909000240787, 0.7921270519020359, 0.7874714549603654,
# 0.7854632683025837, 0.7836462063538413, 0.7816860973027518, 0.7811260843474584, 0.7826188791772042,
# 0.7800090469637393, 0.7793066135596607, 0.7784056678202643, 0.7783930039180803, 0.7800617894507442
# ],
# 'loss': [
# 1.2129996511072807, 0.8089207498588716, 0.7925854784378688, 0.7837273693483895, 0.7770178057682471,
# 0.7712283459233497, 0.7657942494154589, 0.7617266936164807, 0.7575358347528671, 0.7536763259070509,
# 0.7502327760970651, 0.746288952394418, 0.7429369581147989, 0.7393621096899098, 0.7362319128615927,
# 0.73318466221791, 0.7307932809421275, 0.7283218859352288, 0.726398458615269, 0.7244027269489683
# ],
# 'mean_squared_error': [
# 1.2129996511072807, 0.8089207498588716, 0.7925854784378688, 0.7837273693483895, 0.7770178057682471,
# 0.7712283459233497, 0.7657942494154589, 0.7617266936164807, 0.7575358347528671, 0.7536763259070509,
# 0.7502327760970651, 0.746288952394418, 0.7429369581147989, 0.7393621096899098, 0.7362319128615927,
# 0.73318466221791, 0.7307932809421275, 0.7283218859352288, 0.726398458615269, 0.7244027269489683
# ]}
# 0.7800090469637393
# 0.8832

from surprise import Dataset
from sklearn.model_selection import train_test_split
from keras.models import Model
from keras.layers import Dense, Embedding, Input, concatenate, Flatten, multiply
from keras import optimizers, losses, metrics
import numpy as np

def get_data():
    data = Dataset.load_builtin('ml-1m')
    data_set = data.build_full_trainset()
    # print(data_set.n_items)
    # print(data_set.n_users)
    x = []
    y = []
    for u, i, r in data_set.all_ratings():
        x.append((u, i))
        y.append(r)
    X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
    return X_train, X_test, y_train, y_test

class Mlp:
    def __init__(self):
        self.items = 3706
        self.users = 6040

    def data(self):
        X_train, X_test, self.y_train, self.y_test = get_data()
        self.X_train_u = []
        self.X_train_i = []
        self.X_test_u = []
        self.X_test_i = []
        for u, i in X_train:
            self.X_train_u.append(u)
            self.X_train_i.append(i)

        for u, i in X_test:
            self.X_test_u.append(u)
            self.X_test_i.append(i)

    def inference(self):
        u_inputs = Input(shape=(1,), name='u_input_op')
        i_inputs = Input(shape=(1,), name='i_input_op')
        u_embedding = Embedding(input_dim=self.users, output_dim=32, input_length=1, name='u_embedding_op')(u_inputs)
        i_embedding = Embedding(input_dim=self.items, output_dim=32, input_length=1, name='i_embedding_op')(i_inputs)
        u_flatten = Flatten(name='u_flatten_op')(u_embedding)
        i_flatten = Flatten(name='i_flatten_op')(i_embedding)
        attention_u = Dense(32, activation='softmax', name='attention_vec_u')(u_flatten)
        attention_u_mul = multiply([u_flatten, attention_u], name='attention_mul_u')
        attention_i = Dense(32, activation='softmax', name='attention_vec_i')(i_flatten)
        attention_i_mul = multiply([i_flatten, attention_i], name='attention_mul_i')
        u_i_concat = concatenate([attention_u_mul, attention_i_mul], name='concat_op')
        attention_u_i = Dense(64, activation='softmax', name='attention_vec_u_i')(u_i_concat)
        attention_u_i_mul = multiply([u_i_concat, attention_u_i], name='attention_mul_u_i')
        dense_1 = Dense(32, activation='relu', name='dense_1_op')(attention_u_i_mul)
        dense_2 = Dense(16, activation='relu', name='dense_2_op')(dense_1)
        dense_3 = Dense(8, activation='relu', name='dense_3_op')(dense_2)
        output = Dense(1, name='output_op')(dense_3)
        self.model = Model(inputs=[u_inputs, i_inputs], outputs=output)

    def compile(self):
        self.model.summary()
        self.model.compile(
            optimizer=optimizers.Adam(),
            loss=losses.mean_squared_error,
            metrics=[metrics.mean_squared_error]
        )

    def train(self):
        self.his = self.model.fit(
            x=[np.asarray(self.X_train_u), np.asarray(self.X_train_i)],
            y=np.asarray(self.y_train),
            validation_data=([np.asarray(self.X_test_u), np.asarray(self.X_test_i)], np.asarray(self.y_test)),
            epochs=20,
            batch_size=256
        )

    def predict(self):
        self.model.predict()

    def build(self):
        self.data()
        self.inference()
        self.compile()
        self.train()

if __name__ == '__main__':
    mlp_model = Mlp()
    mlp_model.build()
    print(mlp_model.his.history)

max_stars_repo_name: enningxie/RecSys

max_issues_repo_name: enningxie/RecSys

max_forks_repo_name: enningxie/RecSys


text:
# pylint: disable=no-member, invalid-name, too-many-instance-attributes
"""
load raw EWD into memory
"""
# Copyright (c) Benyuan Liu. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
import os
from struct import unpack
import numpy as np

class EWD:
    """EIT raw/waveform data"""

    def __init__(self, file_name):
        """
        RAW data (.EWD) contains only data
        """
        self.file_name = file_name
        self.file_size = os.path.getsize(file_name)
        # 256 measurement waveforms, 128 points per wave, 2 bytes per point
        self.n_wave = 256
        self.n_point = 128
        self.n_data = self.n_wave * self.n_point
        self.frame_size = self.n_data * 2  # signed short
        self.n_frame = int(self.file_size / self.frame_size)
        self.tot_data = int(self.file_size / 2)

        raw = self.load_raw()
        self.wave = self.demodulate(raw)
        scale = 4096 / 65536 / 29.9 * 1000 / 1250
        self.data = (self.wave[:, :256] + 1j * self.wave[:, 256:]) * scale

    def load_raw(self):
        """load raw data"""
        raw = np.zeros((self.n_frame, self.n_data), dtype=np.int)
        with open(self.file_name, "rb") as fh:
            for i in range(self.n_frame):
                d = fh.read(self.frame_size)
                raw[i] = unpack("{}h".format(self.n_data), d)

        return raw

    def demodulate(self, raw):
        """demodulate raw data into [re, im]"""
        wave = np.zeros((self.n_frame, 2 * self.n_wave), dtype=np.double)
        sin_rom = np.sin(2.0 * np.pi * np.arange(self.n_point) / self.n_point)
        cos_rom = np.cos(2.0 * np.pi * np.arange(self.n_point) / self.n_point)
        for i in range(self.n_frame):
            dw = raw[i].reshape(self.n_wave, -1)
            wave_re = np.sum(dw * sin_rom, axis=1)
            wave_im = np.sum(dw * cos_rom, axis=1)
            wave[i] = np.concatenate([wave_re, wave_im]) / (self.n_point / 2.0)

        return wave

    def to_erd(self, src, dst):
        """combine ERD and demodulate EWD to a new file"""
        file_size = os.path.getsize(src)
        header_size = 1024
        data_size = 4096  # 2*256 doubles
        frame_size = header_size + data_size
        n_frame = int(file_size / frame_size)
        assert n_frame == self.n_frame
        with open(src, "rb") as fr, open(dst, "wb") as fw:
            for i in range(n_frame):
                h = fr.read(header_size)
                fr.read(data_size)  # skip data
                d = self.wave[i].tobytes()
                fw.write(h + d)

max_stars_repo_name: fzouari/pyEIT

max_issues_repo_name: fzouari/pyEIT

max_forks_repo_name: fzouari/pyEIT


text:
# @version 1.3.0

import pandas as pd
import numpy as np
import os.path
from os import rename
from scipy.stats import ortho_group
import pyHilbertSort as m

path = input("Please enter the mesh file path : ")
while(not os.path.isfile(path)):
    print("File not accessible")
    path = input("Please enter the mesh file path : ")

M = pd.read_csv(path ,sep = ',',header = None)

outputfile = input("Please enter the path to the output file: ")
while(not isinstance(outputfile, str)):
    print("Output File not accessible")
    path = input("Please enter the path to the output file: ")

globnb = input("Please enter the total number of viruses : ")
while(not globnb.isdigit()):
    print("Non valid entrie")
    globnb = input("Please enter the total number of viruses : ")

nb = input("Please enter the number of viruses per sub-population : ")
while(not nb.isdigit()) and (nb > globnb):
    print("Non valid entrie")
    nb = input("Please enter the number of viruses per sub-population : ")

ordering = input("Do you prefer to include Hilbert ordering (y/n) : ")
while(ordering != 'y') and (ordering != 'n'):
    print("Non valid entrie")
    ordering = input("Do you prefer to include Hilbert ordering (y/n) : ")

#Local population
nb = int(nb)
Rmax = nb
Rmin = 0
stp  = 0.25

#Global population
globnb = int(globnb)
popnb  = int(np.ceil(globnb/nb))
globRmax = np.ceil((popnb+1)**(1/3))
globRmin = 0
globstp  = (Rmax-Rmin)*stp

center = np.unique(np.random.randint(globRmin, globRmax, size=(popnb,3))*globstp, axis=0)
while (len(center) < popnb):
    center = np.vstack((center,np.random.randint(globRmin, globRmax, size=(1,3))*globstp))
    center = np.unique(center, axis=0)

newPath = "singleviursdata/VirusPopulation.txt"
countt=0
for c in center :
    T = ortho_group.rvs(3)

    pos = np.unique(np.random.randint(Rmin, Rmax, size=(nb,3))*stp, axis=0)
    while (len(pos) < nb):
        pos = np.vstack((pos,np.random.randint(Rmin, Rmax, size=(1,3))*stp))
        pos = np.unique(pos, axis=0)

    np.savetxt(newPath, (np.dot(T, (M.values).T ).T + pos[0] + c ), delimiter=",", fmt='%1.4e')

    f = open(newPath,'ab')
    for i in range(1,nb):
        T = ortho_group.rvs(3)
        np.savetxt(f, (np.dot(T, (M.values).T ).T + pos[i] + c), delimiter=",", fmt='%1.4e')
    f.close()
    rename(newPath, outputfile+str(countt)+".txt")
#    rename(newPath,"data/VirusPopulation"+str(nb*M.shape[0])+str(c)+".txt")

    if(ordering == "y"):
        #ordPath = "data/VirusPopulation"+str(nb*M.shape[0])+str(c)+".txt"
        ordPath=outputfile+str(countt)+".txt"
        pts = pd.read_csv(ordPath ,sep = ',',header = None)
        ptsSort,idxSort = m.hilbertSort(3, pts.values)
        np.savetxt(ordPath, ptsSort, delimiter=",", fmt='%1.4e')
    countt=countt+1

max_stars_repo_name: wawando/stars-h

max_issues_repo_name: wawando/stars-h

max_forks_repo_name: wawando/stars-h


text:
"""This module contains the class that defines the interaction between
different modules that govern agent's behavior.
"""
import numpy as np
from perception import BethePerception
from misc import ln, softmax

class BayesianPlanner(object):

    def __init__(self, perception, action_selection, policies,
                 prior_states = None, prior_policies = None,
                 trials = 1, T = 10, number_of_states = 6,
                 number_of_policies = 10):

        #set the modules of the agent
        self.perception = perception
        self.action_selection = action_selection

        #set parameters of the agent
        self.nh = number_of_states #number of states
        self.npi = number_of_policies #number of policies

        if policies is not None:
            self.policies = policies
        else:
            #make action sequences for each policy
            self.policies = np.eye(self.npi, dtype = int)

        self.actions = np.unique(self.policies)
        self.na = len(self.actions)

        if prior_states is not None:
            self.prior_states = prior_states
        else:
            self.prior_states = np.ones(self.nh)
            self.prior_states /= self.prior_states.sum()

        if prior_policies is not None:
            self.prior_policies = prior_policies
        else:
            self.prior_policies = np.ones(self.npi)/self.npi

        #set various data structures
        self.actions = np.zeros((trials, T), dtype = int)
        self.posterior_states = np.zeros((trials, T, self.nh, T, self.npi))
        self.posterior_policies = np.zeros((trials, T, self.npi))
        self.observations = np.zeros((trials, T), dtype = int)

    def reset_beliefs(self, actions):
        self.actions[:,:] = actions
        self.posterior_states[:,:,:] = 0.
        self.posterior_policies[:,:,:] = 0.

        self.perception.reset_beliefs()
        self.planning.reset_beliefs()
        self.action_selection.reset_beliefs()

    def update_beliefs(self, tau, t, observation, response):
        self.observations[tau,t] = observation
        #update beliefs about hidden states
        if t == 0:
            control = None
            prior_states = self.prior_states.copy()
            prior_policies = self.prior_policies
        else:
            control = self.actions[tau, t-1]
            prior_states = self.posterior_states[tau, t-1]
            prior_policies = self.posterior_policies[tau, t-1]

        #update beliefs about states
        if isinstance(self.perception, BethePerception):
            self.posterior_states[tau, t] = self.perception.update_beliefs_states(
                                             tau, t,
                                             observation,
                                             self.policies,
                                             prior_policies)

        else:
            self.posterior_states[tau, t] = self.perception.update_beliefs_states(
                                             observation,
                                             control,
                                             prior_states)

        #update beliefs about policies
        self.posterior_policies[tau, t] = self.perception.update_beliefs_policies()

    def generate_response(self, tau, t):

        #get response probability
        posterior_policies = self.posterior_policies[tau, t]
        non_zero = posterior_policies > 0
        controls = self.policies[:, t][non_zero]
        posterior_policies = posterior_policies[non_zero]

        self.actions[tau, t] = self.action_selection.select_desired_action(tau,
                                        t, posterior_policies, controls)

        return self.actions[tau, t]

class BayesianMFPlanner(object):

    def __init__(self, perception, planning, action_selection,
                 prior_beliefs = None, prior_states = None,
                 prior_policies = None, policies = None,
                 trials = 1, T = 10, number_of_states = 6,
                 number_of_policies = 10):

        #set the modules of the agent
        self.perception = perception
        self.planning = planning
        self.action_selection = action_selection

        #set parameters of the agent
        self.nh = number_of_states #number of states
        self.npi = number_of_policies #number of policies

        if policies is not None:
            self.policies = policies
        else:
            #make action sequences for each policy
            self.policies = np.eye(self.npi, dtype = int)

        self.actions = np.unique(self.policies)
        self.na = len(self.actions)

        if prior_beliefs is not None:
            self.prior_beliefs = prior_beliefs
        else:
            self.prior_beliefs = np.zeros((self.nh, T, self.npi))
            if prior_states is not None:
                self.prior_beliefs[:, 0, :] = 1/self.nh
#                self.prior_beliefs[:, 0, :] = prior_states[:, np.newaxis]
            else:
                self.prior_beliefs[:,0,:] = 1./self.nh

            for pi, cstates in enumerate(policies):
                for t, u in enumerate(cstates):
                    p = self.perception.generative_model_states[:,:,u]\
                        .dot(self.prior_beliefs[:,t,pi])
                    p[:] = 1.
#                    p[p>1e-10] = 1.
                    p /= p.sum()
                    self.prior_beliefs[:, t+1, pi] = p[:]

        if prior_policies is not None:
            self.prior_policies = prior_policies
        else:
            self.prior_policies = np.ones(self.npi)/self.npi

        #set various data structures
        self.actions = np.zeros((trials, T), dtype = int)
        self.posterior_states = np.zeros((trials, T, self.nh, T, self.npi))
        self.posterior_policies = np.zeros((trials, T, self.npi))
        self.observations = np.zeros((trials, T), dtype = int)

    def update_beliefs(self, tau, t, observation, response):
        self.observations[tau,t] = observation
        #update beliefs about hidden states
        if t == 0:
#            control = None
            prior_states = self.prior_beliefs.copy()
            prior_policies = self.prior_policies.copy()
        else:
#            control = self.actions[tau, t-1]
            prior_states = self.prior_beliefs.copy() #self.posterior_states[tau, t-1].copy()
            prior_policies = self.posterior_policies[tau, t-1].copy()

        #update beliefs about states
        self.posterior_states[tau, t], neg_fe_pi = self.perception.update_beliefs_states(
                                             tau, t,
                                             observation,
                                             self.policies,
                                             prior_states,
                                             prior_policies,)

        #update beliefs about policies
        self.posterior_policies[tau, t] = \
            self.perception.update_beliefs_policies()

    def generate_response(self, tau, t):

        #get response probability
        posterior_policies = self.posterior_policies[tau, t]
        non_zero = posterior_policies >= 1e-6
        controls = self.policies[:, t][non_zero]
        posterior_policies = posterior_policies[non_zero]
        posterior_policies /= posterior_policies.sum()

        self.actions[tau, t] = self.action_selection.select_desired_action(tau,
                                        t, posterior_policies, controls)

        return self.actions[tau, t]

max_stars_repo_name: SSchwoebel/ActiveInferenceBPBethe

max_issues_repo_name: SSchwoebel/ActiveInferenceBPBethe

max_forks_repo_name: SSchwoebel/ActiveInferenceBPBethe


text:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from pandas import DataFrame

def get_training_metrics(history):
    # This is needed depending on if you used the pretrained model or you trained it yourself
    if not isinstance(history, DataFrame):
        history = history.history_data

    acc = history['sparse_categorical_accuracy']
    val_acc = history['val_sparse_categorical_accuracy']

    loss = history['loss']
    val_loss = history['val_loss']

    return acc, val_acc, loss, val_loss

def plot_train_eval(history: pd.DataFrame) -> None:
    acc, val_acc, loss, val_loss = get_training_metrics(history)

    acc_plot = pd.DataFrame({"training accuracy": acc, "evaluation accuracy": val_acc})
    acc_plot = sns.lineplot(data=acc_plot)
    acc_plot.set_title('training vs evaluation accuracy')
    acc_plot.set_xlabel('epoch')
    acc_plot.set_ylabel('sparse_categorical_accuracy')
    plt.show()

    loss_plot = pd.DataFrame({"training loss": loss, "evaluation loss": val_loss})
    loss_plot = sns.lineplot(data=loss_plot)
    loss_plot.set_title('training vs evaluation loss')
    loss_plot.set_xlabel('epoch')
    loss_plot.set_ylabel('loss')
    plt.show()

if __name__ == '__main__':
    rng = np.random.default_rng()

    history_data = pd.DataFrame(
        data=rng.random((100, 4)),
        columns=['loss', 'sparse_categorical_accuracy', 'val_loss', 'val_sparse_categorical_accuracy']
    )
    plot_train_eval(history_data)

max_stars_repo_name: jmetzz/ml-laboratory

max_issues_repo_name: jmetzz/ml-laboratory

max_forks_repo_name: jmetzz/ml-laboratory


text:
import csv
import os
import cv2
import numpy as np
# Setting up Tensorflow
import tensorflow as tf
# Setting up Keras
from keras.layers import Flatten, Dense, Lambda, Activation, MaxPooling2D, Cropping2D, Dropout
from keras.layers.convolutional import Convolution2D
from keras.models import Sequential, Model
from PIL import Image
from scipy import ndimage
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from pathlib import PurePosixPath

lines = []
images = []
measurements = []
number_of_epochs = 5
csv_file ='../data/driving_log.csv'
steering_angles = []

def process_image(image):
    # do some pre processing on the image
    #Contrast limited adaptive histogram equalization (CLAHE) is used for improve the visibility level of foggy image or video
    # TODO:
    bw_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    clahe = cv2.createCLAHE(clipLimit = 5)
    final_image = clahe.apply(bw_image) + 30
    return final_image

with open('../data/driving_log.csv') as csvfile:
    reader = csv.reader(csvfile)

    next(reader)   ## skip csv header

    for line in reader:
        lines.append(line)

#Reading in image data from IMG folder and measurement
for line in lines:
    for i in range(3):
        source_path = line[0]
        filename = source_path.split('/')[-1]
        current_path = '../data/IMG/' + filename
        image = ndimage.imread(current_path)
        images.append(image)
        measurement = float(line[3])
        measurements.append(measurement)

#Flipping Images And Steering Measurements this augments the training size by 3
augmented_images, augmented_measurements = [], []
for image, measurement in zip(images, measurements):
    augmented_images.append(image)
    augmented_measurements.append(measurement)
    augmented_images.append(cv2.flip(image,1))
    augmented_measurements.append(measurement*-1.0)

X_train = np.array(augmented_images)
y_train = np.array(augmented_measurements)

# My Neural Network that using transfer learning from Nvidia's "End to End Learning for Self-Driving Cars" (https://images.nvidia.com/content/tegra/automotive/images/2016/solutions/pdf/end-to-end-dl-using-px.pdf)
# can be found below
model = Sequential()

# set up lambda layer
model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(160,320,3)))

# crop out parts of the image where trees are(top) and hood of car(bottom) with cropping2D layer
model.add(Cropping2D(cropping=((70,25), (0,0))))

# starts with a convolutional and maxpooling layers
model.add(Convolution2D(24,5,5,subsample=(2, 2),activation="relu"))
#MaxPooling was added to the network to fix issue of going off track after the bridge
model.add(MaxPooling2D(pool_size=(2, 2), strides=(1, 1)))

# then a convolutional and maxpooling layer
model.add(Convolution2D(36,5,5,subsample=(2, 2),activation="relu"))

# then a convolutional
model.add(Convolution2D(48,5,5,subsample=(2, 2),activation="relu"))

# then another convolutional
model.add(Convolution2D(64,3,3,activation="relu"))

model.add(Convolution2D(64,3,3,activation="relu"))

model.add(Flatten())

# Next, four fully connected layers
model.add(Dense(100))
#model.add(Activation(relu))
model.add(Dense(50))
#model.add(Activation(relu))
model.add(Dense(10))
#model.add(Activation(relu))
model.add(Dense(1))
#model.add(Activation(relu))

model.summary()

model.compile(loss='mse', optimizer='adam')
model.fit(X_train, y_train, validation_split=0.2, shuffle=True, epochs=number_of_epochs)
model.save('model.h5')

max_stars_repo_name: moeec/Behavioral-Cloning

max_issues_repo_name: moeec/Behavioral-Cloning

max_forks_repo_name: moeec/Behavioral-Cloning