text:
import numpy as np
from matplotlib import pylab as plt

#ILE_fmins = [10., 15., 20., 25., 30., 35., 40.]
ILE_data = [8664., 2172., 2674., 2174., 2162., 1892., 1890.] # runtimes in s (avg of 2 runs)
ILE_fmins = [15., 20., 25., 30., 35., 40.]
T1_data = [2195., 2679., 2124., 2189., 1885., 1861.]
EOB_data = [2202., 2407., 1859., 1847., 1584., 1564.]

rt_40 = 1861. # runtime of Bayesian PE using fmin=40.

def Bayes_runtime(fmin, rt_40):
    """
    Compute runtime of Bayesian PE for a min. freq. fmin
    given the rt_40 as the runtime when fmin=40 Hz.
    """
    return rt_40 * (fmin / 40.)**(-8./3.)

fmins = np.arange(10., 40., 0.1)

plt.semilogy(fmins, Bayes_runtime(fmins, rt_40), 'k-', label="Waveform-limited scaling")
plt.scatter(ILE_fmins, T1_data, label="rapid PE (T1)", c='b')
plt.scatter(ILE_fmins, EOB_data, label="rapid PE (EOBNRv2)", marker='x', c='r')
plt.title('PE Runtimes for $(1.55+1.23) M_\odot$ BNS signal')
plt.legend()
plt.xlabel("min. freq. (Hz)")
plt.ylabel("runtime (s)")
plt.show()

max_stars_repo_name: spfanning/research-projects-RIT

max_issues_repo_name: spfanning/research-projects-RIT

max_forks_repo_name: spfanning/research-projects-RIT


text:
import os
import pydicom
import PIL.Image as Image
import numpy as np
from torch.utils.data import Dataset
import torchvision.transforms as transforms
import cv2

class Chaos(Dataset):

    def __init__(self, data_dir, mode):

        base_dir = os.path.join(data_dir, 'CT')
        patient_numbers = os.listdir(base_dir)
        patient_numbers.sort(key=lambda x: eval(x))
        image_dirs = [os.path.join(base_dir, index, 'DICOM_anon') for index in patient_numbers]
        mask_dirs = [os.path.join(base_dir, index, 'Ground') for index in patient_numbers]

        # 5-fold cross validation
        fold_size = len(image_dirs) // 5
        k=1
        val_list = range(k*fold_size, (k+1)*fold_size)

        self.train_image_paths = []
        self.val_image_paths = []
        self.train_mask_paths = []
        self.val_mask_paths = []

        for i in range(len(image_dirs)):

            image_files = os.listdir(image_dirs[i])
            image_files.sort(key=image_name_key)
            mask_files = os.listdir(mask_dirs[i])
            mask_files.sort(key=mask_name_key)

            if i in val_list:
                for image_file in image_files:
                    self.val_image_paths.append(os.path.join(image_dirs[i], image_file))
                for mask_file in mask_files:
                    self.val_mask_paths.append(os.path.join(mask_dirs[i], mask_file))
            else:
                for image_file in image_files:
                    self.train_image_paths.append(os.path.join(image_dirs[i], image_file))
                for mask_file in mask_files:
                    self.train_mask_paths.append(os.path.join(mask_dirs[i], mask_file))

        self.mean = [0.3667]
        self.std = [0.3533]
        self.mode = mode

    def __getitem__(self, i):

        if self.mode == 'train':
            img_path, mask_path = self.train_image_paths[i], self.train_mask_paths[i]
        elif self.mode == 'val':
            img_path, mask_path = self.val_image_paths[i], self.val_mask_paths[i]

        mask = Image.open(mask_path).convert('L')
        mask = transforms.Resize((256,256), interpolation=Image.NEAREST)(mask)
        mask_tensor = transforms.ToTensor()(mask)

        dataset = pydicom.dcmread(img_path)
        HU_img = dataset.RescaleSlope * dataset.pixel_array + dataset.RescaleIntercept
        MIN_BOUND = -1000.0
        MAX_BOUND = 400.0
        # change the scope to 0-1
        img = (HU_img - MIN_BOUND) / (MAX_BOUND - MIN_BOUND)
        img[img>1] = 1.
        img[img<0] = 0
        # resize
        img = cv2.resize(img, (256,256), interpolation=cv2.INTER_CUBIC)
        img_tensor = transforms.ToTensor()(img)
        img_tensor = transforms.Normalize(mean=self.mean, std=self.std)(img_tensor)

        return {'image': img_tensor, 'mask': mask_tensor}

    def __len__(self):
        if self.mode == 'train':
            return len(self.train_image_paths)
        elif self.mode == 'val':
            return len(self.val_image_paths)

    def cal_mean_std(self):
        image_paths = self.train_image_paths + self.val_image_paths
        image_array = np.zeros((len(image_paths), 512, 512))
        for i in range(len(image_paths)):
            dataset = pydicom.dcmread(image_paths[i])
            HU_img = dataset.RescaleSlope * dataset.pixel_array + dataset.RescaleIntercept
            MIN_BOUND = -1000.0
            MAX_BOUND = 400.0
            img = (HU_img - MIN_BOUND) / (MAX_BOUND - MIN_BOUND)
            img[img>1] = 1.
            img[img<0] = 0
            image_array[i,:,:] = img
        return np.mean(image_array), np.std(image_array)

def image_name_key(image_name):
    if image_name[0] == 'i':
        return int(image_name[1:5])
    elif image_name[0:3] == 'IMG':
        return int(image_name[-8:-4])

def mask_name_key(mask_name):
    return int(mask_name[-7:-4])

ext: py

reason: import numpy


text:
import numpy as np
import pandas as pd
import mne
from mne.datasets.sleep_physionet._utils import _check_subjects, _fetch_one, _data_path

BASE_URL = 'https://physionet.org/physiobank/database/sleep-edfx/sleep-cassette/'
AGE_SLEEP_RECORDS = 'data/SC-index.csv'

EVENT_ID = {
    'Sleep stage W': 0,
    'Sleep stage 1': 1,
    'Sleep stage 2': 2,
    'Sleep stage 3/4': 3,
    'Sleep stage R': 4,
    'Sleep stage ?': 5,
    'Movement time': 6
}

ANNOTATIONS_EVENT_ID = {
    'Sleep stage W': 0,
    'Sleep stage 1': 1,
    'Sleep stage 2': 2,
    'Sleep stage 3': 3,
    'Sleep stage 4': 3,
    'Sleep stage R': 4,
    'Sleep stage ?': 5,
    'Movement time': 6
}

THIRTY_MINS_IN_SECS = 1800
EPOCH_DURATION = 30. # in seconds

"""
Code extracted from `https://github.com/mne-tools/mne-python/blob/maint/0.19/mne/datasets/sleep_physionet/age.py`
to change subject range from 20 to 82, in order to have access to the full dataset. Previously to this customization,
a magic number was used and limited the api to get only 20 subjects.
"""
def fetch_data(subjects, recording=[1, 2], path=None, local_data_path=AGE_SLEEP_RECORDS, force_update=False, update_path=None, base_url=BASE_URL, verbose=None):  # noqa: D301
    """Get paths to local copies of PhysioNet Polysomnography dataset files.
    This will fetch data from the publicly available subjects from PhysioNet's
    study of age effects on sleep in healthy subjects [1]_[2]_. This
    corresponds to a subset of 20 subjects, 10 males and 10 females that were
    25-34 years old at the time of the recordings. There are two night
    recordings per subject except for subject 13 since the second record was
    lost.
    See more details in
    `physionet website <https://physionet.org/physiobank/database/sleep-edfx/sleep-cassette/>`_.
    Parameters
    ----------
    subjects : list of int
        The subjects to use. Can be in the range of 0-19 (inclusive).
    recording : list of int
        The night recording indices. Valid values are : [1], [2], or [1, 2].
    path : None | str
        Location of where to look for the PhysioNet data storing location.
        If None, the environment variable or config parameter
        ``MNE_DATASETS_PHYSIONET_SLEEP_PATH`` is used. If it doesn't exist, the
        "~/mne_data" directory is used. If the Polysomnography dataset
        is not found under the given path, the data
        will be automatically downloaded to the specified folder.
    force_update : bool
        Force update of the dataset even if a local copy exists.
    update_path : bool | None
        If True, set the MNE_DATASETS_EEGBCI_PATH in mne-python
        config to the given path. If None, the user is prompted.
    %(verbose)s
    Returns
    -------
    paths : list
        List of local data paths of the given type.
    """
    records = np.loadtxt(local_data_path,
                         skiprows=1,
                         delimiter=',',
                         dtype={'names': ('subject', 'record', 'type', 'sha',
                                          'fname'),
                                'formats': ('<i2', 'i1', '<S9', 'S40', '<S22')}
                         )
    psg_records = records[np.where(records['type'] == b'PSG')]
    hyp_records = records[np.where(records['type'] == b'Hypnogram')]

    path = _data_path(path=path, update_path=update_path)
    params = [path, force_update, base_url]

    _check_subjects(subjects, 83)

    fnames = []
    for subject in subjects:
        for idx in np.where(psg_records['subject'] == subject)[0]:
            if psg_records['record'][idx] in recording:
                psg_fname = _fetch_one(psg_records['fname'][idx].decode(),
                                       psg_records['sha'][idx].decode(),
                                       *params)
                hyp_fname = _fetch_one(hyp_records['fname'][idx].decode(),
                                       hyp_records['sha'][idx].decode(),
                                       *params)
                fnames.append([psg_fname, hyp_fname])

    return fnames

def get_epochs(files, subject_files, df_subject_information, max_time, cropping=True, equalize_event_counts=False):

    all_subject_epochs = []
    print("Creating epochs for files : ", files)
    for i in files:
        # 1. get all epochs of one person
        raw_data = mne.io.read_raw_edf(subject_files[i][0], preload=False, verbose=False, exclude=["Temp rectal", "Event marker", "Resp oro-nasal", "EMG submental", "EOG horizontal"])
        # 2. add the sleep stage annotation to the raw data
        annotations = mne.read_annotations(subject_files[i][1])
        raw_data.set_annotations(annotations, emit_warning=False)
        # 3. We crop everything before lights off and 30 minutes after getting up of bed
        if cropping:
            lights_off = df_subject_information.loc[i, 'LightsOffSecond']
            night_duration = df_subject_information.loc[i, 'NightDuration']
            woke_up_time = lights_off + night_duration
            raw_data.crop(tmin=lights_off, tmax=min(woke_up_time + THIRTY_MINS_IN_SECS, raw_data.times[-1]))

        # 4. transform into epochs
        events, annot_event_id = mne.events_from_annotations(raw_data, event_id=ANNOTATIONS_EVENT_ID.copy(), chunk_duration=EPOCH_DURATION, verbose=False)

        # 5. Few files do not have N3 sleep (i.e. SC4202EC-Hypnogram), so we have to filter out key-value pairs that are not in the annotations.
        event_ids = {
            event_key: EVENT_ID[event_key]
            for event_key in EVENT_ID
            if EVENT_ID[event_key] in annot_event_id.values()
        }

        try:
            epochs = mne.Epochs(raw=raw_data, events=events, event_id=event_ids, tmin=0., tmax=max_time, baseline=None, verbose=False)
        except ValueError as e:
            # If no ? epochs are created, we need to ignore this class
            event_ids.pop('Sleep stage ?', None)
            epochs = mne.Epochs(raw=raw_data, events=events, event_id=event_ids, tmin=0., tmax=max_time, baseline=None, verbose=False)

        if equalize_event_counts:
            epochs.equalize_event_counts(event_ids.keys())

        all_subject_epochs.append(epochs)

    print("Concatenating all epochs together...")
    return mne.concatenate_epochs(all_subject_epochs)

max_stars_repo_name: PolyCortex/polydodo

max_issues_repo_name: PolyCortex/polydodo

max_forks_repo_name: PolyCortex/polydodo


text:
import numpy as np

def breaker(line):
    array = [line[i:i+4] for i in range(0, len(line), 4)]
    for word in array:
        if len(word) < 4:
            temp = word
            array.remove(word)
            temp += 'X'*(4 - len(temp))
            array.append(temp)
    print(array)
    return array

def packer(list):
  return "".join(list)

packer(breaker('HELLOWORLD')

max_stars_repo_name: vandewinckel/Python

max_issues_repo_name: vandewinckel/Python

max_forks_repo_name: vandewinckel/Python


text:
import os
import sys
sys.path.append(os.path.abspath('..'))

import numpy as np
from ctensor import Tensor
from ctensor.functional import leaky_relu, sigmoid, binary_cross_entropy
from ctensor.optim import Adam

# Data Generation
x = np.random.rand(1000, 5)*10 - 5
x_scale = np.array([1.2, 2, 0.4, 0.7, 0.2]).reshape(1, 5)
x = x*x_scale
y = x[:, 0]**2-2*x[:, 1]+10*x[:, 2]+0.04*x[:, 3]+np.abs(2*x[:, 4])
y = ((y+np.random.rand(1000)) > 17).astype(np.float)

# Tensor Converting
X = Tensor(x, requires_grad=False)
Y = Tensor(y.reshape(1000, 1), requires_grad=False)

# Parameters
M1 = Tensor(np.random.randn(5, 100)/500)
B1 = Tensor(np.zeros((1, 100)))

M2 = Tensor(np.random.randn(100, 1)/100)
B2 = Tensor(np.zeros((1, 1)))

# Optimizer
adam = Adam([M1, B1, M2, B2])

# Compute Graph Definition, note that the graph is actually dynamic

def compute(X):
    X1 = leaky_relu((X @ M1) + B1, 0.01)
    X2 = sigmoid((X1 @ M2) + B2)
    return X2

for _ in range(10000):
    indices = np.random.randint(0, 1000, size=128)
    X_ = X[indices]
    Y_ = Y[indices]
    pred_y = compute(X_)
    loss = binary_cross_entropy(pred_y, Y_)
    adam.zero_grad()
    loss.backward()
    adam.step(1e-3)
    if _ % 1000 == 0:
        print(loss)

# If succeeded, the score would be relatively high
pred_y = compute(X)
print(((pred_y.data > 0.5) == Y.data).mean())

max_stars_repo_name: WannaFy/cTensor

max_issues_repo_name: WannaFy/cTensor

max_forks_repo_name: WannaFy/cTensor


text:
# -*- coding: utf-8 -*-
"""
@file:test_feature1.py
@time:2019/6/1 20:06
@author:Tangj
@software:Pycharm
@Desc
"""
import pandas as pd
import numpy as np
import time
tt = time.time()
fea = pd.DataFrame()
test_bid = pd.read_csv('../usingData/test/test_bid.csv')
request = pd.read_csv('../usingData/test/Request_list.csv')
log = pd.read_csv('../usingData/test/test_log.csv')
print(test_bid.shape)
print(request.columns)
print(log.columns)
# 先把比较简单的，request长度找出来
fea['ad_id'] = request['ad_id'].values
ad_id = request['ad_id'].values
request_list = request['RequestList'].values
request_len = []
for ad in request_list:
    ads = ad.split("|")
    lens = len(ads)
    request_len.append(lens)
fea['request_nums'] = request_len

print(time.localtime(tt))
'''
请求的uid的个数
找出对应的人群的，每一个广告都有一个dataframe，然后将那个log文件和它merge起来，然后统计uid的个数就好了呗
把log里面只取出来uid和对应的request id和positionid，然后将其用逗号连接，变成一个字符串
'''

log_uid = log['uid'].values
log_reqid = log['RequestId'].values
log_posiId = log['PositionId'].values
request = []
for i, k in enumerate(log_uid):
    s = str(log_reqid[i]) + ',' + str(log_posiId[i])
    request.append(s)
log_new = pd.DataFrame()
log_new['uid'] = log_uid
log_new['request'] = np.array(request)
uid_len = []
k = 0
# 找出每个aid对应的request集合，然后作为一个dataframe，和上面的进行merge
for request_l in request_list:
    tt = time.time()
    request_i = request_l.split('|')
    request_new = pd.DataFrame()
    request_new['request'] = request_i
    # print(request_new.shape)
    request_new = pd.merge(request_new, log_new, on='request', how='left')
    # print(request_new.shape)
    uid_len.append(request_new['uid'].unique().shape[0])
    print(k)
    k += 1
fea['uid_nums'] = uid_len
fea.to_csv('../usingData/feature/uid_req_test.csv', index=False)


text:
import argparse
import json
import multiprocessing
import os
import pickle

import numpy
from deap import tools, base, cma, algorithms, creator

from tl_connector import fitness
import nn

creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
creator.create("Individual", list, fitness=creator.FitnessMin)

def evaluate_ind(x, config):
    return sum(fitness(x, config)),

def main():

    parser = argparse.ArgumentParser(description='Evolution of EV charging controllers')
    parser.add_argument('-c', '--config', help='configuration file', type=str, required=True)
    parser.add_argument('-s', '--seed', help='seed of random number generator', type=int, default=-1)

    args = parser.parse_args()

    config = json.load(open(args.config, 'r'))

    seed = args.seed
    if seed == -1:
        seed = numpy.random.randint(0, 1_000_000_000)

    log_prefix = config['log_prefix']

    net = nn.NeuralNetwork([17, 5, 1], nn.relu) # create network to get vectorized size, activations do not matter
    if config.get('controller_type', 'nn') == 'ensemble':
        net = nn.NeuralNetwork([10, 5, 1], nn.relu)
    N = net.vectorized_size()

    numpy.random.seed(seed)

    hof = tools.HallOfFame(1)
    stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats.register("avg", numpy.mean)
    stats.register("std", numpy.std)
    stats.register("min", numpy.min)
    stats.register("max", numpy.max)

    toolbox = base.Toolbox()

    pool = multiprocessing.Pool(config['cpus'])
    toolbox.register('map', pool.map)
    toolbox.register("evaluate", evaluate_ind, config=config)

    strategy = cma.Strategy(centroid=0.1 * numpy.random.randn(N), sigma=config['sigma'], lambda_=config['pop_size'])
    toolbox.register("generate", strategy.generate, creator.Individual)
    toolbox.register("update", strategy.update)

    pop, log = algorithms.eaGenerateUpdate(toolbox, ngen=config['max_gen'], stats=stats, halloffame=hof)

    if not os.path.isdir('results'):
        os.mkdir('results')

    numpy.save(f'results/{log_prefix}_best_sol_{seed}.npy', hof.items[0])
    pickle.dump(log, open(f'results/{log_prefix}_stats_{seed}.pkl', 'wb'))
    pickle.dump(pop, open(f'results/{log_prefix}_pop_{seed}.pkl', 'wb'))
    with open(f'results/{log_prefix}_logbook_{seed}.txt', 'w') as logfile:
        logfile.write(str(log))

if __name__ == '__main__':
    main()


text:
import numpy as np
from .fmks.func import deprecate

class Filter(object):
    """
    Wrapper class for convolution with a kernel and resizing of a kernel
    """
    @deprecate
    def __init__(self, Fkernel, basis):
        """
        Instantiate a Filter.

        Args:
            Fkernel: an array representing a convolution kernel
            basis: an instance of a bases class.

        """
        self.basis = basis
        self._Fkernel = Fkernel

    def _frequency_2_real(self, copy=False):
        """
        Converts the kernel from frequency space to real space with
        the origin shifted to the center.

        Args:
            copy (boolean): indicates if _Fkernel should be copied. This is
                used to make sure the influence cofficients remained unchanged.

        Returns:
            an array in real space
        """
        Fkernel = self._Fkernel
        if copy:
            Fkernel = self._Fkernel.copy()
        return np.fft.fftshift(self.basis._ifftn(Fkernel),
                               axes=self.basis._axes)

    def _real_2_frequency(self, kernel):
        """
        Converts a kernel from real space to frequency space.

        Args:
          kernel: an array representing a convolution kernel in real space

        Returns:
          an array in frequency space
        """
        return self.basis._fftn(np.fft.ifftshift(kernel,
                                axes=self.basis._axes))

    def convolve(self, X):
        """
        Convolve X with a kernel in frequency space.

        Args:
          X: array to be convolved

        Returns:
          convolution of X with the kernel
        """
        FX = self.basis._fftn(X)
        if FX.shape[1:] != self._Fkernel.shape[1:]:
            raise RuntimeError("Dimensions of X are incorrect.")
        Fy = self._sum(FX * self._Fkernel)
        return self.basis._ifftn(Fy)

    def _sum(self, Fy):
        return np.sum(Fy, axis=-1)

    def resize(self, size):
        """
        Changes the size of the kernel to size.

        Args:
          size: tuple with the shape of the new kernel
        """
        if len(size) != len(self._Fkernel.shape[1:-1]):
            raise RuntimeError("length of resize shape is incorrect.")
        if not np.all(size >= self._Fkernel.shape[1:-1]):
            raise RuntimeError("resize shape is too small.")
        kernel = self._frequency_2_real()
        kernel_pad = self._zero_pad(kernel, size)
        self._Fkernel = self._real_2_frequency(kernel_pad)
        self.basis._axes_shape = kernel_pad.shape[1:-1]

    def _zero_pad(self, kernel, size):
        """
        Zero pads a real space array with zeros and does a Fourier transform

        Args:
            kernel: real space array
            size: The size of the zero padded array.

        Returns:
            Fourier transform of a zero padded kernel

        """
        if len(size) != kernel.ndim:
            size = kernel.shape[:1] + tuple(size) + kernel.shape[-1:]
        padsize = np.array(size) - np.array(kernel.shape)
        paddown = padsize // 2
        padup = padsize - paddown
        padarray = np.concatenate((padup[..., None],
                                   paddown[..., None]), axis=1)
        pads = tuple([tuple(p) for p in padarray])
        kernel_pad = np.pad(kernel, pads, 'constant', constant_values=0)
        return kernel_pad

class Correlation(Filter):

    """
    Computes the autocorrelation for a microstructure

    >>> n_states = 2
    >>> X = np.array([[[0, 1, 0],
    ...                [0, 1, 0],
    ...                [0, 1, 0]]])
    >>> from pymks.bases import DiscreteIndicatorBasis
    >>> basis = DiscreteIndicatorBasis(n_states=n_states)
    >>> X_ = basis.discretize(X)
    >>> filter_ = Correlation(X_, basis)
    >>> X_auto = filter_.convolve(X_)
    >>> X_test = np.array([[[[3., 0.  ],
    ...                      [6., 3.],
    ...                      [3., 0.  ]],
    ...                     [[3., 0.  ],
    ...                      [6., 3.],
    ...                      [3., 0.  ]],
    ...                     [[3., 0.  ],
    ...                      [6., 3.],
    ...                      [3., 0.  ]]]])
    >>> assert(np.allclose(X_auto, X_test))

    Args:
        X_: The discretized microstructure function, an
            `(n_samples, n_x, ..., n_states)` shaped array
            where `n_samples` is the number of samples, `n_x` is thes
            patial discretization, and n_states is the number of local states.

    Returns:
        Autocorrelations for microstructure X_
    """

    def __init__(self, kernel, basis):
        """
        Instantiate a Correlation.

        Args:
            kernel: an array representing a convolution kernel
            basis: an instance of a bases class.

        """
        self.basis = basis
        if self.basis._axes_shape > kernel.shape[1:-1]:
                kernel = self._zero_pad(kernel, self.basis._axes_shape)
        Fkernel = self.basis._fftn(kernel)
        super(Correlation, self).__init__(np.conjugate(Fkernel), basis)

    def convolve(self, X):
        """
        Convolve X with a kernel in frequency space.

        Args:
            X: array to be convolved

        Returns:
            correlation of X with the kernel
        """
        if X.shape[1:-1] < self.basis._axes_shape:
            X = self._zero_pad(X, self.basis._axes_shape)
        FX = self.basis._fftn(X)
        Fy = self._sum(FX * self._Fkernel)
        correlation = self.basis._ifftn(Fy)
        return np.fft.fftshift(correlation, axes=self.basis._axes)

    def _sum(self, Fy):
        return Fy

max_stars_repo_name: vivelakorea/pymks

max_issues_repo_name: vivelakorea/pymks


text:
import numpy as np
import math
import pandas as pd
import html
import mgt2001

def BayesTHM(pre_probs, event='D'):
    """
Output: (Bayes):
>>> [num1, num2] >> meaning that given event B has occurred, the probability that the previous event occurred is num1
============
Input:
>>> BayesTHM(np.array[[0.9, 0.1], # being the previous conditions
                          [0.99, 0.9], # being the conditional probabilities: ([P(B | A), P(B | A̅)])
                          [0.01, 0.1]]) # being the complementaries of the conditional probabilities: ([P(B̅ | A), P(B̅ | A̅)])

The deafult number of columns is 2.

Pass in `event='B'` to specify the post event you are to compare.
    """
    events = [event, html.unescape('{}̅'.format(event))]
    n = pre_probs.shape[1]
    m = pre_probs.shape[0] - 1

    # prior_probs is an array representing the previous probability
    # cond_probs is a matrix representing the previous condition probability

    # only the first row being G and G_c respectively
    prior_probs = pre_probs[0, :]
    cond_probs = pre_probs[1:, :]

    # joint_probs is a matrix representing the joint probability
    # p_c_sums is an array representing the probability of prior event
    # Bayes_M_G is a matrix representing the post conditional probability

    joint_probs = np.zeros((m, n))
    p_c_sums = np.zeros(n)
    bayes = np.zeros((m, n))

    for i in range(m):
        joint_probs[i, ] = prior_probs * cond_probs[i, ]
        p_c_sums[i] = np.sum(joint_probs[i, ])
        bayes[i, ] = joint_probs[i, ] / p_c_sums[i]
        to_print = """The probability of post event {i}:
{p_c_sum:.7f}
================
The Bayes probability given the post event {i}:
{bayes}
        """.format(i=events[i], p_c_sum=p_c_sums[i], bayes=bayes[i, ])
        print(to_print)

def portfolio_analysis(stock_df, target_stocks, pss, ddof=0):
    """
    Return list of portfolios

    Usage:
    df = pd.read_excel('Xr07-TSE.xlsx')
    df = df.set_index(['Year', 'Month'])
    month_dict = {'January': 1, 'February': 2, 'March': 3, 'April': 4, 'May': 5, 'June': 6, 'July': 7, 'August': 8, 'September': 9, 'October': 10, 'November': 11, 'December': 12}
    df.rename(index=month_dict, inplace=True)

    target_stocks = ['BMO', 'MG', 'POW', 'RCL.B']
    pss = np.array([[.25, .25, .25, .25], [.2,.6, .1, .1], [.1, .2, .3, .4]])

    portfolios = portfolio_analysis(df[target_stocks], target_stocks, pss)

    --------------------
    cov_mat = np.cov(stock_df[target_stocks].values, rowvar=False) # [i, i] = variance of each stock
    cor_mat = np.corrcoef(df[target_stocks].values, rowvar = False)
    """

    NUMOFSTOCKS = len(target_stocks)
    NUMOFPORT = len(pss)

    cov_mat = np.cov(stock_df[target_stocks].values, rowvar=False, ddof=ddof)
    stock_des = pd.DataFrame(
        data=cov_mat, columns=target_stocks, index=target_stocks)

    stock_des.loc['Expected Returns'] = stock_df.mean()
    # stock_df.mean() # [mgt2001.geomean(stock_df.iloc[:, i]) for i in range(len(stock_df.columns))]

    portfolios = [stock_des.copy() for i in range(NUMOFPORT)]
    for i in range(NUMOFPORT):
        portfolios[i].loc['Weights'] = pss[i]
        portfolios[i]['Total'] = pd.Series(
            data=(portfolios[i].loc['Weights'].sum()), index=['Weights'])

    expected_values = [0 for i in range(NUMOFPORT)]
    var = [0 for i in range(NUMOFPORT)]
    std = list()

    for i in range(NUMOFPORT):
        expected_values[i] += (portfolios[i].loc['Weights']
                               * portfolios[i].loc['Expected Returns']).sum()

    for i in range(NUMOFPORT):
        var[i] = np.dot(portfolios[i].loc['Weights'][0:NUMOFSTOCKS].to_numpy(), np.dot(
            cov_mat, portfolios[i].loc['Weights'][0:NUMOFSTOCKS].to_numpy().transpose()))

    std = [math.sqrt(var[i]) for i in range(NUMOFPORT)]

    for i in range(NUMOFPORT):
        result = """======== Portfolio {i} Return ========
Expected Value:{tab}{tab}{exp:.7f}
Variance:{tab}{tab}{var:.7f}
Standard Deviation:{tab}{std:.7f}
        """.format(tab='\t', i=i+1, exp=expected_values[i], var=var[i], std=std[i])
        print(result)
    return portfolios

def covariance(x, mu_x, y, mu_y, prob):
    '''
    Usage: `covariance(ibm_x, ibm.expect(), ms_y, ms.expect(), prob)`
    Where `prob` equals
        → x
        ↓ y

    Prob can be a two dimensional array or a `numpy` array.
    '''
    cov = 0
    for i, valy in enumerate(y):
        for j, valx in enumerate(x):
            cov += prob[i][j] * (valx - mu_x) * (valy - mu_y)
    return cov


text:
# -*- coding: utf-8 -*-
"""Find near positive definite correlation and covariance matrices

Created on Sun Aug 19 15:25:07 2012
Author: Josef Perktold

TODO:
    add examples for cov_nearest from script log

Notes
-----
We are looking at eigenvalues before and after the conversion to psd matrix.
As distance measure for how close the change in the matrix is, we consider
the sum of squared differences (Frobenious norm without taking the square root)

"""

import numpy as np
from statsmodels.stats.correlation_tools import (
                 corr_nearest, corr_clipped)

examples = ['all']

if 'all' in examples:
    # x0 is positive definite
    x0 = np.array([[1, -0.2, -0.9], [-0.2, 1, -0.2], [-0.9, -0.2, 1]])
    # x has negative eigenvalues, not definite
    x = np.array([[1, -0.9, -0.9], [-0.9, 1, -0.9], [-0.9, -0.9, 1]])
    #x = np.array([[1, 0.2, 0.2], [0.2, 1, 0.2], [0.2, 0.2, 1]])

    n_fact = 2

    print('evals original', np.linalg.eigvalsh(x))
    y = corr_nearest(x, n_fact=100)
    print('evals nearest', np.linalg.eigvalsh(y))
    print(y)

    y = corr_nearest(x, n_fact=100, threshold=1e-16)
    print('evals nearest', np.linalg.eigvalsh(y))
    print(y)

    y = corr_clipped(x, threshold=1e-16)
    print('evals clipped', np.linalg.eigvalsh(y))
    print(y)

    np.set_printoptions(precision=4)
    print('\nMini Monte Carlo')
    # we are simulating a uniformly distributed symmetric matrix
    #     and find close positive definite matrix
    # original can be far away from positive definite,
    #     then original and converted matrices can be far apart in norm
    # results are printed for visual inspection of different cases

    k_vars = 5
    diag_idx = np.arange(k_vars)
    for ii in range(10):
        print()
        x = np.random.uniform(-1, 1, size=(k_vars, k_vars))
        x = (x + x.T) * 0.5
        x[diag_idx, diag_idx] = 1
        #x_std = np.sqrt(np.diag(x))
        #x = x / x_std / x_std[:,None]
        print()
        print(np.sort(np.linalg.eigvals(x)), 'original')

        yn = corr_nearest(x, threshold=1e-12, n_fact=200)
        print(np.sort(np.linalg.eigvals(yn)), ((yn - x)**2).sum(), 'nearest')

        yc = corr_clipped(x, threshold=1e-12)
        print(np.sort(np.linalg.eigvals(yc)), ((yc - x)**2).sum(), 'clipped')

    import time
    t0 = time.time()
    for _ in range(100):
        corr_nearest(x, threshold=1e-15, n_fact=100)

    t1 = time.time()
    for _ in range(1000):
        corr_clipped(x, threshold=1e-15)
    t2 = time.time()

    print('\ntime (nearest, clipped):', t1 - t0, t2 - t1)

if 'all' in examples:
    # example for test case against R
    x2 = np.array([ 1,     0.477, 0.644, 0.478, 0.651, 0.826,
                   0.477, 1,     0.516, 0.233, 0.682, 0.75,
                   0.644, 0.516, 1,     0.599, 0.581, 0.742,
                   0.478, 0.233, 0.599, 1,     0.741, 0.8,
                   0.651, 0.682, 0.581, 0.741, 1,     0.798,
                   0.826, 0.75,  0.742, 0.8,   0.798, 1]).reshape(6,6)

    y1 = corr_nearest(x2, threshold=1e-15, n_fact=200)
    y2 = corr_clipped(x2, threshold=1e-15)
    print('\nmatrix 2')
    print(np.sort(np.linalg.eigvals(x2)), 'original')
    print(np.sort(np.linalg.eigvals(y1)), ((y1 - x2)**2).sum(), 'nearest')
    print(np.sort(np.linalg.eigvals(y1)), ((y2 - x2)**2).sum(), 'clipped')

reason: import numpy,from statsmodels


text:
from __future__ import annotations

import numpy as np
from fast_enum import FastEnum
from numba import njit

from vectorized2d import Array2D

class Point2D(Array2D):
    class Pairing(metaclass=FastEnum):
        ALL = 0
        ALIGNED = 1

    @staticmethod
    @njit
    def _pairwise_diff(self: Point2D, other: Point2D) -> np.ndarray:
        """
        This function returns the pairwise difference(s) between all the pairs of points from self and other.

        Note1: this function heavily utilizes numpy array broadcasting features. That is, instead of tiling/repeating
        `self` and `other` according to their corresponding lengths, we add singleton dimensions to `self` (dim=1) and
        `other` (dim=0). This way, numpy is able to implicitly broadcast the values across dim=1 for `self` and dim=0
        for `other`, saving precious time that might be spent on redundant copying otherwise.

        Note2: after expanding dimensions and broadcasting the resulting shape is (len(self), len(other), 2), thus,
        we need to reshape the result back to a (Nx2) shape.
        """
        # TODO: add a conditional parallel jit for larger arrays (~len(s)xlen(o) > 10_000)
        self_reshaped = self.reshape((len(self), 1, 2))
        other_reshaped = other.reshape((1, len(other), 2))
        return np.ascontiguousarray(self_reshaped - other_reshaped).reshape(-1, 2)

    def euclid_dist(self, other: Point2D, *, pairing: Pairing = Pairing.ALL) -> np.ndarray:
        """
        Calculate the euclidean distance(s) between self and other.

        Note: ALIGNED pairing mode is supported only when self and other have same shape.

        :param other: The target point(s) for distance calculations
        :param pairing: An enum, specifies whether to calculate distances between
                        ALL (pairwise) or ALIGNED (corresponding points).
        :return: If pairing mode is ALIGNED:
                    a 1D numpy array of euclidean distance(s) between corresponding pairs of self and other.
                 Otherwise, if pairing mode is ALL:
                    a 2D numpy array of shape=(len(self), len(other)), of euclidean distance(s) between
                    all pairs of self and other.

        """
        if pairing is self.Pairing.ALIGNED or len(self) == 1 or len(other) == 1:
            dists = (self - other).norm
        else:
            dists = self._pairwise_diff(self, other).view(type(self)).norm

        if pairing is self.Pairing.ALL:
            dists = dists.reshape(len(self), len(other))

        return dists

    def euclid_dist_squared(self, other: Point2D, *, pairing: Pairing = Pairing.ALL) -> np.ndarray:
        """
        Calculate the euclidean distance(s) squared between self and other.

        Note: ALIGNED pairing mode is supported only when self and other have same shape.

        :param other: The target point(s) for distance calculations
        :param pairing: An enum, specifies whether to calculate distances between
                        ALL (pairwise) or ALIGNED (corresponding points).
        :return: If pairing mode is ALIGNED:
                    a 1D numpy array of euclidean distance(s) squared between corresponding pairs of self and other.
                 Otherwise, if pairing mode is ALL:
                    a 2D numpy array of shape=(len(self), len(other)), of euclidean distance(s) squared between
                    all pairs of self and other.

        """
        if pairing is self.Pairing.ALIGNED or len(self) == 1 or len(other) == 1:
            dists = (self - other).norm_squared
        else:
            dists = self._pairwise_diff(self, other).view(type(self)).norm_squared

        if pairing is self.Pairing.ALL:
            dists = dists.reshape(len(self), len(other))

        return dists

reason: import numpy,from numba


text:
"""Template problem for robust optimisation

Authors: N. Banglawala, EPCC, 2019
License: MIT

"""

###############################################################################

import numpy as np

###############################################################################

class RobustTemplateProblem(object):
    """Template class for robust problems. The compulsory methods defined here
    must be included in all robust problem classes.

    Attributes
    ----------
    These attributes must be set

    bounds : list of lists, dtype=float
        list of lower and upper bounds of problem. First list gives lower
        bounds for each dimension, second list gives upper bounds for each
        dimension.

    disturbance_bounds : list of lists, dtype=float
        list of lower and upper bounds of disturbance neighbourhood. First
        list gives lower bounds for each dimension, second list gives upper
        bounds for each dimension.

    fitness_func : func
        fitness or objective function. Should take multi-dimensional ndarray
        as input and output single-valued real

    """

    def __init__(self, bounds, disturbance_bounds, fitness_func)

        self.lbound = np.array(bounds[0])
        self.ubound = np.array(bounds[1])
        self.dlbound = np.array(disturbance_bounds[0])
        self.dubound = np.array(disturbance_bounds[1])
        self.fitness = fitness_func

    # compulsory method
    # input: None
    # output: tuple(lower bounds, upper bounds)
    #
    def get_bounds(self):
        return self.lbound, self.ubound

    # compulsory method
    # input: None
    # output: tuple(lower disturbance bounds, upper disturbance bounds)
    #
    def get_disturbance_bounds(self):
        return self.dlbound, self.dubound

    # compulsory method
    # returns fitness of individual x
    # input: x, ndarray, dtype=float
    # output: fitness of x, float
    #
    def calc_fitness(self, x):
        fitness_value = self.fitness(x)
        return fitness_value

    # compulsory method
    # method to create and evolve new individual from parents
    # input: parents, ndarray of ndarray, dtype=float (default is 2 parents)
    #        e.g. parent 1 is parents[0, :], parent 2 is parents[1, :]
    # output: new x, ndarray, dtype=float
    #
    def create_new_individual(self, parents):
        x = np.array([])
        return x

    # compulsory method
    # !!! DO NO EDIT !!!
    # wrapper for adding individual to population
    # input: population index of x, int
    # output: None
    #
    def add_individual(self, x_idx):
        pass

max_stars_repo_name: EPCCed/pyrobust

max_issues_repo_name: EPCCed/pyrobust

max_forks_repo_name: EPCCed/pyrobust


text:
#!/usr/local/bin/python

# Python script plots data from semi analytic disc runs

import matplotlib.pyplot as plt
import numpy as np

nlayercol=11
nlogcol = 10

prefix = raw_input('What is the file prefix? ')
fileno = input('Which dump is to be plotted? ')

# Input parameters

fileno = int(fileno)

layer = prefix+'layer.'+str(fileno)
logfile = prefix+'.log'

# Open log file, to find time of dump

logdata = np.genfromtxt(logfile)

time = logdata[fileno-1,0]

print 'Time is ',time ,' years'

# Open profile file and read

layerdata = np.genfromtxt(layer,skip_header=1)
layerdata.reshape(layerdata.size/nlayercol,nlayercol)

# Set up plot data for loop

# Filenames
outputstring=['r','sigma','sigma_m', 'sigma_tot', 'cs_m','kappa_m','gamma_m','mu_m', 'tau_m', 'nu_m', 'alpha_m']

# Plot labels
ylabels = [r'r (AU)',r'$\Sigma_g$ (g cm $^{-2}$)',r'$\Sigma_m$ (g cm $^{-2}$)',r'$\Sigma_{tot}$ (g cm $^{-2}$)',
           r' $c_{s,m}$ (cm s$^{-1}$)',r'$\kappa_m$ (cm$^{2}$ g$^{-1}$)',r'$\gamma_m$',r'$\mu_m$',r'$ \tau_m $',r'$\nu_{m}$',r'$ \alpha_m $']

# Log the y axis? True/False
setylog=[False, True, True,True,True,True,False,False,True,True,True]

# y limits - set defaults first
ymin=[]
ymax=[]
for i in range(nlayercol):
    ymin.append(0.0)
    ymax.append(0.0)

# Now define any non-default limits

# Now generate layer plots

for i in range(1,nlayercol):
    print 'Plotting ',outputstring[i]
    fig1 = plt.figure()
    ax = fig1.add_subplot(111)
    ax.set_xlabel(ylabels[0])
    ax.set_xscale('log')

    ax.set_ylabel(ylabels[i])

    if setylog:
        ax.set_yscale('log')

    if ymin[i]!=ymax[i]:
        print 'Setting limits ',ymin[i],ymax[i]
        ax.set_ylim(ymin[i],ymax[i])

    line = ax.plot(layerdata[:,0],layerdata[:,i], label='Time = '+str(time)+'yr')
    ax.legend()

    outputfile = prefix+outputstring[i]+'.ps'
    plt.savefig(outputfile, format='ps')

# Finish layer plots

print 'Plotting complete for file ',layer

# Now plot log data

outputstring = ['t', 'mgrav','mmag','mdisc','grav_max','mag_max','sig_max','tot_lum','mdot_grav', 'mdot_mag']

# Plot labels
ylabels = [r't (yr)',r'$M_{grav}$ ($M_{\odot}$)',r'$M_{MRI}$ ($M_{\odot}$)',r'$M_{disc}$ ($M_{\odot}$)',
           r' $\Sigma_{grav,max}$ (g cm$^{-2}$)',r' $\Sigma_{MRI,max}$ (g cm$^{-2}$)',r' $\Sigma_{max}$ (g cm$^{-2}$)',
           r'$L_{tot}$ ($L_{\odot}$)', r'$\dot{M}_{grav}$ ($M_{\odot} yr^{-1}$)', r'$\dot{M}_{mag}$ $(M_{\odot} yr^{-1}$)']

# Log the y axis? True/False
setylog=[False, False, True,True,]

# y limits - set defaults first
ymin=[]
ymax=[]
for i in range(nlogcol):
    ymin.append(0.0)
    ymax.append(0.0)

for i in range(1,nlogcol):
    print 'Plotting ',outputstring[i]
    fig1 = plt.figure()
    ax = fig1.add_subplot(111)
    ax.set_xlabel(ylabels[0])
    ax.set_xscale('log')

    ax.set_ylabel(ylabels[i])

    if setylog:
        ax.set_yscale('log')

    if ymin[i]!=ymax[i]:
        print 'Setting limits ',ymin[i],ymax[i]
        ax.set_ylim(ymin[i],ymax[i])

    line = ax.plot(logdata[:,0],logdata[:,i])

    outputfile = prefix+outputstring[i]+'time.ps'
    plt.savefig(outputfile, format='ps')

# Extra plots outside of the loop: plot mdot vs sigmax for grav and MRI layers

print 'Plotting mdot vs sigma for grav layer'

fig1 = plt.figure()
ax = fig1.add_subplot(111)
ax.set_xlabel(ylabels[4])
ax.set_ylabel(ylabels[8])
ax.set_xscale('log')
ax.set_yscale('log')

line = ax.scatter(logdata[:,4],logdata[:,8])

outputfile = 'mdot_vs_sigma_grav.ps'
plt.savefig(outputfile, format='ps')

print 'Plotting mdot vs sigma for MRI layer'

figmag = plt.figure()
ax = figmag.add_subplot(111)
ax.set_xlabel(ylabels[5])
ax.set_ylabel(ylabels[9])
ax.set_xscale('log')
ax.set_yscale('log')

line = ax.scatter(logdata[:,5],logdata[:,9])

outputfile = 'mdot_vs_sigma_mri.ps'
plt.savefig(outputfile, format='ps')
plt.show()
# Finish layer plots

print 'Plotting complete for file ',logfile


text:
import math
import random
import progressbar
import psycopg2

import tensorflow as tf
import numpy as np

from scipy import interpolate

'''
    This script converts the available gas price data into a format that's favourable for training. It creates the three
    files training.tfrecords validation.tfrecords and test.tfrecord, in which all available data is divided into
    training, validation and test data. When looking at the average entries per day it becomes clear that it should be
    sufficient to interpolate the data every two hours.
'''

# Parse user input
flags = tf.flags
flags.DEFINE_string("data_path", 'data_', "Path of the folder containing prices of gas stations")
FLAGS = flags.FLAGS

# Ratios in which to split the data (training, validation, test)
dataset_ratios = (0.8, 0.1, 0.1)

# Number of data samples taken per file
n_samples_per_file = 5

# Extract distinct months of 'hours_per_month' hours(last month and previous months) for training of the neural network
hours_per_month = 31 * 24

# Number of data points per month
n_data_points = int(hours_per_month / 2)

def append_to_tfrecord(writer, dates, price_data):
    """Append the data to the current writer"""

    # First, clean the data
    dates, price_data = clean_data(dates, price_data)

    # Ignore, if nothing to write
    if len(dates) <= n_data_points:
        return

    # Parse entries to a new format: The first entry is seen as hour zero. All following entries are relative
    # to the first one.
    hours = []
    prices = []
    reference_date = dates[0]

    # Iterate over all entries
    for index, date in enumerate(dates):
        time_diff = date - reference_date
        hours.append(time_diff.total_seconds() / 3600.0)
        prices.append(int(price_data[index]))

    # Interpolate the data at every two hours
    f = interpolate.interp1d(hours, prices, kind='linear')
    hours = np.linspace(0, hours[-1], (hours[-1] + 1) / 2)
    prices = f(hours)

    # On top of just the last month of the data, generate n_samples_per_file random hours in the second half of
    # the dataset that are each the last known prices for another new data set
    current_index = len(prices) - n_data_points
    for _ in range(n_samples_per_file):
        # Generate data set
        example = create_data_set(prices, current_index)

        if example is None:
            break

        # Serialize to string and write to the file
        writer.write(example.SerializeToString())

        # Generate random new index in the second half of the data set
        current_index = random.randint(math.floor(len(prices) / 2), len(prices) - n_data_points)

        # Abort if not enough previous months
        if current_index - n_data_points < 0:
            print("Aborted, not enough entries for current index")
            break

def create_data_set(_prices, month_index):
    """Creates a new data set from all data in _prices. Month index represents the first entry of the month that
    is going to be predicted"""
    next_month = _prices[month_index:month_index + n_data_points]
    prev_months = []

    n_prev_months = 0
    while (month_index - n_data_points) >= 0:
        month_index -= n_data_points
        prev_months.append(_prices[month_index:month_index + n_data_points])

        n_prev_months += 1

    # Abort if not enough months available
    if len(prev_months) == 0:
        return None

    # Create feature
    feature = {'prev_months': tf.train.Feature(float_list=
                                               tf.train.FloatList(value=np.array(prev_months).reshape(-1))),
               'next_month': tf.train.Feature(float_list=
                                              tf.train.FloatList(value=np.array(next_month))),
               'n_prev_months': tf.train.Feature(int64_list=
                                                 tf.train.Int64List(value=[n_prev_months]))}

    # Create an example protocol buffer
    example = tf.train.Example(features=tf.train.Features(feature=feature))

    return example

def load_data_from_database():
    """This generator function returns the data of each gas station one after another"""

    # Connect to an existing database
    conn_string = "host='localhost' dbname='carefuel' user='postgres' password='NJuJh1A!Wln..'"
    conn = psycopg2.connect(conn_string)

    # Open a cursor to perform database operations
    cur = conn.cursor()

    # Get all gas station IDs
    cur.execute("SELECT ID FROM gas_station;")
    gas_station_IDs = cur.fetchall()

    # Iterate over all gas stations
    with progressbar.ProgressBar(max_value=len(gas_station_IDs)) as bar:
        for index, ID in enumerate(gas_station_IDs):
            bar.update(index)
            try:
                # Get all prices of this database
                cur.execute("SELECT date, e5, e10, diesel FROM gas_station_information_history WHERE stid='" + ID[0] +
                            "' ORDER BY date ASC;")
                results = cur.fetchall()

                # Extract into new data structure
                dates = []
                e5 = []
                e10 = []
                diesel = []

                # Ignore gas stations with not enough entries
                if results is None:
                    continue
                if len(results) < 400:
                    continue

                for entry in results:
                    dates.append(entry[0])
                    e5.append(entry[1])
                    e10.append(entry[2])
                    diesel.append(entry[3])

                yield dates, {'e5': e5, 'e10': e10, 'diesel': diesel}
            except Exception as e:
                print("Error at index", index)
                print(e)
                continue

    # Close communication with the database
    cur.close()
    conn.close()

def clean_data(dates, prices):
    """
    This function takes the two lists of dates and prices and returns two new lists that are cleared of errenous data
    """
    new_dates = []
    new_prices = []
    for i, price in enumerate(prices):
        # Assume that every price that is < 100 and > 3000, is garbage
        if 100 < price < 3000:
            new_dates.append(dates[i])
            new_prices.append(price)

    return new_dates, new_prices

def main(_):
    # Number of gas stations in database
    n_gas_stations = 15110

    # Split all data into three parts (training, validation, testing) depending on the ratios defined in config
    training_index = 0
    validation_index = math.floor(dataset_ratios[0] * n_gas_stations)
    testing_index = math.floor(n_gas_stations - dataset_ratios[2] * n_gas_stations)

    # Create a tf_record writer for each file
    writers = {}
    for data_set in ['training', 'validation', 'testing']:
        for fuel_type in ['e5', 'e10', 'diesel']:
            path = FLAGS.data_path + "/" + fuel_type + '_' + data_set
            writers[fuel_type + '_' + data_set] = tf.python_io.TFRecordWriter(path)

    # Iterate over all gas stations using a generator function, just in case something breaks during execution, handle
    # the exception in a way that the files can still be closed gracefully
    try:
        gas_station_gen_func = load_data_from_database()
        for index, data in enumerate(gas_station_gen_func):
            # Unpack data
            dates = data[0]
            fuel_prices = data[1]

            # Assign correct data set
            if index < validation_index:
                data_set = 'training'
            elif index < testing_index:
                data_set = 'validation'
            else:
                data_set = 'testing'

            # Append new data to file in parallel
            for fuel_type in ['e5', 'e10', 'diesel']:
                append_to_tfrecord(writers[fuel_type + '_' + data_set], dates, fuel_prices[fuel_type])

    except Exception as e:
        print("Something went wrong during creation of tfrecord files")
        print(e.with_traceback())

    # Close all writers
    for data_set in ['training', 'validation', 'testing']:
        for fuel_type in ['e5', 'e10', 'diesel']:
            print("bla")
            writers[fuel_type + '_' + data_set].close()

if __name__ == '__main__':
    tf.app.run()

reason: import numpy,from scipy


text:
# coding=utf-8
# main codes, call functions at stokes_flow.py
# Zhang Ji, 20170518

import sys

import petsc4py

petsc4py.init(sys.argv)

import numpy as np
from src.stokes_flow import problem_dic, obj_dic
import src.stokes_flow as sf
from petsc4py import PETSc
from src.geo import *
import pickle

# def print_case_info(**problem_kwargs):
#     comm = PETSc.COMM_WORLD.tompi4py()
#     rank = comm.Get_rank()
#     size = comm.Get_size()
#
#     fileHandle = problem_kwargs['fileHandle']
#     radius = problem_kwargs['radius']
#     deltaLength = problem_kwargs['deltaLength']
#     matrix_method = problem_kwargs['matrix_method']
#     u = problem_kwargs['u']
#
#     PETSc.Sys.Print('sphere radius: %f, delta length: %f, velocity: %f' % (radius, deltaLength, u))
#
#     err_msg = "Only 'pf', 'rs', 'tp_rs', and 'lg_rs' methods are accept for this main code. "
#     assert matrix_method in ('rs', 'tp_rs', 'lg_rs', 'rs_precondition', 'tp_rs_precondition', 'lg_rs_precondition', 'pf'), err_msg
#     epsilon = problem_kwargs['epsilon']
#     if matrix_method in ('rs', 'rs_precondition', 'pf'):
#         PETSc.Sys.Print('create matrix method: %s, epsilon: %f'
#                         % (matrix_method, epsilon))
#     elif matrix_method in ('tp_rs', 'tp_rs_precondition'):
#         twoPara_n = problem_kwargs['twoPara_n']
#         PETSc.Sys.Print('create matrix method: %s, epsilon: %f, order: %d'
#                         % (matrix_method, epsilon, twoPara_n))
#     elif matrix_method in ('lg_rs', 'lg_rs_precondition'):
#         legendre_m = problem_kwargs['legendre_m']
#         legendre_k = problem_kwargs['legendre_k']
#         PETSc.Sys.Print('create matrix method: %s, epsilon: %f, m: %d, k: %d, p: %d'
#                         % (matrix_method, epsilon, legendre_m, legendre_k, (legendre_m + 2 * legendre_k + 1)))
#
#     solve_method = problem_kwargs['solve_method']
#     precondition_method = problem_kwargs['precondition_method']
#     PETSc.Sys.Print('solve method: %s, precondition method: %s'
#                     % (solve_method, precondition_method))
#     PETSc.Sys.Print('output file headle: ' + fileHandle)
#     PETSc.Sys.Print('MPI size: %d' % size)

def get_problem_kwargs(**main_kwargs):
    OptDB = PETSc.Options()
    radius = OptDB.getReal('r', 1)
    n = OptDB.getInt('n', 200)
    deltaLength = np.sqrt(4 * np.pi * radius * radius / n)
    epsilon = OptDB.getReal('e', -1)
    rel_omega = OptDB.getReal('rel_omega', 1)
    rel_u = OptDB.getReal('rel_u', 1)
    fileHandle = OptDB.getString('f', 'try_forcefree')
    solve_method = OptDB.getString('s', 'gmres')
    precondition_method = OptDB.getString('g', 'none')
    plot = OptDB.getBool('plot', False)
    matrix_method = OptDB.getString('sm', 'pf')
    restart = OptDB.getBool('restart', False)
    twoPara_n = OptDB.getInt('tp_n', 1)
    legendre_m = OptDB.getInt('legendre_m', 3)
    legendre_k = OptDB.getInt('legendre_k', 2)
    n_sphere_check = OptDB.getInt('n_sphere_check', 2000)
    n_node_threshold = OptDB.getInt('n_threshold', 10000)
    random_velocity = OptDB.getBool('random_velocity', False)
    getConvergenceHistory = OptDB.getBool('getConvergenceHistory', False)
    pickProblem = OptDB.getBool('pickProblem', False)

    n_obj = OptDB.getInt('n', 1)
    n_obj_x = OptDB.getInt('nx', n_obj)
    n_obj_y = OptDB.getInt('ny', n_obj)
    distance = OptDB.getReal('dist', 3)
    distance_x = OptDB.getReal('distx', distance)
    distance_y = OptDB.getReal('disty', distance)
    move_delta = np.array([distance_x, distance_y, 1])
    # field_range: describe a sector area.
    field_range = np.array([[-3, -3, -3], [n_obj_x - 1, n_obj_y - 1, 0] * move_delta + [3, 3, 3]])
    n_grid = np.array([n_obj_x, n_obj_y, 1]) * 20

    problem_kwargs = {
        'name':                  'try_forcefree',
        'matrix_method':         matrix_method,
        'deltaLength':           deltaLength,
        'epsilon':               epsilon,
        'delta':                 deltaLength * epsilon,  # for rs method
        'solve_method':          solve_method,
        'precondition_method':   precondition_method,
        'field_range':           field_range,
        'n_grid':                n_grid,
        'plot':                  plot,
        'fileHandle':            fileHandle,
        'region_type':           'rectangle',
        'twoPara_n':             twoPara_n,
        'legendre_m':            legendre_m,
        'legendre_k':            legendre_k,
        'radius':                radius,
        'rel_omega':             rel_omega,
        'rel_u':                 rel_u,
        'random_velocity':       random_velocity,
        'n_obj_x':               n_obj_x,
        'n_obj_y':               n_obj_y,
        'move_delta':            move_delta,
        'restart':               restart,
        'n_sphere_check':        n_sphere_check,
        'n_node_threshold':      n_node_threshold,
        'getConvergenceHistory': getConvergenceHistory,
        'pickProblem':           pickProblem
    }

    for key in main_kwargs:
        problem_kwargs[key] = main_kwargs[key]
    return problem_kwargs

def main_fun(**main_kwargs):
    problem_kwargs = get_problem_kwargs(**main_kwargs)
    # print_case_info(**problem_kwargs)

    fileHandle = problem_kwargs['fileHandle']
    radius = problem_kwargs['radius']
    deltaLength = problem_kwargs['deltaLength']
    matrix_method = problem_kwargs['matrix_method']
    n_obj_x = problem_kwargs['n_obj_x']
    n_obj_y = problem_kwargs['n_obj_y']
    move_delta = problem_kwargs['move_delta']
    random_velocity = problem_kwargs['random_velocity']
    getConvergenceHistory = problem_kwargs['getConvergenceHistory']
    pickProblem = problem_kwargs['pickProblem']
    epsilon = problem_kwargs['epsilon']
    rel_u = problem_kwargs['rel_u']
    rel_omega = problem_kwargs['rel_omega']
    norm_rel_u = np.array((1, 1, 1))
    norm_rel_omega = np.array((0, 0, 0))
    rel_U = np.hstack((rel_u * norm_rel_u, rel_omega * norm_rel_omega))
    center = np.zeros(3)

    n = int(16 * radius * radius / deltaLength / deltaLength)
    sphere_geo0 = sphere_geo()  # force geo
    sphere_geo0.create_n(n, radius)
    sphere_geo0.node_rotation(theta=np.pi / 5)
    sphere_geo1 = sphere_geo0.copy()
    if matrix_method in ('pf',):
        sphere_geo1.create_n(n, radius + deltaLength * epsilon)

    obj_sphere = obj_dic[matrix_method]()
    obj_sphere_kwargs = {'name': 'sphereObj_0_0'}
    obj_sphere.set_data(sphere_geo1, sphere_geo0, **obj_sphere_kwargs)

    name = 'obj_composite1'
    obj_composite = sf.ForceFreeComposite(center, name)
    obj_composite.add_obj(obj_sphere, rel_U=rel_U)

    problem = sf.ForceFreeProblem(**problem_kwargs)
    problem.add_obj(obj_composite)
    problem.create_matrix()

    problem.print_info()
    residualNorm = problem.solve()

    problem.pickmyself(fileHandle)
    with open(fileHandle + '_pick.bin', 'rb') as input:
        unpick = pickle.Unpickler(input)
        problem = unpick.load()
        problem.unpick_myself()
    problem.create_matrix()
    problem.print_info()
    residualNorm = problem.solve()
    problem.pickmyself(fileHandle)

    PETSc.Sys.Print(obj_composite.get_ref_U())
    PETSc.Sys.Print(obj_composite.get_total_force())

    problem.vtk_obj(fileHandle)
    geo_check = sphere_geo()  # force geo
    geo_check.create_n(n * 2, radius)
    geo_check.set_rigid_velocity(rel_U + obj_composite.get_ref_U())
    obj_check = obj_dic[matrix_method]()
    obj_check.set_data(geo_check, geo_check, **obj_sphere_kwargs)
    problem.vtk_check(fileHandle + '_check', obj_check)

    problem.vtk_self(fileHandle)
    obj_sphere.vtk(fileHandle)
    force_sphere = obj_sphere.get_force_x()
    PETSc.Sys().Print('---->>>%s: Resultant at x axis is %f' % (str(problem), force_sphere.sum()/(6*np.pi*radius)))

    return True

if __name__ == '__main__':
    main_fun()


text:
import os
os.environ["PYTHONIOENCODING"] = "utf-8"
#1 geforce
#0 titan
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
from tensorflow.keras import regularizers
from tensorflow.keras import metrics
from tensorflow.keras.models import *
from tensorflow.keras.layers import *
from tensorflow.keras.optimizers import *
from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler
from tensorflow.keras import backend as K
from tensorflow.keras.callbacks import CSVLogger, TensorBoard, ModelCheckpoint
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.constraints import MaxNorm

from network.layers import FullGatedConv2D, GatedConv2D, OctConv2D
from tensorflow.keras.layers import Conv2D, Bidirectional, LSTM, GRU, Dense
from tensorflow.keras.layers import Dropout, BatchNormalization, LeakyReLU, PReLU
from tensorflow.keras.layers import Input, Add, Activation, Lambda, MaxPooling2D, Reshape
from tensorflow.keras.models import load_model

import tensorflow as tf
# config = tf.compat.v1.ConfigProto()
# config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
# session = tf.compat.v1.Session(config=config)

from PIL import Image
from tqdm import tqdm
import random
import sys
import codecs
import re
import cv2
import tqdm
from glob import glob
from tqdm import tqdm
from data import preproc as pp
#from computeMetric import computeMetrics
import string
################################modifiier ici le path des dossiers##################################################
##########################################################################################################
##########################################################################################################
rootPath='handwritten-text-recognition/'
DatabasePath='BaseKHATT/DatasetKHATT/'
##########################################################################################################
##########################################################################################################
##########################################################################################################

# define parameters
source = "khatt"
arch = "flor" ########ne pas modifier, nous utilisons architeture crnn de flor
batch_size=32
# define paths
source_path = os.path.join("..", "data", f"{source}.hdf5")
output_path = os.path.join("..", "output-KHATT-distorted"   , source, arch)
target_path = os.path.join(output_path, "checkpoint_weights.hdf5")
os.makedirs(output_path, exist_ok=True)

# define input size, number max of chars per line and list of valid chars
max_text_length = 128  ####not change this value
img_width=1024 #########for crnn
img_height=128 #########for crnn
input_size_crnn = (1024,128, 1)
input_size = (128,1024, 1) #############for the GAN

##########################################################################################################
##########################################################################################################
##########################################################################################################
def get_callbacks(logdir, checkpoint, monitor="val_loss", verbose=1):
        """Setup the list of callbacks for the model"""

        callbacks = [

          CSVLogger(
                filename=os.path.join(logdir, "epochs.log"),
                separator=";",
                append=True),
            TensorBoard(
                log_dir=logdir,
                histogram_freq=10,
                profile_batch=0,
                write_graph=True,
                write_images=False,
                update_freq="epoch"),
            ModelCheckpoint(
                filepath=checkpoint,
                monitor=monitor,
                save_best_only=True,
                save_weights_only=True,
                verbose=verbose),
            EarlyStopping(
                monitor=monitor,
                min_delta=1e-8,
                patience=15,
                restore_best_weights=True,
                verbose=verbose),
            ReduceLROnPlateau(
                monitor=monitor,
                min_delta=1e-8,
                factor=0.2,
                patience=20,
                verbose=verbose)
        ]

        return callbacks

def normalizeTranscription(text_line):
	text_line = text_line.replace('sp', ' sp ')
	text_line = text_line.replace('A', 'A ')
	text_line = text_line.replace('B', 'B ')
	text_line = text_line.replace('E', 'E ')
	text_line = text_line.replace('M', 'M ')
	text_line = text_line.replace('  ', ' ')
	return  text_line

def read_file_shuffle(list_file_path):
	char_file = codecs.open(list_file_path, 'r', 'utf-8')

	list0 = []
	for l in char_file:
		list0.append(l.strip())
	random.shuffle(list0)
	return list0
def read_file(list_file_path):
	char_file = codecs.open(list_file_path, 'r', 'utf-8')

	list0 = []
	for l in char_file:
		list0.append(l.strip())

	return list0
def read_file_char(list_file_path):
	char_file = codecs.open(list_file_path, 'r', 'utf-8')

	list0 = []
	for l in char_file:
		list0.append(l.strip())

	return list0

####################################
#charlatin =str(string.printable[:95])
#f=codecs.open(rootPath+ 'src/SetsKHATT/CHAR_LIST','w','utf-8')
#for s in charlatin:
	#print(s)
	#f.write(s +'\n')
#f.close()
####################################
charset_base=read_file_char(rootPath+ 'src/Sets/CHAR_LIST')
#print(charset_base)

#######################CRNN CTC Recognize##########################
def ctc_loss_lambda_func(y_true, y_pred):
	"""Function for computing the CTC loss"""

	if len(y_true.shape) > 2:
		y_true = tf.squeeze(y_true)

	# y_pred.shape = (batch_size, string_length, alphabet_size_1_hot_encoded)
	# output of every model is softmax
	# so sum across alphabet_size_1_hot_encoded give 1
	#               string_length give string length
	input_length = tf.math.reduce_sum(y_pred, axis=-1, keepdims=False)
	input_length = tf.math.reduce_sum(input_length, axis=-1, keepdims=True)

	# y_true strings are padded with 0
	# so sum of non-zero gives number of characters in this string
	label_length = tf.math.count_nonzero(y_true, axis=-1, keepdims=True, dtype="int64")

	loss = K.ctc_batch_cost(y_true, y_pred, input_length, label_length)

	# average loss across all entries in the batch
	loss = tf.reduce_mean(loss)

	return loss

def build_crnn():

	############################# Model Creation########################################
	from network.model import flor

	# create and compile HTRModel
	inputs, outputs = flor(input_size_crnn, len(charset_base) + 1)

	optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001)

	# create and compile
	model = Model(inputs=inputs, outputs=outputs)
	model.compile(optimizer=optimizer, loss=ctc_loss_lambda_func)

	return model

def encode_txt(text):
	encoded=[]
	cc=text.split()
	for item in cc:
		index = charset_base.index(item)

		encoded.append(index)

	encoded=encoded[::-1]	############this is done only for arabic, otherwise remove this line

	return encoded

def train_crnn( crnn ,ep_start=0, epochs=130, batch_size=32):

	batch_train_gt_path=[]

	list_image_train = read_file_shuffle(rootPath + 'src/Sets/list_train')
	list_image_valid = read_file_shuffle(rootPath + 'src/Sets/list_valid')
	list_lines = read_file(rootPath + 'src/Sets/lines.txt')

	batch_txt = []
	nb=0
	x_train_rcnn=[] ############images in batch
	y_train_rcnn=[] ##ground truth of this batch
	for im in tqdm(list_image_train):
		###########read Grund truth text
		matched_lines = [s for s in list_lines if im in s]
		#print(matched_lines)
		l = matched_lines[0]
		l1 = l.split()
		text_line = l1[8]
		line = normalizeTranscription(text_line)
		len_trancription=len(line.split())
		if len_trancription < max_text_length : ###########this conditioning the CRNN recognize

			batch_train_gt_path = []
			batch_train_gt_path.append('Hito-docs/DatasetKHATT1/' + im + '.tif')
			#print(batch_train_gt_path)
			imgx=pp.preprocess(batch_train_gt_path[0],input_size_crnn)
			x_train_rcnn.append(imgx)
			encoded_txt=encode_txt(line)
			y_train_rcnn.append(encoded_txt)
			del imgx
			del encoded_txt

			batch_train_gt_path=[]
			batch_txt=[]
			batch = 0
		nb=nb+1
	y_train_rcnn = [np.pad(y, (0, max_text_length - len(y))) for y in y_train_rcnn]
	y_train_rcnn = np.asarray(y_train_rcnn, dtype=np.int16)
	x_train_rcnn = np.asarray(x_train_rcnn)
	x_train_rcnn = pp.normalization(x_train_rcnn)

	#################################################################################################"
	batch_txt = []
	nb=0
	x_valid_rcnn=[] ############images in batch
	y_valid_rcnn=[] ##ground truth of this batch
	batch_valid_gt_path=[]
	for im in tqdm(list_image_valid):
		###########read Grund truth text
		matched_lines = [s for s in list_lines if im in s]
		#print(matched_lines)
		l = matched_lines[0]
		l1 = l.split()
		text_line = l1[8]
		line = normalizeTranscription(text_line)
		len_trancription=len(line.split())
		if len_trancription < max_text_length : ###########this conditioning the CRNN recognize

			batch_valid_gt_path = []
			batch_valid_gt_path.append('Hito-docs/DatasetKHATT1/' + im + '.tif')
			img=pp.preprocess(batch_valid_gt_path[0],input_size_crnn)
			x_valid_rcnn.append(img)
			encoded_txt=encode_txt(line)
			y_valid_rcnn.append(encoded_txt)
			del img
			del encoded_txt

			batch_valid_gt_path=[]
			batch_txt=[]
			batch = 0
		nb=nb+1
	y_valid_rcnn = [np.pad(y, (0, max_text_length - len(y))) for y in y_valid_rcnn]
	y_valid_rcnn = np.asarray(y_valid_rcnn, dtype=np.int16)
	x_valid_rcnn = pp.normalization(x_valid_rcnn)

	validation_data=(x_valid_rcnn,y_valid_rcnn)

	callbacks = get_callbacks(logdir=output_path, checkpoint=target_path, verbose=0)
	crnn.fit(x_train_rcnn,y_train_rcnn, validation_data=validation_data,batch_size=batch_size,initial_epoch=ep_start, epochs=epochs, verbose=1,
		 callbacks=callbacks,shuffle=True,validation_freq=1)
	return   crnn

def train(nepochs,batch_size):
	crnn = build_crnn()
	crnn = train_crnn(crnn, ep_start=0, epochs=nepochs, batch_size=batch_size)
def ocr_crnn(filename,dtgen,model):
	text = ''
	input_size = (1024, 128, 1)

	im=pp.preprocess(filename,input_size)
	x_test = []
	x_test.append(im)
	x_test=pp.normalization(x_test)

	# predict() function will return the predicts with the probabilities
	predicts, _ = model.predict(x=x_test,
								use_multiprocessing=False,
								ctc_decode=True,
								verbose=0)

	# decode to string
	predicts = [dtgen.tokenizer.decode(x[0]) for x in predicts]
	text=predicts[0]
	s=text.split()
	s=s[::-1]
	reco=' '.join(s)
	reco=reco.strip()
	print(reco)
	return reco

def loadCRNNModel():
	from data.generator import DataGenerator
	input_size = (1024, 128, 1)
	dtgen = DataGenerator(source=source_path,
						  batch_size=batch_size,
						  charset=charset_base,
						  max_text_length=max_text_length)

	from network.model import HTRModel

	# create and compile HTRModel
	model = HTRModel(architecture=arch,
					 input_size=input_size,
					 vocab_size=dtgen.tokenizer.vocab_size,
					 beam_width=10,
					 stop_tolerance=20,
					 reduce_tolerance=15)

	model.compile(learning_rate=0.001)
	model.summary(output_path, "summary.txt")

	# get default callbacks and load checkpoint weights file (HDF5) if exists

	model.load_checkpoint(target='handwritten-text-recognition/output-KHATT-distorted/khatt/flor/checkpoint_weights.hdf5')
	return dtgen,model
def recognition_hard():
	path_test = 'Hito-docs/DatasetKHATT1_hard3/'
	set='distorted_hard3'
	list_lines = read_file(rootPath + 'src/Sets/lines.txt')
	dtgen, model = loadCRNNModel()
	list_image_valid = read_file(rootPath + 'src/Sets/list_test')
	list_reco_c = []
	list_reco_w = []
	list_truth_c = []
	list_truth_w = []
	i=0
	for im in list_image_valid:
		matched_lines = [s for s in list_lines if im in s]
		# print(matched_lines)
		l = matched_lines[0]
		l1 = l.split()
		text_line = l1[8]
		text_line = normalizeTranscription(text_line)
		truth_char = text_line
		li = text_line.split()

		if len(li) < 128:
			print('im : ' + str(i))
			i=i+1
			gen_txt = ocr_crnn(path_test + im + '.tif', dtgen, model)
			list_reco_c.append(gen_txt + '\n')
			list_truth_c.append(truth_char + '\n')
			words = gen_txt
			words = words.replace(' ', '')
			words = words.replace(' ', '')
			words = words.replace('sp', ' ')

			list_reco_w.append(words + '\n')
			twords = truth_char
			twords = twords.replace(' ', '')
			twords = twords.replace(' ', '')
			twords = twords.replace('sp', ' ')
			list_truth_w.append(twords + '\n')

	path_result ='handwritten-text-recognition/output-KHATT-distorted/khatt/flor/hard3'
	if not os.path.exists(path_result):
		os.makedirs(path_result)

	f = codecs.open(path_result + 'c_reco_' + set + '.txt', 'w', 'utf-8')
	f.writelines(list_reco_c)
	f.close()

	f = codecs.open(path_result + 'c_truth_' + set + '.txt', 'w', 'utf-8')
	f.writelines(list_truth_c)
	f.close()

	f1 = codecs.open(path_result + 'w_reco_' + set + '.txt', 'w', 'utf-8')
	f1.writelines(list_reco_w)
	f1.close()

	f1 = codecs.open(path_result + 'w_truth_' + set + '.txt', 'w', 'utf-8')
	f1.writelines(list_truth_w)
	f1.close()
	#####################compute result CER%
	command3 = 'wer -a -e  ' + path_result + 'c_truth_' + set + '.txt' + ' ' + path_result + 'c_reco_' + set + '.txt  >' + path_result + '/evaluate' + set + '_CER.txt'
	os.system(command3)
	#####################compute result WER%
	command2 = 'wer -a -e ' + path_result + 'w_truth_' + set + '.txt' + ' ' + path_result + 'w_reco_' + set + '.txt  >' + path_result + '/evaluate' + set + '_WER.txt'
	os.system(command2)

def recognition_easy():
	path_test = 'Hito-docs/DatasetKHATT1/'

	set='distorted_default_'
	list_lines = read_file(rootPath + 'src/Sets/lines.txt')
	dtgen, model = loadCRNNModel()
	list_image_valid = read_file(rootPath + 'src/Sets/list_test')
	list_reco_c = []
	list_reco_w = []
	list_truth_c = []
	list_truth_w = []
	i=0
	for im in list_image_valid:
		matched_lines = [s for s in list_lines if im in s]
		# print(matched_lines)
		l = matched_lines[0]
		l1 = l.split()
		text_line = l1[8]
		text_line = normalizeTranscription(text_line)
		truth_char = text_line
		li = text_line.split()

		if len(li) < 128:
			print('im : ' + str(i))
			i=i+1
			gen_txt = ocr_crnn(path_test + im + '.tif', dtgen, model)
			list_reco_c.append(gen_txt + '\n')
			list_truth_c.append(truth_char + '\n')
			words = gen_txt
			words = words.replace(' ', '')
			words = words.replace(' ', '')
			words = words.replace('sp', ' ')

			list_reco_w.append(words + '\n')
			twords = truth_char
			twords = twords.replace(' ', '')
			twords = twords.replace(' ', '')
			twords = twords.replace('sp', ' ')
			list_truth_w.append(twords + '\n')

	path_result ='handwritten-text-recognition/output-KHATT-distorted/khatt/flor/default'
	if not os.path.exists(path_result):
		os.makedirs(path_result)

	f = codecs.open(path_result + 'c_reco_' + set + '.txt', 'w', 'utf-8')
	f.writelines(list_reco_c)
	f.close()

	f = codecs.open(path_result + 'c_truth_' + set + '.txt', 'w', 'utf-8')
	f.writelines(list_truth_c)
	f.close()

	f1 = codecs.open(path_result + 'w_reco_' + set + '.txt', 'w', 'utf-8')
	f1.writelines(list_reco_w)
	f1.close()

	f1 = codecs.open(path_result + 'w_truth_' + set + '.txt', 'w', 'utf-8')
	f1.writelines(list_truth_w)
	f1.close()
	#####################compute result CER%
	command3 = 'wer -a -e  ' + path_result + 'c_truth_' + set + '.txt' + ' ' + path_result + 'c_reco_' + set + '.txt  >' + path_result + '/evaluate' + set + '_CER.txt'
	os.system(command3)
	#####################compute result WER%
	command2 = 'wer -a -e ' + path_result + 'w_truth_' + set + '.txt' + ' ' + path_result + 'w_reco_' + set + '.txt  >' + path_result + '/evaluate' + set + '_WER.txt'
	os.system(command2)

if __name__ == '__main__':
	train(150,32)

	recognition_hard()

max_stars_repo_name: sanakhamekhem/GAN-HTR

max_issues_repo_name: sanakhamekhem/GAN-HTR

max_forks_repo_name: sanakhamekhem/GAN-HTR


text:
#!/usr/bin/env python
# coding: utf-8

import numpy as np
import keras
from keras.models import Sequential
from keras.layers.core import Dense
from keras.layers import Dropout,Conv2D
from sklearn.model_selection import train_test_split
import csv
import pandas as pd
import matplotlib.pyplot as plt
from keras.utils import to_categorical
from sklearn.preprocessing import LabelBinarizer
encoder = LabelBinarizer()

print("This is the points model")

main = pd.read_csv('dataset/main.csv')
home = pd.read_csv('dataset/main.csv',usecols=['Team'])
away = pd.read_csv('dataset/main.csv',usecols=['Opponent']).values
ha = pd.read_csv('dataset/main.csv',usecols=['Home']).values
#One Hot encoding

home = encoder.fit_transform(home)
away = encoder.fit_transform(away)
homeaway = encoder.fit_transform(ha)

team = np.append(home,away,axis=1)

#Loading stats
#Preparing Data
statistics = pd.read_csv('dataset/main.csv')
statistics = statistics[['FieldGoalsAttempted','FreeThrows','FreeThrowsAttempted','FreeThrows.','OffRebounds','TotalRebounds','Assists','Steals','Blocks','Turnovers','TotalFouls',
                        'Opp.FreeThrows',
                        'Opp.FreeThrowsAttempted', 'Opp.FreeThrows.', 'Opp.OffRebounds', 'Opp.TotalRebounds',
                        'Opp.Assists', 'Opp.Steals', 'Opp.Blocks', 'Opp.Turnovers', 'Opp.TotalFouls',
                        'Opp.FieldGoalsAttempted', 'Opp.FieldGoals.', 'Opp.3PointShots']].values
oneHot = np.append(team,homeaway,axis=1)
statistics = np.append(oneHot,statistics,axis=1)
#Output data
winLoss =  pd.read_csv('dataset/main.csv',usecols=['WINorLOSS']).values
winLoss = encoder.fit_transform(winLoss)
points = pd.read_csv('dataset/main.csv',usecols=['TeamPoints','OpponentPoints']).values
points = np.append(points,winLoss,axis=1)

X_train, X_test, y_train, y_test = train_test_split(statistics, points, test_size=0.20)

#MODEL
model = Sequential()
keras.optimizers.adam(lr=0.1)

model.add(Dense(30, activation='relu',input_dim=85))
model.add(Dropout(0.30))
model.add(Dense(25, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(3, activation = 'relu'))

model.compile(loss='mse',
              optimizer='adam',
              metrics=['accuracy'])

history = model.fit(np.asarray(X_train),np.asarray(y_train), epochs=100,batch_size=50)

#Testing
print(model.evaluate(X_test,y_test))
model.summary()
plt.plot(history.history['acc'])
plt.title('Model Accuracy on Training')
plt.ylabel('Accuracy')
plt.xlabel('Epochs')
plt.show()
plt.plot(history.history['loss'])
plt.title('Model loss reduction on Training')
plt.ylabel('Accuracy')
plt.xlabel('Epochs')
plt.show()

#CLI UI
#CAN ONLY KEYBOARD INTERUPPT!
# while(True):
# #NEED TO LOAD DATA AGAIN!
#     stats = pd.read_csv('dataset/predict.csv')
#     stats
#     home = pd.read_csv('dataset/predict.csv',usecols=['Team'])
#     away = pd.read_csv('dataset/predict.csv',usecols=['Opponent']).values
#     ha = pd.read_csv('dataset/predict.csv',usecols=['Home']).values
#     home = encoder.fit_transform(home)
#     away = encoder.fit_transform(away)
#     homeaway = encoder.fit_transform(ha)

#     t1 = input("Enter home team: ")
#     t2 = input("Enter away team: ")

#     #Finding the row where these teams match
#     location = stats.loc[(stats['Team']==t1) & (stats['Opponent']==t2)&(stats['Home']=='Home')].index
#     teams = np.append(home[location[0]],away[location[0]])
#     teams = np.append(teams,homeaway[location[0]])
#     statsPredict = stats[(stats['Team']==t1) & (stats['Opponent']==t2)&(stats['Home']=='Home')]
#     statsPredict = statsPredict[['FieldGoalsAttempted','FreeThrows','FreeThrowsAttempted','FreeThrows.','OffRebounds','TotalRebounds','Assists','Steals','Blocks','Turnovers','TotalFouls',
#                             'Opp.FreeThrows',
#                             'Opp.FreeThrowsAttempted', 'Opp.FreeThrows.', 'Opp.OffRebounds', 'Opp.TotalRebounds',
#                             'Opp.Assists', 'Opp.Steals', 'Opp.Blocks', 'Opp.Turnovers', 'Opp.TotalFouls',
#                             'Opp.FieldGoalsAttempted', 'Opp.FieldGoals.', 'Opp.3PointShots']].values

#     statsPredict = statsPredict.mean(axis=0)
#     statsPredict = np.transpose(statsPredict)
#     teams = np.append(teams,statsPredict)
#     #PREDICTION!!!!!
#     #PRINTING
#     prediction = model.predict(np.asarray([teams]))
#     print(prediction)
    # if(prediction>0.5):
    #     print(t1, "WINS")
    # else:
    #     print(t2,"WINS")

max_stars_repo_name: vineetred/NBAPredictor

max_issues_repo_name: vineetred/NBAPredictor

max_forks_repo_name: vineetred/NBAPredictor