text:
# TensorMap.py
#
# The TensorMap takes any kind of data stored in an hd5 file
# tags it with a semantic interpretation and converts it into a structured numpy tensor.
# TensorMaps can be used as inputs, outputs, or hidden layers of models made by the model factory.
# TensorMaps can perform any computation by providing a callback function called tensor_from_file.
# A default tensor_from_file will be attempted when a callback tensor_from_file is not provided.
# TensorMaps guarantee shape, name, interpretation and mapping from hd5 to numpy array.

import logging
import datetime
from enum import Enum, auto
from typing import Any, Union, Callable, Dict, List, Optional, Tuple

import h5py
import numcodecs
import numpy as np
import tensorflow as tf
from tensorflow.keras import Model
from tensorflow.keras.utils import to_categorical

from ml4h.normalizer import Normalizer, Standardize, ZeroMeanStd1
from ml4h.defines import StorageType, JOIN_CHAR, STOP_CHAR, PARTNERS_READ_TEXT
from ml4h.metrics import sentinel_logcosh_loss, survival_likelihood_loss, cox_hazard_loss, pearson
from ml4h.metrics import per_class_recall, per_class_recall_3d, per_class_recall_4d, per_class_recall_5d
from ml4h.metrics import per_class_precision, per_class_precision_3d, per_class_precision_4d, per_class_precision_5d

MEAN_IDX = 0
STD_IDX = 1
DEFAULT_TIME_TO_EVENT_CHANNELS = {'event': 0, 'follow_up_days': 1}

class Interpretation(Enum):
    """Interpretations give TensorMaps semantics encoded by the numpy array the TensorMap yields.
    Interpretations tell us the kind of thing stored but nothing about its size.
    For example, a binary label and 2D pixel mask for segmentation should both have interpretation CATEGORICAL.
    CONTINUOUS Interpretations are the default and make sense for scalar values like height and weight
    as well as multidimensional arrays of raw pixel or voxel values.
    Providing explicit interpretations in TensorMap constructors is encouraged.
    Interpretations are used to set reasonable defaults values when explicit arguments are not provided."""
    CONTINUOUS = auto()
    CATEGORICAL = auto()
    EMBEDDING = auto()
    LANGUAGE = auto()
    TIME_TO_EVENT = auto()
    SURVIVAL_CURVE = auto()
    DISCRETIZED = auto()
    MESH = auto()

    def __str__(self):
        """class Interpretation.FLOAT_ARRAY becomes float_array"""
        return str.lower(super().__str__().split('.')[1])

class TimeSeriesOrder(Enum):
    NEWEST = 'NEWEST'
    OLDEST = 'OLDEST'
    RANDOM = 'RANDOM'

def _convert_old_normalization(normalization: Optional[Dict]) -> Optional[Normalizer]:
    """
    For backward compatibility. New TensorMaps should use a Normalizer.
    """
    if normalization is None:
        return
    if 'mean' in normalization and 'std' in normalization:
        return Standardize(mean=normalization['mean'], std=normalization['std'])
    if 'zero_mean_std1' in normalization:
        return ZeroMeanStd1()
    else:
        raise NotImplementedError(f'Cannot convert {normalization}')

class TensorMap(object):
    """Tensor maps encode the semantics, shapes and types of tensors available

        The mapping can be to numpy nd arrays, categorical labels, or continuous values.
        The tensor shapes can be inferred for categorical TensorMaps which provide a channel mapping dictionary.
        The channel map is a dict mapping a description string to an index into a numpy array i.e the tensor.
        For categorical data the resulting tensor is a one hot vector with a 1 at the channel index and zeros elsewhere.
        In general, new data sources require new TensorMaps and new tensor writers.
        Input and output names are treated differently to allow self mappings, for example auto-encoders
    """
    def __hash__(self):
        return hash((self.name, self.shape, self.interpretation))

    def __init__(
        self,
        name: str,
        interpretation: Optional[Interpretation] = Interpretation.CONTINUOUS,
        loss: Optional[Union[str, Callable]] = None,
        shape: Optional[Tuple[int, ...]] = None,
        model: Optional[Model] = None,
        metrics: Optional[List[Union[str, Callable]]] = None,
        parents: Optional[List["TensorMap"]] = None,
        sentinel: Optional[float] = None,
        validator: Optional[Callable] = None,
        cacheable: Optional[bool] = True,
        activation: Optional[Union[str, Callable]] = None,
        days_window: int = 1825,
        path_prefix: Optional[str] = None,
        loss_weight: Optional[float] = 1.0,
        channel_map: Optional[Dict[str, int]] = None,
        storage_type: Optional[StorageType] = None,
        dependent_map: Optional["TensorMap"] = None,
        augmentations: Optional[List[Callable[[np.ndarray], np.ndarray]]] = None,
        normalization: Optional[Normalizer] = None,
        annotation_units: Optional[int] = 32,
        tensor_from_file: Optional[Callable] = None,
        time_series_limit: Optional[int] = None,
        time_series_order: Optional[TimeSeriesOrder] = TimeSeriesOrder.NEWEST,
        time_series_lookup: Optional[Dict[int,Tuple]] = None,
        discretization_bounds: Optional[List[float]] = None,
    ):
        """TensorMap constructor

        :param name: String name of the tensor mapping
        :param interpretation: Enum specifying semantic interpretation of the tensor: is it a label, a continuous value an embedding...
        :param loss: Loss function or str specifying pre-defined loss function
        :param shape: Tuple of integers specifying tensor shape
        :param model: The model that computes the embedding layer, only used by embedding tensor maps
        :param metrics: List of metric functions of strings
        :param parents: List of TensorMaps which must be attached to the model graph before this one
        :param sentinel: If set, this value should never naturally occur in this TensorMap, it will be used for masking loss function
        :param validator: boolean function that validates a numpy arrays (eg checks ranges or NaNs)
        :param cacheable: boolean true if tensors made by this TensorMap can be cached.  Avoid this if there is randomness in tensor construction.
        :param activation: String specifying activation function
        :param days_window: Number of days to consider for survival curve TensorMaps, the longest possible follow up.
        :param path_prefix: Path prefix of HD5 file groups where the data we are tensor mapping is located inside hd5 files
        :param loss_weight: Relative weight of the loss from this tensormap
        :param channel_map: Dictionary mapping strings indicating channel meaning to channel index integers
        :param storage_type: StorageType of tensor map
        :param dependent_map: TensorMap that depends on or is determined by this one
        :param augmentations: Tensor shape preserving transformations not applied at validation or test time
        :param normalization: Dictionary specifying normalization values
        :param annotation_units: Size of embedding dimension for unstructured input tensor maps.
        :param tensor_from_file: Function that returns numpy array from hd5 file for this TensorMap
        :param time_series_limit: If set, indicates dynamic shaping and sets the maximum number of tensors in a time series to use
        :param time_series_order: When selecting tensors in a time series, use newest, oldest, or randomly ordered tensors
        :param time_series_lookup: Dict of time intervals filtering which tensors are used in a time series
        :param discretization_bounds: List of floats that delineate the boundaries of the bins that will be used
                                          for producing categorical values from continuous values
        """
        self.name = name
        self.interpretation = interpretation

        self.loss = loss
        self.model = model
        self.shape = shape
        self.path_prefix = path_prefix
        self.metrics = metrics
        self.parents = parents
        self.sentinel = sentinel
        self.validator = validator
        self.cacheable = cacheable
        self.activation = activation
        self.days_window = days_window
        self.loss_weight = loss_weight
        self.channel_map = channel_map
        self.storage_type = storage_type
        self.augmentations = augmentations
        self.normalization = normalization if isinstance(normalization, Normalizer) else _convert_old_normalization(normalization)
        self.dependent_map = dependent_map
        self.annotation_units = annotation_units
        self.tensor_from_file = tensor_from_file
        self.time_series_limit = time_series_limit
        self.time_series_order = time_series_order
        self.time_series_lookup = time_series_lookup
        self.discretization_bounds = discretization_bounds

        # Infer loss from interpretation
        if self.loss is None and self.is_categorical():
            self.loss = 'categorical_crossentropy'
        elif self.loss is None and self.is_continuous() and self.sentinel is not None:
            self.loss = sentinel_logcosh_loss(self.sentinel)
        elif self.loss is None and self.is_continuous():
            self.loss = 'mse'
        elif self.loss is None and self.is_survival_curve():
            self.loss = survival_likelihood_loss(self.shape[0]//2)
        elif self.loss is None and self.is_time_to_event():
            self.loss = cox_hazard_loss
        elif self.loss is None and self.is_language():
            self.loss = 'categorical_crossentropy'
        elif self.loss is None:
            self.loss = 'mse'

        # Infer activation from interpretation
        if self.activation is None and (self.is_categorical() or self.is_language()):
            self.activation = 'softmax'
        elif self.activation is None and self.is_continuous():
            self.activation = 'linear'
        elif self.activation is None and (self.is_survival_curve() or self.is_time_to_event()):
            self.activation = 'sigmoid'

        # Infer shape from channel map or interpretation
        if self.shape is None:
            self.shape = (2,) if self.is_time_to_event() else (len(channel_map),)
            # Setting time_series_limit indicates dynamic shaping which is always accompanied by 1st dim of None
            if self.time_series_limit is not None:
                self.shape = (None,) + self.shape

        if self.channel_map is None and self.is_time_to_event():
            self.channel_map = DEFAULT_TIME_TO_EVENT_CHANNELS

        if self.discretization_bounds is not None:
            self.input_shape = self.shape
            self.input_channel_map = self.channel_map
            self.shape = self.input_shape[:-1] + (len(self.discretization_bounds)+1,)
            self.channel_map = {f'channel_{k}': k for k in range(len(self.discretization_bounds) + 1)}

        self.infer_metrics()

        if self.tensor_from_file is None:
            self.tensor_from_file = _default_tensor_from_file

        if self.validator is None:
            self.validator = lambda tm, x, hd5: None

    def __repr__(self):
        return f'TensorMap({self.name}, {self.shape}, {self.interpretation})'

    def __eq__(self, other):
        if not isinstance(other, TensorMap):
            return NotImplemented
        else:
            self_attributes = self.__dict__.items()
            other_attributes = other.__dict__.items()

            for (self_field, self_value), (other_field, other_value) in zip(self_attributes, other_attributes):
                if self_field != other_field:
                    return False
                if not _is_equal_field(self_value, other_value):
                    logging.debug(f"Comparing two '{self.name}' tensor maps: '{self_field}' values '{self_value}' and '{other_value}' are not equal.")
                    return False
            return True

    def output_name(self):
        return JOIN_CHAR.join(['output', self.name, str(self.interpretation)])

    def input_name(self):
        return JOIN_CHAR.join(['input', self.name, str(self.interpretation)])

    def embed_name(self):
        return JOIN_CHAR.join(['embed', self.name, str(self.interpretation)])

    def is_categorical(self):
        """For most cases categorical and discretized TensorMaps should be handled in the same way.
        The two main differences are:
            1. Discretized TensorMaps are read from disk as continuous values before they are discretized
            2. Discretization is applied to discretized TensorMaps (obvious)
        """
        return self.interpretation == Interpretation.CATEGORICAL or self.interpretation == Interpretation.DISCRETIZED

    def is_continuous(self):
        return self.interpretation == Interpretation.CONTINUOUS

    def is_embedding(self):
        return self.interpretation == Interpretation.EMBEDDING

    def is_language(self):
        return self.interpretation == Interpretation.LANGUAGE

    def is_mesh(self):
        return self.interpretation == Interpretation.MESH

    def is_time_to_event(self):
        return self.interpretation == Interpretation.TIME_TO_EVENT

    def is_survival_curve(self):
        return self.interpretation == Interpretation.SURVIVAL_CURVE

    def is_discretized(self):
        return self.interpretation == Interpretation.DISCRETIZED

    def axes(self):
        return len(self.shape)

    def static_shape(self, limit=6):
        return tuple([size if size else limit for size in self.shape])

    def hd5_key_guess(self):
        if self.path_prefix is None:
            return f'/{self.name}/'
        else:
            return f'/{self.path_prefix}/{self.name}/'

    def hd5_first_dataset_in_group(self, hd5, key_prefix):
        if key_prefix not in hd5:
            raise ValueError(f'Could not find key:{key_prefix} in hd5.')
        data = hd5[key_prefix]
        if isinstance(data, h5py.Dataset):
            return data
        deeper_key_prefix = f'{key_prefix}{min(hd5[key_prefix])}/'
        return self.hd5_first_dataset_in_group(hd5, deeper_key_prefix)

    def normalize(self, np_tensor):
        if self.normalization is None:
            return np_tensor
        return self.normalization.normalize(np_tensor)

    def discretize(self, np_tensor):
        if not self.is_discretized():
            return np_tensor
        return to_categorical(
            np.digitize(np_tensor, bins=self.discretization_bounds),
            num_classes=len(self.discretization_bounds) + 1,
        )

    def postprocess_tensor(self, np_tensor, augment: bool, hd5: h5py.File):
        self.validator(self, np_tensor, hd5)
        np_tensor = self.apply_augmentations(np_tensor, augment)
        np_tensor = self.normalize(np_tensor)
        return self.discretize(np_tensor)

    def rescale(self, np_tensor):
        if self.normalization is None:
            return np_tensor
        return self.normalization.un_normalize(np_tensor)

    def apply_augmentations(self, tensor: np.ndarray, augment: bool) -> np.ndarray:
        if augment and self.augmentations is not None:
            for augmentation in self.augmentations:
                tensor = augmentation(tensor)
        return tensor

    def infer_metrics(self):
        if self.metrics is None and self.is_categorical():
            self.metrics = ['categorical_accuracy']
            if self.axes() == 1:
                self.metrics += per_class_precision(self.channel_map)
                self.metrics += per_class_recall(self.channel_map)
            elif self.axes() == 2:
                self.metrics += per_class_precision_3d(self.channel_map)
                self.metrics += per_class_recall_3d(self.channel_map)
            elif self.axes() == 3:
                self.metrics += per_class_precision_4d(self.channel_map)
                self.metrics += per_class_recall_4d(self.channel_map)
            elif self.axes() == 4:
                self.metrics += per_class_precision_5d(self.channel_map)
                self.metrics += per_class_recall_5d(self.channel_map)
        elif self.metrics is None and self.is_continuous() and self.axes() == 1:
            self.metrics = [pearson]
        elif self.metrics is None:
            self.metrics = []

def make_range_validator(minimum: float, maximum: float):
    def _range_validator(tm: TensorMap, tensor: np.ndarray, hd5: h5py.File):
        if not ((tensor > minimum).all() and (tensor < maximum).all()):
            raise ValueError(f'TensorMap {tm.name} failed range check.')
    return _range_validator

def no_nans(tm: TensorMap, tensor: np.ndarray, hd5: h5py.File):
    if np.isnan(tensor).any():
        raise ValueError(f'Skipping TensorMap {tm.name} with NaNs.')

def _translate(val, cur_min, cur_max, new_min, new_max):
    val -= cur_min
    val /= (cur_max - cur_min)
    val *= (new_max - new_min)
    val += new_min
    return val

def str2date(d):
    parts = d.split('-')
    if len(parts) < 2:
        raise ValueError(f'cant make date from {d}')
    return datetime.date(int(parts[0]), int(parts[1]), int(parts[2]))

def _is_equal_field(field1: Any, field2: Any) -> bool:
    """We consider two fields equal if
            a. they are not functions and they are equal, or
            b. one or both are functions and their names match
        If the fields are lists, we check for the above equality for corresponding
        elements from the list.
    """
    if isinstance(field1, list) and isinstance(field2, list):
        if len(field1) != len(field2):
            return False
        elif len(field1) == 0:
            return True

        fields1 = map(_get_name_if_function, field1)
        fields2 = map(_get_name_if_function, field2)

        return all([f1 == f2] for f1, f2 in zip(sorted(fields1), sorted(fields2)))
    else:
        return _get_name_if_function(field1) == _get_name_if_function(field2)

def _get_name_if_function(field: Any) -> Any:
    """We assume 'field' is a function if it's 'callable()'"""
    if callable(field):
        return field.__name__
    else:
        return field

def _default_continuous_tensor_from_file(tm, hd5, input_shape, input_channel_map):
    """ input_shape and input_channel_map are supplied as arguments rather than accessed as attributes of tm
    so that this function can be applied to TensorMaps that are to be discretized for which tm.input_shape and
    tm.input_channel_map reflect the state of the TensorMap post-discretization
    """
    missing = True
    continuous_data = np.zeros(input_shape, dtype=np.float32)
    if tm.hd5_key_guess() in hd5:
        missing = False
        data = tm.hd5_first_dataset_in_group(hd5, tm.hd5_key_guess())
        if tm.axes() > 1 or tm.shape[0] > 1:
            continuous_data = np.array(data)
        elif hasattr(data, "__shape__"):
            continuous_data[0] = data[0]
        else:
            continuous_data[0] = data[()]
    if missing and input_channel_map is not None and tm.path_prefix in hd5:
        for k in input_channel_map:
            if k in hd5[tm.path_prefix]:
                missing = False
                continuous_data[input_channel_map[k]] = hd5[tm.path_prefix][k][0]
    if missing and tm.sentinel is None:
        raise ValueError(f'No value found for {tm.name}, channel map is {input_channel_map} hd5_key_guess:{tm.hd5_key_guess()}')
    elif missing:
        continuous_data[:] = tm.sentinel
    return continuous_data

def _default_tensor_from_file(tm, hd5, dependents={}):
    """Reconstruct a tensor from an hd5 file

    Arguments
        tm: The TensorMap that describes the type of tensor to make
        hd5: The file where the tensor was saved
        dependents: A dict that maps dependent TensorMaps to numpy arrays
            if self has a dependent TensorMap it will be constructed and added here

    Returns
        A numpy array whose dimension and type is dictated by tm
    """
    if tm.is_categorical() and not tm.is_discretized():
        index = 0
        missing = True
        categorical_data = np.zeros(tm.shape, dtype=np.float32)
        if tm.hd5_key_guess() in hd5:
            data = tm.hd5_first_dataset_in_group(hd5, tm.hd5_key_guess())
            if tm.storage_type == StorageType.CATEGORICAL_INDEX or tm.storage_type == StorageType.CATEGORICAL_FLAG:
                index = int(data[0])
                categorical_data[index] = 1.0
            else:
                categorical_data = np.array(data)
            missing = False
        elif tm.storage_type == StorageType.CATEGORICAL_FLAG:
            categorical_data[index] = 1.0
            missing = False
        elif tm.path_prefix in hd5 and tm.channel_map is not None:
            for k in tm.channel_map:
                if k in hd5[tm.path_prefix]:
                    categorical_data[tm.channel_map[k]] = 1.0
                    missing = False
        if missing:
            raise ValueError(f"No HD5 data found at prefix {tm.path_prefix} found for tensor map: {tm.name}.")
        return categorical_data
    elif tm.is_continuous():
        return _default_continuous_tensor_from_file(tm, hd5, tm.shape, tm.channel_map)
    elif tm.is_discretized():
        return _default_continuous_tensor_from_file(tm, hd5, tm.input_shape, tm.input_channel_map)
    elif tm.is_embedding():
        input_dict = {}
        for input_parent_tm in tm.parents:
            input_dict[input_parent_tm.input_name()] = np.expand_dims(input_parent_tm.tensor_from_file(input_parent_tm, hd5), axis=0)
        return tm.model.predict(input_dict)
    elif tm.is_language():
        tensor = np.zeros(tm.shape, dtype=np.float32)
        if PARTNERS_READ_TEXT in tm.name:
            caption = decompress_data(data_compressed=hd5[tm.name][()], dtype=hd5[tm.name].attrs['dtype'])
        else:
            caption = str(tm.hd5_first_dataset_in_group(hd5, tm.hd5_key_guess())[()]).strip()
        char_idx = np.random.randint(tm.shape[0], len(caption) + 1)
        if char_idx == len(caption):
            next_char = STOP_CHAR
        else:
            next_char = caption[char_idx]
        if tm.dependent_map is not None:
            dependents[tm.dependent_map] = np.zeros(tm.dependent_map.shape, dtype=np.float32)
            dependents[tm.dependent_map][tm.dependent_map.channel_map[next_char]] = 1.0
            for i, k in enumerate(range(char_idx - tm.shape[0], char_idx)):
                if caption[k] not in tm.dependent_map.channel_map:
                    logging.warning(f'Could not find character {caption[k]} in channel map: {tm.dependent_map.channel_map}')
                    continue
                tensor[i, tm.dependent_map.channel_map[caption[k]]] = 1.0
        return tensor
    else:
        raise ValueError(f'No default tensor_from_file for TensorMap {tm.name} with interpretation: {tm.interpretation}')

def decompress_data(data_compressed: np.array, dtype: str) -> np.array:
    """Decompresses a compressed byte array. If the primitive type of the data
    to decompress is a string, calls decode using the zstd codec. If the
    primitive type of the data to decompress is not a string (e.g. int or
    float), the buffer is interpreted using the passed dtype."""
    codec = numcodecs.zstd.Zstd()
    data_decompressed = codec.decode(data_compressed)
    if dtype == 'str':
        data = data_decompressed.decode()
    else:
        data = np.frombuffer(data_decompressed, dtype)
    return data


text:
#!/usr/bin/env python3

from tensorflow.keras import callbacks
import argparse
import data
import model as m
import numpy as np
import os
import tensorflow as tf
import triplet_selection
import util as u

parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--img-dir', type=str, default='imgs')
parser.add_argument('--batch-size', type=int, default=16, help="note: effective batch is x3")
parser.add_argument('--embedding-dim', type=int, default=64, help="image embedding dim")
parser.add_argument('--learning-rate', type=float, default=1e-3, help="learning rate for adam")
parser.add_argument('--margin', type=float, default=1e-3, help="hinge loss margin")
parser.add_argument('--negative-frame-range', type=int, default=None,
                    help="select negative +/- this value; if None use entire range. only valid"
                         " for --negative-selection-mode=ranged_frame")
parser.add_argument('--negative-selection-mode', type=str, default='random_frame_random_run',
                    help='one of [random_frame_random_run, random_frame_same_run, ranged_frame]')
parser.add_argument('--epochs', type=int, default=20)
parser.add_argument('--steps-per-epoch', type=int, default=20)
parser.add_argument('--run', type=str, default='.',
                    help='run name to use as postfix for model saving, tb output')
parser.add_argument('--model-input', type=str, default=None,
                    help='if set, load weights from this model file')

opts = parser.parse_args()
print(opts)

u.ensure_dir_exists("runs/%s" % opts.run)

triplet_selector = triplet_selection.TripletSelection(opts.img_dir,
                                                      opts.negative_frame_range,
                                                      opts.negative_selection_mode)

examples = data.a_p_n_iterator(opts.batch_size, triplet_selector)

model, inputs, loss_fn = m.construct_model(opts.embedding_dim,
                                           opts.model_input,
                                           opts.learning_rate,
                                           opts.margin)

class NumZeroLossCB(callbacks.Callback):

    def __init__(self, batch_size=16):
        self.batch_size = batch_size
        self.sess = tf.Session()
        self.examples = (data.a_p_n_iterator(self.batch_size, triplet_selector)
                         .make_one_shot_iterator().get_next())
        self.summary_writer = tf.summary.FileWriter("tb/%s" % opts.run)

    def on_epoch_end(self, epoch, logs):
        # TODO: how do we just use the models iterator here? don't care
        # that it's "wastes" examples doing this eval, it's all generator
        # based anyways...
        next_egs = self.sess.run(self.examples)
        sess = tf.keras.backend.get_session()
        summaries = sess.run(loss_fn.summaries, feed_dict={inputs: next_egs})
#        percentage_non_zero = np.count_nonzero(per_elem_loss) / self.batch_size
        # log stats
        for summary in summaries:
            self.summary_writer.add_summary(summary, global_step=epoch)
        self.summary_writer.flush()

callbacks = [callbacks.ModelCheckpoint(filepath="runs/%s/model.{epoch}.hdf5" % opts.run),
             callbacks.TensorBoard(log_dir="tb/%s" % opts.run),
             callbacks.TerminateOnNaN(),
             NumZeroLossCB()]
model.fit(examples,
          epochs=opts.epochs,
          verbose=1,
          steps_per_epoch=opts.steps_per_epoch,
          callbacks=callbacks)

ext: py

reason: import numpy


text:
#
# NEUCHIPS CONFIDENTIAL
#
# Copyright (C) 2018-2020 NEUCHIPS Corp.
# All Rights Reserved.
# Author: Tzu-Jen Julius Lo <tzujen_lo@neuchips.ai>
#
# The information and source code contained herein is the exclusive property
# of NEUCHIPS CORPORATION and may not be disclosed, examined or reproduced
# in whole or in part without explicit written authorization from the company.
#
import ctypes
from datetime import datetime
import math
import multiprocessing as mp
import numpy as np
import os
import queue
import threading
import time
import torch

#
# FPGA control registers layout
#
REG_ADDR_BASE = 0x4001000
REG_ADDR_CTRL = REG_ADDR_BASE
REG_ADDR_BUFFER_SET = REG_ADDR_BASE + 0x8
REG_ADDR_BATCH_CTRL = REG_ADDR_BASE + 0x14

REG_ADDR_BATCH_BITMAP = REG_ADDR_BASE + 0x18
REG_ADDR_BATCH_BITMAP_HI = REG_ADDR_BASE + 0x1c
REG_ADDR_DMA_ITER = REG_ADDR_BASE + 0x20
REG_ADDR_DMA_ITER_RES = REG_ADDR_BASE + 0x24

REG_ADDR_BATCH_START = REG_ADDR_BASE + 0xc
REG_ADDR_INIT_DL_CMPLT = REG_ADDR_BASE + 0x130
REG_ADDR_BATCH_FINISH = REG_ADDR_BASE + 0x134

LSHFT_BCHCTRL_BURST_LENGTH = 16

MASK_CTRL_RESET = 0x10000
MASK_BCHCTRL_BATCH_SIZE = 0x7f

#
# PCIE configurations
#
PCIE_VENDOR_ID = 0x1172
PCIE_PRODUCT_ID = 0xe003

USER_BAR = 4

DMA_CHUNK_SIZE = 0x80000000

bytes_per_inf = 308  # for batch 16
bytes_per_dma_iter = 16384
bytes_per_dma_cycle = 64

settings = [
    0x1a801241, 0x0d036666, 0x35028700, 0x350c8700,
    0x008015e4, 0x00d04c5e, 0x00000000, 0x0fa4a133,
    0x03012201, 0x01002030, 0x01230000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
    0x0307b1a0, 0x00000000, 0x03ffffff, 0x00000000,
    0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000,
    0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000,
    0x0000000f, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x33400333, 0x32342333, 0x33232414, 0x00000044,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x33400333, 0x32342333, 0x33232414, 0x00000044,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x33400333, 0x32342333, 0x33232414, 0x00000044,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x33400333, 0x32342333, 0x33232414, 0x00000044,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x478e2280, 0x31b27500, 0x31d43980,
    0x34c3a280, 0x000013fd, 0xe99c5300, 0x000008a0,
    0x0000137e, 0x00000000, 0x31057d80, 0x2e9eac00,
    0x000013ef, 0x00000000, 0x34eb3600, 0x00001277,
    0x000013f9, 0x00000ea7, 0x000013e1, 0x00000000,
    0x25fbab80, 0x00000000, 0x304b3b00, 0x47da6400,
    0x00001312, 0x000013bd, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000000, 0x00000000, 0x00000000, 0x00000000,
    0x00000007, 0x47f3ec00, 0x47f8ee00, 0x31e2e980,
    0x00000000, 0x00000000, 0x00000000, 0x00000190,
    0x50000020, 0x10c00000, 0x28000000, 0xf0400040,
    0x00000000, 0x7ff00030, 0x7800000a, 0x00000000,
    0x78000006, 0x00000000, 0x50100020, 0x10c40000,
    0x28000000, 0xf0400040, 0x00003800, 0x7ff00030,
    0x78000007, 0x00000000, 0x40000020, 0x10a40000,
    0x30000020, 0x10400040, 0x04007010, 0x1ff00030,
    0x7a000012, 0x00000000, 0x50000010, 0x10c40000,
    0x20000010, 0x10500040, 0x04087808, 0x1ff00030,
    0x0014000f, 0x0ff00030, 0x40000020, 0x10a00000,
    0x2800001e, 0x10200040, 0x000c800f, 0x0ff00030,
    0x2800001e, 0x10200040, 0x001b800f, 0x0ff00030,
    0x50100020, 0x10a00000, 0x2800001e, 0x10200040,
    0x50300020, 0x10a40000, 0x2800001e, 0x10200040,
    0x0023000f, 0x0ff00030, 0x50200020, 0x10a00000,
    0x40000020, 0x11000000, 0x30000040, 0x10600040,
    0x042a8020, 0x1ff00030, 0x78000012, 0x00000000,
    0x044a9020, 0x1ff00030, 0x40100020, 0x11000000,
    0x30000040, 0x10600040, 0x043a8820, 0x1ff00030,
    0x30000040, 0x10600040, 0x045a9820, 0x1ff00030,
    0x40200020, 0x11000000, 0x30000040, 0x10600040,
    0x20000040, 0x10600040, 0x046aa020, 0x1ff00030,
    0x7800000a, 0x00000000, 0x40300020, 0x11040000,
    0x50100020, 0x11240000, 0x20000040, 0x10600040,
    0x047aa820, 0x1ff00030, 0x50000020, 0x11200000,
    0x40000020, 0x11040000, 0x30000020, 0x10600040,
    0x048ab010, 0x1ff00030, 0x78000006, 0x00000000,
    0x50000000, 0x21340000, 0x20000010, 0x10700040,
    0x0092b808, 0x1ff00030, 0x78000007, 0x00000000
]

CARD_DEV_PATH = "/dev/altera_pcie"

#
# FPGA DDR4 layout
#
MEM_ADDR_ONCHIP = 0x00000000
MEM_ADDR_DDR4A = 0x800000000
MEM_ADDR_DDR4B = 0xa00000000
MEM_ADDR_DDR4C = 0xc00000000
MEM_ADDR_DDR4D = 0xe00000000

MEM_ADDR_SETTINGS = MEM_ADDR_ONCHIP
MEM_ADDR_BOT = MEM_ADDR_ONCHIP + 0x240
MEM_ADDR_TOP = MEM_ADDR_ONCHIP + 0x0

MEM_ADDR_RES = MEM_ADDR_ONCHIP
MEM_ADDR_INFS = MEM_ADDR_DDR4A + 0x147f3ec00
MEM_ADDR_RES_2 = MEM_ADDR_ONCHIP + 0x800
MEM_ADDR_INFS_2 = MEM_ADDR_DDR4A + 0x147f8ee00

#
# Enumerations
#
CARD_STAT_INVALID = -1
CARD_STAT_READY = 0
CARD_STAT_OCCUPIED = 1

def jprint(*args):
    return
    pid = os.getpid()
    iid = threading.get_ident()
    print("[%s] (%d:%03d)" % (datetime.now().strftime("%H:%M:%S.%f"), pid,
                              iid % 1000), *args)

#
# RecAccelBuff - RecAccel in/output buffer
#
class RecAccelBuff():

    def __init__(self, bid, addr_in, addr_out):
        self.bid = bid
        self.addr_in = addr_in
        self.addr_out = addr_out
        self.sync = mp.Event()
        # no need to wait for reading at the first time
        self.sync.set()

#
# RecAccel - NEUCHIPS RecAccel
#
# reg_read() - read a word from a register
# reg_read_locked() - read a word with @self.hwlock locked
# reg_write() - write a word to a register
# reg_write_locked() - write a word with @self.hwlock locked
#
# reg_write_read() - write a word and then verify it
# reg_write_bitwise_pulse() - raise a pulse
# reg_write_bitwise() - bitwise write to a register
#
# dma_read() - read @size bytes from FPGA DDR4
# dma_write() - write @data to FPGA DDR4
# dma_read_res_in_torch() - read results in troch
# mem_dump() - memory dump for debugging
#
# ready() - readiness of RecAccel
# hardware_init() - set up hardware initial configuration
# rundtrip_time() - estimated RecAccel inference time
# predict() - major function to interact with RecAccel
#
class RecAccel():

    def __init__(self, cid, init=True, batch_size=16, freq_mhz=200,
                 paral_dram=True):
        super().__init__()

        self.cid = cid
        self.status = CARD_STAT_INVALID

        if os.path.exists(CARD_DEV_PATH + str(cid)) is False:
            return

        self.clib = ctypes.cdll.LoadLibrary(
            "./python/cpython/lib/terasic_pcie_qsys.so")
        self.clib.PCIE_Read32.argtypes = (ctypes.c_uint, ctypes.c_int,
                                          ctypes.c_uint64,
                                          ctypes.POINTER(ctypes.c_uint))
        self.clib.PCIE_DmaRead.argtypes = (ctypes.c_uint, ctypes.c_uint64,
                                           ctypes.POINTER(ctypes.c_uint8),
                                           ctypes.c_uint)
        self.clib.PCIE_DmaWrite.argtypes = (ctypes.c_uint, ctypes.c_uint64,
                                            ctypes.POINTER(ctypes.c_uint8),
                                            ctypes.c_uint)

        # get a pcie file descriptor
        self.fd = self.clib.PCIE_Open(PCIE_VENDOR_ID, PCIE_PRODUCT_ID, cid)
        if self.fd == 0:
            return

        # for RecAccel HW synchronization
        self.hwlock = mp.Lock()

        self.batch_size = batch_size
        self.freq_mhz = freq_mhz
        self.mem_addr_ddr4 = [MEM_ADDR_DDR4A, MEM_ADDR_DDR4B,
                              MEM_ADDR_DDR4C, MEM_ADDR_DDR4D]
        self.init_dl_tol_cnt = 1000
        self.batch_extra_tol_cnt = 100

        #
        self.status = CARD_STAT_READY
        if init is True:
            self.status = self.hardware_init(MEM_ADDR_SETTINGS)

        #
        # data exchange
        #
        self.runq = mp.Queue(10)
        self.waitq = mp.Queue(10)

        # sync events
        self.n_tickets = 100
        self.tickets = mp.Queue()
        for t in range(self.n_tickets):
            self.tickets.put(t)
        self.tickets_event = [mp.Event() for t in range(self.n_tickets)]

        # in/out buffers
        self.buffs_pool = []
        self.buffs_pool.append(RecAccelBuff(0, MEM_ADDR_INFS,
                                            MEM_ADDR_RES))
        self.buffs_pool.append(RecAccelBuff(1, MEM_ADDR_INFS_2,
                                            MEM_ADDR_RES_2))
        self.buffs = mp.Queue()
        for b in range(len(self.buffs_pool)):
            self.buffs.put(b)

        self.paral_dram = paral_dram

    # reg_read - read a word from a register
    #
    # Return: a word
    def reg_read(self, addr, _fd=None):
        res = ctypes.c_uint(0)
        with self.hwlock:
            self.clib.PCIE_Read32(_fd or self.fd, USER_BAR, addr,
                                  ctypes.byref(res))
        # print(addr, res.value)
        return res.value

    # reg_read_locked - read a word with @self.hwlock locked
    #
    # Return: a word
    def reg_read_locked(self, addr, _fd=None):
        res = ctypes.c_uint(0)
        self.clib.PCIE_Read32(_fd or self.fd, USER_BAR, addr,
                              ctypes.byref(res))
        return res.value

    # reg_write - write a word to a register
    # @addr: address to write
    # @data: a word
    # @_fd: associated file descriptor
    #
    # Return: status of writing
    def reg_write(self, addr, data, _fd=None):
        with self.hwlock:
            self.clib.PCIE_Write32(_fd or self.fd, USER_BAR, addr, data)

    # reg_write_locked - write a word with @self.hwlock locked
    # @addr: address to write
    # @data: a word
    # @_fd: associated file descriptor
    #
    # Return: status of writing
    def reg_write_locked(self, addr, data, _fd=None):
        return self.clib.PCIE_Write32(_fd or self.fd, USER_BAR, addr, data)

    # reg_write_read - write a word and then verify it
    # @addr: address to write
    # @data: a word
    # @_fd: associated file descriptor
    #
    # Return: none
    def reg_write_read(self, addr, data, _fd=None):
        fd = _fd or self.fd
        with self.hwlock:
            self.reg_write_locked(addr, data, fd)
            readback = self.reg_read_locked(addr, fd)
        if readback != data:
            print("write %08x: 0x%08x (expect 0x%08x)" %
                  (addr, readback, data))

    # reg_write_bitwise_pulse - raise a pulse
    # @addr: address to write
    # @data: a word; each outstanding bit has a pulse simultaneously
    # @_fd: associated file descriptor
    #
    # Return: none
    def reg_write_bitwise_pulse(self, addr, data, _fd=None):
        fd = _fd or self.fd
        with self.hwlock:
            orig = self.reg_read_locked(addr, fd)
            self.reg_write_locked(addr, (orig | data), fd)
            self.reg_write_locked(addr, (orig & (~data)), fd)

    # reg_write_bitwise - bitwise write to a register
    # @addr: address to write
    # @data: a word, each outstanding bit is written
    # @_fd: associated file descriptor
    #
    # Return: none
    def reg_write_bitwise(self, addr, data, _fd=None):
        fd = _fd or self.fd
        with self.hwlock:
            orig = self.reg_read_locked(addr, fd)
            self.reg_write_locked(addr, (orig | data), fd)

    # dma_read - read @size bytes from FPGA DDR4
    #
    # Return: a byte list
    def dma_read(self, addr, size, log=True):
        jprint("dma_read  0x%09x %u" % (addr, size))

        res = []
        _addr = addr
        _size = size

        # Loop by @DMA_CHUNK_SIZE
        while _size > 0:
            chunk_size = min(_size, DMA_CHUNK_SIZE)
            data = (ctypes.c_uint8 * chunk_size)()
            ctypes.cast(data, ctypes.POINTER(ctypes.c_uint8))

            with self.hwlock:
                self.clib.PCIE_Write32(self.fd, USER_BAR, REG_ADDR_BASE + 0x10,
                                       0x8)
                ret = self.clib.PCIE_DmaRead(self.fd, _addr, data, chunk_size)
            if ret != 1:
                jprint("DMA read  0x%09x %12lu bytes failed" %
                       (_addr, chunk_size))
                exit(1)

            _addr += chunk_size
            _size -= chunk_size
            res.extend(data)
            del data

        if log is True:
            jprint("DMA read  0x%09x %12lu bytes" % (addr, size))
        return res

    # dma_write - write @data to FPGA DDR4
    # @addr: address to write
    # @data: data in bytes
    # @verify: true to perform read back verification
    #
    # Return: None
    def dma_write(self, addr, data, verify=False):
        jprint("dma_write 0x%09x %u" % (addr, len(data)))

        if len(data) % 4 != 0:
            print("DMA write 0x%09x %12lu bytes is not word-sized" %
                  (addr, len(data)))
            return

        assert(isinstance(data, bytes))

        _addr = addr
        _data = data
        _size = len(data)

        # Loop by @DMA_CHUNK_SIZE
        while _size > 0:
            chunk_size = min(_size, DMA_CHUNK_SIZE)
            _d = _data[:chunk_size]
            chars = (ctypes.c_uint8 * chunk_size)()
            ctypes.memmove(ctypes.addressof(chars), _d, chunk_size)
            del _d
            chars_len = ctypes.sizeof(chars)

            # Issue DMA operation
            with self.hwlock:
                self.clib.PCIE_Write32(self.fd, USER_BAR, REG_ADDR_BASE + 0x10,
                                       0x8)
                ret = self.clib.PCIE_DmaWrite(self.fd, _addr, chars, chars_len)
            if ret != 1:
                jprint("DMA write 0x%09x %12lu bytes failed" %
                       (_addr, chars_len))
                exit(1)

            if verify is True:
                res = self.dma_read(_addr, chars_len, log=False)
                for i in range(chars_len):
                    if res[i] == chars[i]:
                        continue

                    print("\nDMA verify error 0x%09x: %02x != %02x\n" %
                          (_addr+i, res[i], chars[i]))
                    print("DMA write 0x%09x %12lu bytes failed" %
                          (_addr, chars_len))
                    exit(1)

            del chars

            _addr += chunk_size
            _data = _data[chunk_size:]
            _size -= chunk_size

        jprint("DMA write 0x%09x %12lu bytes" % (addr, len(data)))

    # dma_read_res_in_torch - read results in troch
    # @addr: address of results to read
    # @size: expected size in byte
    # @q: de-quantization order
    #
    # Return: inference results in torch
    def dma_read_res_in_torch(self, addr, size, q):
        # 4-bytes alignment DRAM access
        words = 4
        rdata = self.dma_read(addr, words * math.ceil(size/words))

        res = np.asarray(rdata[:size]).astype(np.float32)
        res /= (1 << q)
        res = torch.from_numpy(res.reshape(size, 1))
        return res

    # mem_dump - memory dump for debugging
    #
    # Return: none
    def mem_dump(self, data, size):
        print(size)
        for i in range(0, size, 16):
            print("%02x %02x %02x %02x %02x %02x %02x %02x  \
                   %02x %02x %02x %02x %02x %02x %02x %02x" %
                  (data[i], data[i+1], data[i+2], data[i+3],
                   data[i+4], data[i+5], data[i+6], data[i+7],
                   data[i+8], data[i+9], data[i+10], data[i+11],
                   data[i+12], data[i+13], data[i+14], data[i+15]))

    # ready - readiness of RecAccel
    #
    # Return: true if RecAccel is available, otherwise false
    def ready(self):
        return (self.status == CARD_STAT_READY)

    # hardware_init - set up hardware initial configuration
    # @addr: address for config.
    #
    # Return: none
    def hardware_init(self, addr):

        def _word_to_chars(word):
            res = []
            wlen = len(word)
            for i in range(0, wlen, 8):
                for j in range(7, -1, -1):
                    t = word[i+j]
                    for _ in range(4):
                        res.append(t & 0xff)
                        t >>= 8
            return bytes(res)

        # write setting
        self.dma_write(addr, _word_to_chars(settings))

        # reset
        self.reg_write_bitwise_pulse(REG_ADDR_CTRL, MASK_CTRL_RESET)

        # kick-off FPGA to download settings
        self.reg_write_bitwise(REG_ADDR_CTRL, 0x1)
        # wait for completion
        cmplt = 0
        cnt = 0
        while cmplt == 0:
            cnt += 1
            if cnt > self.init_dl_tol_cnt:
                return CARD_STAT_INVALID
            cmplt = self.reg_read(REG_ADDR_INIT_DL_CMPLT)
        # clear by software
        self.reg_write(REG_ADDR_INIT_DL_CMPLT, 0)
        return CARD_STAT_READY

    # rundtrip_time - estimated RecAccel process time
    # @infs: inference count
    #
    # Return: time in second
    def rundtrip_time(self, infs):
        bursts = math.ceil(infs/self.batch_size)
        kilo_cycles = (bursts // 2) * 21
        if bursts % 2 == 1:
            kilo_cycles += 14
        # second per kilo cycles = 1000 * (1 / (freq_mhz() * 1000000))
        return (kilo_cycles * (1 / (self.freq_mhz * 1000)))

    # predict - major function to interact with RecAccel
    # @bid: buffer id for RecAccel usage
    # @batch_size: batch size
    # @burst_len: burst length
    # @fd: default file descriptor is acquired in RecAccel instance process,
    #      allowing other process to deploy its own @fd for parallelism
    #
    # Return: none
    def predict(self, bid, batch_size=16, burst_len=1, _fd=None):

        def _init(bid, batch_size, burst_len, fd):
            # select input buffer
            self.reg_write(REG_ADDR_BUFFER_SET, bid, fd)

            data = burst_len << LSHFT_BCHCTRL_BURST_LENGTH
            data |= (batch_size & MASK_BCHCTRL_BATCH_SIZE)
            self.reg_write_read(REG_ADDR_BATCH_CTRL, data, fd)

            # set batch bitmap
            data = (1 << batch_size) - 1
            self.reg_write_read(REG_ADDR_BATCH_BITMAP, data, fd)
            # TODO: batch_size > 32
            self.reg_write_read(REG_ADDR_BATCH_BITMAP_HI, 0, fd)

            # set DMA cycles
            data = bytes_per_inf * self.batch_size
            self.reg_write_read(REG_ADDR_DMA_ITER,
                                math.ceil(data / bytes_per_dma_iter), fd)
            data %= bytes_per_dma_iter
            self.reg_write_read(REG_ADDR_DMA_ITER_RES,
                                math.ceil(data / bytes_per_dma_cycle), fd)

        def _start(fd):
            self.reg_write(REG_ADDR_BATCH_START, 1, fd)

        # _wait - wait RecAccel for completion
        #
        # Return: none
        def _wait(self, n_infs, fd):
            done = 0
            cnt = 0
            time.sleep(self.rundtrip_time(n_infs))
            while done != 1:
                cnt += 1
                if cnt > self.batch_extra_tol_cnt:
                    print("RecAccel-%d" % (self.cid),
                          "failed to predict of time-out")
                    r = self.reg_read(REG_ADDR_BASE + 0x4, fd)
                    print("cycles", r)

                    self.status = CARD_STAT_INVALID
                    res = np.ones(n_infs).astype(float)
                    res /= (1 << 7)  # an arbitrary quantization factor
                    res = torch.from_numpy(res.reshape(n_infs, 1))
                    return res

                time.sleep(0.000010)
                done = self.reg_read(REG_ADDR_BATCH_FINISH, fd)
            # clear by software
            self.reg_write(REG_ADDR_BATCH_FINISH, 0, fd)

            # Debug logging
            # r = self.reg_read(REG_ADDR_BASE + 0x4, fd)
            # print("cycles", r)

        # Entry of prediction
        jprint("RecAccel-%d" % (self.cid), batch_size, burst_len)

        fd = _fd or self.fd
        _init(bid, batch_size, burst_len, fd)
        _start(fd)
        _wait(self, batch_size * burst_len, fd)

if __name__ == "__main__":
    acard = RecAccel(0, init=False)
    _data = acard.dma_read(MEM_ADDR_RES, 128)
    acard.mem_dump(_data, 128)


text:
# code base by Nanineye#2417

import os
import numpy as np
from PIL import Image
from numba import jit
from numba.core import types
from numba.typed import Dict

from utils.setup import stats
from utils.image.image_utils import hex_to_rgb
from utils.image.ciede2000 import ciede2000, rgb2lab
from utils.log import get_logger

logger = get_logger(__name__)

class InvalidStyleException(Exception):
    def __init__(self, *args: object) -> None:
        super().__init__(*args)

def parse_style_image(style_image: Image.Image):
    try:
        img_array = np.array(style_image)
        mask = img_array[:, :, 3] != 0
        symbols_per_line = 16
        style_size = int(style_image.width / symbols_per_line)
        res = np.zeros((symbols_per_line * symbols_per_line, style_size, style_size))
        color_idx = 0
        for i in range(symbols_per_line):
            for j in range(symbols_per_line):
                x0 = i * style_size
                y0 = j * style_size
                x1 = x0 + style_size
                y1 = y0 + style_size
                symbol = mask[x0:x1, y0:y1]
                res[color_idx] = symbol
                color_idx += 1
        return res, style_size

    except Exception:
        return None, None

def get_style_from_name(style_name):
    try:
        basepath = os.path.dirname(__file__)
        styles_folder = os.path.abspath(
            os.path.join(basepath, "..", "..", "..", "resources", "styles")
        )
        filename = os.path.join(styles_folder, style_name + ".png")
        img = Image.open(filename)
    except FileNotFoundError:
        raise InvalidStyleException(f"Couldn't find '{filename}'")

    style_array, style_size = parse_style_image(img)
    if style_array is None:
        raise InvalidStyleException("Couldn't parse the style image.")

    style = {"name": style_name, "size": style_size, "array": style_array}
    return style

none = {"name": "none", "size": 1, "array": np.array([[[1]]] * 255)}

dotted = {
    "name": "dotted",
    "size": 3,
    "array": np.array([[[0, 0, 0], [0, 1, 0], [0, 0, 0]]] * 255),
}

plus = {
    "name": "plus",
    "size": 3,
    "array": np.array([[[0, 1, 0], [1, 1, 1], [0, 1, 0]]] * 255),
}

bigdotted = {
    "name": "bigdotted",
    "size": 5,
    "array": np.array(
        [
            [
                [0, 0, 0, 0, 0],
                [0, 1, 1, 1, 0],
                [0, 1, 1, 1, 0],
                [0, 1, 1, 1, 0],
                [0, 0, 0, 0, 0],
            ]
        ]
        * 255
    ),
}

STYLES = [none, dotted, plus, bigdotted]
custom_styles = ["custom", "pgcustom", "numbers"]

for s in custom_styles:
    try:
        STYLES.append(get_style_from_name(s))
    except InvalidStyleException as e:
        logger.warning(f"failed to load '{s}' style: {e}")

def get_style(style_name: str):
    for style in STYLES:
        if style["name"] == style_name:
            return style
    return None

def get_rgba_palette():
    palette = stats.get_palette()
    palette = np.array([c["value"] for c in palette])
    res = []
    for i in palette:
        c = hex_to_rgb(i, "RGBA")
        res.append(c)
    return np.array(res)

def stylize(style, stylesize, palette, glow_opacity=0):
    res = np.zeros((len(palette), stylesize, stylesize, 4))
    for i in range(len(palette)):
        cstyle = np.zeros((stylesize, stylesize, 4))
        glow_value = np.copy(palette[i])
        glow_value[3] = glow_opacity * 255
        cstyle[:, :] = glow_value
        for j in range(stylesize):
            for k in range(stylesize):
                if style[i, j, k]:
                    cstyle[j, k] = palette[i] * style[i, j, k]
        res[i] = cstyle
    return res

@jit(
    nopython=True,
    cache=True,
    locals={"color_bit": types.uint64, "mapped_color_idx": types.uint8},
)
def _fast_reduce(array, palette, dist_func):
    cache = Dict.empty(key_type=types.uint64, value_type=types.uint8)
    res = np.empty(array.shape[:2], dtype=np.uint8)
    width = array.shape[1]
    for idx in range(array.shape[0] * array.shape[1]):
        i = idx // width
        j = idx % width
        alpha = array[i, j, 3]
        if alpha > 128:
            color = array[i, j, :3]
            color_bit = (color[0] << 16) + (color[1] << 8) + color[2]
            if color_bit in cache:
                mapped_color_idx = cache[color_bit]
            else:
                mapped_color_idx = dist_func(color, palette)
                cache[color_bit] = mapped_color_idx
        else:
            mapped_color_idx = 255
        res[i, j] = mapped_color_idx
    return res

def reduce(array: np.array, palette: np.array, matching="fast") -> np.array:
    """Convert an image array of RGBA colors to an array of palette index
    matching the nearest color in the given palette

    Parameters
    ----------
    array: a numpy array of RGBA colors (shape (h, w, 4))
    palette: a numpy array (shape (h, w, 1))
    matching: the algorithm to use to match the colors
    (fast = Euclidean distance, accurate = CIEDE2000)
    """
    matchings = ["fast", "accurate"]
    msg = f"Unkown matching '{matching}', choose from: {', '.join(matchings)}"
    assert matching in matchings, msg

    # convert to array just in case
    palette = np.asarray(palette, dtype=np.uint8)
    array = np.asarray(array, dtype=np.uint8)

    # Get rid of the alpha component
    palette = palette[:, :3]

    if matching == "fast":
        dist_func = nearest_color_idx_euclidean
    else:
        dist_func = nearest_color_idx_ciede2000
        palette = np.asarray([rgb2lab(color) for color in palette])

    res = _fast_reduce(array, palette, dist_func)
    return res

@jit(nopython=True, cache=True)
def nearest_color_idx_ciede2000(color, palette) -> int:
    """
    Find the nearest color to `color` in `palette` using CIEDE2000.

    Parameters
    ----------
    color: a rgb ndarray of uint8 of shape (3,)
    palette: a lab ndarray of shape (palette_size,3)
    """
    lab = rgb2lab(color)

    min_distance = np.inf
    nearest_color_idx = -1
    for i, palette_lab in enumerate(palette):
        distance = ciede2000(lab, palette_lab)
        if distance < min_distance:
            min_distance = distance
            nearest_color_idx = i
    return nearest_color_idx

@jit(nopython=True, cache=True)
def nearest_color_idx_euclidean(color, palette) -> int:
    """
    Find the nearest color to `color` in `palette` using the Euclidean distance

        Parameters
    ----------
    color: a rgb ndarray of uint8 of shape (3,)
    palette: a rgb ndarray of uint8 of shape (palette_size,3)

    """
    distances = np.sum((palette - color) ** 2, axis=1)
    return np.argmin(distances)

@jit(nopython=True, cache=True)
def fast_templatize(n, m, st, red, style_size):
    res = np.zeros((style_size * n, style_size * m, 4), dtype=np.uint8)
    for i in range(n):
        for j in range(m):
            if red[i, j] != 255:  # non-alpha values
                res[
                    style_size * i : style_size * i + style_size,
                    style_size * j : style_size * j + style_size,
                ] = st[red[i][j]]
    return res

def templatize(
    style: dict, image: Image.Image, glow_opacity=0, palette=None
) -> np.ndarray:
    style_array = style["array"]
    style_size = style["size"]
    image_array = np.array(image)

    n = image_array.shape[0]
    m = image_array.shape[1]

    if palette is None:
        palette = get_rgba_palette()

    st = stylize(style_array, style_size, palette, glow_opacity)
    res = fast_templatize(n, m, st, image_array, style_size)
    return res

reason: import numpy,from numba


text:
"""Topic Models from FOMC meeting minutes

- NMF, LSA, LDA, PLSI matrix decomposition models

Terence Lim
License: MIT
"""
import numpy as np
import pandas as pd
from pandas import Series, DataFrame
import os
import sklearn.feature_extraction, sklearn.decomposition
from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
from matplotlib.ticker import (MultipleLocator, AutoMinorLocator)
from finds.unstructured import Unstructured
from finds.readers import fetch_fomc
from finds.database import MongoDB
from settings import settings
mongodb = MongoDB(**settings['mongodb'])
fomc = Unstructured(mongodb, 'FOMC')
logdir = os.path.join(settings['images'], 'fomc')

# retrieve recessions dates for plotting
from finds.alfred import Alfred
alf = Alfred(api_key=settings['fred']['api_key'])
usrec = alf('USREC')
usrec.index = pd.DatetimeIndex(usrec.index.astype(str), freq='infer')
g = usrec.astype(bool) | usrec.shift(-1, fill_value=0).astype(bool)
g = (g != g.shift(fill_value=0)).cumsum()[g].to_frame()
g = g.reset_index().groupby('USREC')['date'].agg(['first','last'])
vspans = [(v[0], v[1]) for k, v in g.iterrows()]

# Update FOMC Minutes
dates = fomc['minutes'].distinct('date')       # check dates stored
catalog = fetch_fomc()     # check for new dates in fomc site, and retrieve
print(f"FOMC: {len(catalog)} dates {min(catalog.keys())}-{max(catalog.keys())}")
docs = {d: fetch_fomc(url) for d, url in catalog.items() if d not in dates}
print('NEW:', ", ".join([f"{k}: {len(v)} chars " for k,v in docs.items()]))

# keep starting approx on
# "Developments in Financial Markets and Open Market Operations
# earlier is re-affirmation of general policy statement
# delete schedule next meeting/approve minutes/signature block
import subprocess
import tempfile
results = list()   # loop to edit out head and tail of document
for date, initial_message in docs.items():  # manually trim text
    with tempfile.NamedTemporaryFile(suffix=".tmp") as f: # save temp file
        f.write(initial_message.encode("utf-8"))
        f.flush()
        subprocess.call([os.environ.get('EDITOR','emacs'), "-nw", f.name])
        f.seek(0)
        edited_text = f.read().decode("utf-8")        # keep edited text
        results.append({'date': date, 'text' : edited_text})
for doc in results: # store docs for new dates
    fomc.insert('minutes', doc, keys=['date'])

# Retrieve text
docs = Series({doc['date']: doc['text'] for doc in fomc.select('minutes')},
              name='minutes').sort_index()
DataFrame(docs)

# set stop words and vectorize inputs
StopWords = [w for w in set(STOPWORDS) if "'" not in w]
StopWords += ['january', 'february', 'march', 'april', 'may', 'june',
              'july', 'august', 'september', 'october', 'november',
              'december', 'first', 'second', 'third', 'fourth', 'twelve',
              'participants', 'members', 'meeting']
ngram_range = (2, 3)
max_df, min_df, max_features = 0.5, 3, 5000 # some reasonable constraints
tfidf_vectorizer = sklearn.feature_extraction.text.TfidfVectorizer(
    strip_accents='unicode',
    lowercase=True,
    stop_words=StopWords,
    ngram_range=ngram_range,        # (2, 2) for bigrams
    max_df=max_df, min_df=min_df, max_features=max_features,
    token_pattern=r"\b[^\d\W][^\d\W][^\d\W]+\b") #r'\b[^\d\W]+\b'
tf_vectorizer = sklearn.feature_extraction.text.CountVectorizer(
    strip_accents='unicode',
    lowercase=True,
    stop_words=StopWords,
    ngram_range=ngram_range,        # (2, 2) for bigrams
    max_df=max_df, min_df=min_df, max_features=max_features,
    token_pattern=r"\b[^\d\W][^\d\W][^\d\W]+\b")

# Define models
n_components = 4
algos = {
    'LSA': (sklearn.decomposition.TruncatedSVD(n_components=n_components),
            tfidf_vectorizer),
    'LDA': (sklearn.decomposition.LatentDirichletAllocation(
        n_components=n_components,
        learning_method='online',
        learning_offset = 50.0,
        max_iter = 5,
        random_state = 42), tf_vectorizer),
    'PLSI': (sklearn.decomposition.NMF(
        n_components=n_components,
        beta_loss='kullback-leibler',
        solver='mu',
        alpha=0.1,
        l1_ratio=0.5,
        max_iter=1000,
        random_state = 42), tf_vectorizer),
    'NMF': (sklearn.decomposition.NMF(
        n_components=n_components,
        random_state=42,
        alpha=0.1,
        l1_ratio=0.5), tfidf_vectorizer)}

models = dict()
dates = dict()
for ifig, (name, (base, vectorizer)) in enumerate(algos.items()):
    vectorized = vectorizer.fit_transform(docs.to_list())
    feature_names = vectorizer.get_feature_names()
    models[name] = base.fit(vectorized)
    topics = models[name].transform(vectorized)

    fig, ax = plt.subplots(num=1+ifig, clear=True, figsize=(5,4))
    date = pd.DatetimeIndex(docs.index.astype(str))
    ax.plot(date, topics)
    for a,b in vspans:
        if b >= min(date):
            ax.axvspan(a, min(b, max(date)), alpha=0.3, color='grey')
    ax.set_title(name)
    ax.legend([f"{i+1}" for i in range(n_components)] + ['Recession'],
              loc='lower left')
    plt.tight_layout(pad=2)
    for topic in range(topics.shape[1]):
        arg = DataFrame({'t': np.argmax(topics, axis=1), 'dt': docs.index})
        g = (arg!=arg.shift()).cumsum().groupby('t').agg(['first','last'])-1
        g['topic'] = arg.loc[g.iloc[:,1], 't'].values
    dates[name] = {topic: [(arg['dt'].iloc[row[0]], arg['dt'].iloc[row[1]])
                           for key, row in g.iterrows() if row[2] == topic]
                   for topic in range(topics.shape[1])}
    plt.savefig(os.path.join(logdir, name + 'topics.jpg'))

"""Display top features as wordclouds
- NMF model appears to provide most intuitive outputs
- 1990's appears to be dominated by concerns of domestic non-financial debt
- 2000's pivoted to high-tech and inflation
- 2008-2015 by mortage-backed securities and asset purchases
- 2016-2019 by fed fund target rates, with the previous topic returning in 2020
"""

for ifig, (name, model) in enumerate(models.items()):
    wc = WordCloud(height=300, width=500, colormap='cool')
    top_n = 20
    fig, axes = plt.subplots(2, 2, num=ifig+5, figsize=(10,6), clear=True)
    for topic, score in enumerate(model.components_):
        words = {feature_names[i].replace(' ','_') : score[i]
                 for i in score.argsort()[:-top_n - 1:-1]}
        print("Topic", topic+1, dates[name])
        print(list(words.keys()))
        ax = axes[topic//2, topic % 2]
        ax.imshow(wc.generate_from_frequencies(words))
        ax.axes.yaxis.set_visible(False)   # make axes ticks invisible
        ax.xaxis.set_ticks([])
        ax.xaxis.set_ticklabels([])
        ax.set_title(f"{name} Topic {topic}", fontsize=10)
        s = ", ".join([f"{d[0]}-{d[1]}" if d[0] != d[1] else f"{d[0]}"
                       for d in dates[name][topic]])
        ax.set_xlabel('Dates: ' + (s if len(s)<75 else '-- many --'),
                      fontsize=9, loc='left')
        plt.tight_layout(pad=2)
    plt.savefig(os.path.join(logdir, name + 'words.jpg'))
plt.show()

"""
traditional monetary policy -- reserve conditions and restraint, domestic and non-financial debt
demand factors: productivity, software and equipment, energy prices, inventories, inflation
labor market and long-run federal funds target objectives
asset repurchases, mbs, credit, loans, agency, treasury securities
"""

max_stars_repo_name: terence-lim/investment-data-science

max_issues_repo_name: terence-lim/investment-data-science

max_forks_repo_name: terence-lim/investment-data-science


text:
import os
import numpy as np
import librosa
from inputs import get_bit_rates_and_waveforms, get_truth_ds_filename_pairs
from inputs import read_file_pair, next_batch
import tensorflow as tf
from models import deep_residual_network

# Constants describing the training process.
BATCH_SIZE = 16

waveform_reduction_factor = 1
file_name_lists_dir = '/home/paperspace/Documents/EnglishSpeechUpsampler/aux'
model_checkpoint_file_name = os.path.join(file_name_lists_dir,
                                    'model_checkpoints/' +\
                                    'deep_residual_deep_residual_0_final.ckpt')

test_truth_ds_pairs = get_truth_ds_filename_pairs(file_name_lists_dir,
                                                  'test')

br_pairs, wf_pairs = get_bit_rates_and_waveforms(test_truth_ds_pairs[0])
true_br = br_pairs[0]
true_wf = wf_pairs[0]
waveform_max = int(true_wf.size/waveform_reduction_factor)
# true_wf = true_wf[::waveform_reduction_factor]
true_wf = true_wf[:waveform_max]
# reshape for mono waveforms
true_wf = true_wf.reshape((-1, 1))

# ################
# MODEL DEFINITION
# ################

bits_per_second = true_wf.size/10

train_flag, x, model = deep_residual_network(true_wf.dtype, true_wf.shape,
                                             tensorboard_output=False)

# placeholder for the truth label
y_true = tf.placeholder(true_wf.dtype,
                        shape=x.get_shape())

# ################
# ################

# #############
# LOSS FUNCTION
# #############

with tf.name_scope('waveform_mse'):
    waveform_mse = tf.reduce_mean(tf.square(tf.subtract(y_true, model)))
tf.summary.scalar('waveform_mse', waveform_mse)

# #############
# #############

# Add ops to restore all the variables.
saver = tf.train.Saver()

# create session
sess = tf.Session()

# restore model from checkpoint file
saver.restore(sess, model_checkpoint_file_name)

# #############
# TRAINING LOOP
# #############

test_loss_file = open('test_loss.txt', 'w')
count = 0
for pair in next_batch(BATCH_SIZE, test_truth_ds_pairs):
    loss_test = sess.run([waveform_mse],
                         feed_dict={train_flag: False,
                                    x: pair[1],
                                    y_true: pair[0]}
                         )
    test_loss_file.write('{}\n'.format(np.mean(loss_test)))
    print("Iteration {}, Test Loss {}".format((count + 1), loss_test))
    count += 1

test_loss_file.close()

truth, example = read_file_pair(test_truth_ds_pairs[0])
y_reco = model.eval(feed_dict={train_flag: False,
                               x: example.reshape(1, -1, 1)},
                    session=sess).flatten()

print('difference between truth and example (first 20 elements)')
print(truth.flatten()[:20] - example.flatten()[:20])
print('difference between truth and reconstruction (first 20 elements)')
print(truth.flatten()[:20] - y_reco[:20])

# if waveform_reduction_factor == 1:
print('writting output audio files')
librosa.output.write_wav('full_train_test_true.wav',
                         y=truth.flatten(), sr=true_br)
librosa.output.write_wav('full_train_test_ds.wav',
                         y=example.flatten(), sr=true_br)
librosa.output.write_wav('full_train_test_reco.wav',
                         y=y_reco, sr=true_br)
# #############
# #############

max_stars_repo_name: jeffersonchou/EnglishSpeechUpsampler

max_issues_repo_name: jeffersonchou/EnglishSpeechUpsampler

max_forks_repo_name: jeffersonchou/EnglishSpeechUpsampler


text:
import glob
import codecs
import string, re, pickle, math
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize,sent_tokenize
from tqdm import tqdm
import numpy as np
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
import json
import warnings

warnings.filterwarnings("ignore")

sent=[]
with open('F:/MTECH1/NLP/Assignment6/Challenge.jsonl') as f:
	for i in f:
		data=json.loads(i)
		sent.append(data)

f = codecs.open(f'F:/MTECH1/NLP/Assignment6/data.txt', encoding='utf-8')
sentence_list1=[]

for line in f:
	line=re.sub(r'\b\d+\b','',line)
	line=line.lower().split()
	token=[word.strip(string.punctuation) for word in line]
	sentence_list1.append(token)
#print(sentence_list1)
tagged_document=[TaggedDocument(sentence,tags=[str(i)]) for i,sentence in enumerate(sentence_list1)]

#print(tagged_document)
model =Doc2Vec(tagged_document, size = 200, window = 400, min_count = 1, workers = 10)
vocab=model.wv

print(vocab.vocab)
score=0
answer_list=[]
for i in range(0,len(sent)):
	if(sent[i]['answerKey']=="A"):
		answer_list.append(0)
	elif(sent[i]['answerKey']=="B"):
		answer_list.append(1)
	elif(sent[i]['answerKey']=="C"):
		answer_list.append(2)
	elif(sent[i]['answerKey']=="D"):
		answer_list.append(3)
	else:
		answer_list.append(sent[i]['answerKey'])

for i in tqdm(range(0,len(sent))):
	query1=''
	query1=sent[i]['question']['stem']
	temp_list1=query1.lower().split()
	token1=[word.strip(string.punctuation) for word in temp_list1]
	"""stopword=set(stopwords.words('english'))
	for w in token1:
		if w not in stopword:
			temp_list1.append(w)"""
	temp_list1=[]
	temp_list1=token1.copy()
	similarity_list=[]
	for j in range(len(sent[i]['question']['choices'])):

		query2=''
		query2=sent[i]['question']['choices'][j]['text']
		temp_list2=query2.lower().split()
		token2=[word.strip(string.punctuation) for word in temp_list2]
		"""stopword=set(stopwords.words('english'))
		temp_list2=[]
		for w in token2:
			if w not in stopword:
				temp_list2.append(w)"""
		temp_list2=[]
		temp_list2=token2.copy()
		temp_list3=temp_list1+temp_list2
		new_list=[]
		for word in temp_list3:
			if word not in vocab.vocab:
				pass
			else:
				new_list.append(word)
		#print(new_list)
		similar_doc = model.wv.most_similar(new_list)
		print(len(new_list))

		similarity_list.append(similar_doc)
	predicted_answer=[]
	max_value=max(similarity_list)
	for a in range(0,len(similarity_list)):
		if(max_value==similarity_list[a]):
			predicted_answer.append(a)

	if(len(predicted_answer)==0):
		score=score+0
	else:
		if(answer_list[i]) in predicted_answer:
			score=score+(1/(len(predicted_answer)))
print()
print("Similarity Score is:",score)
print("Accuracy:",score/500)


text:
# -*- coding: utf-8 -*-
"""
Use Extended Kalman Filter on GPS + IMU for better localization.
"""
# Author: Runsheng Xu <rxx3386@ucla.edu>
# License: MIT

import math
import numpy as np

class ExtentedKalmanFilter(object):
    """
    Extended Kalman Filter implementation for gps and imu.

    Parameters
    -dt : float
        The step time for kalman filter calculation.

    Attributes
    -Q : numpy.array
        predict state covariance.
    -R : numpy.array
        Observation x,y position covariance.
    -time_step : float
        The step time for kalman filter calculation.
    -xEst : numpy.array
        Estimated x values.
    -PEst : numpy.array
        The estimated P values.
    """

    def __init__(self, dt):

        self.Q = np.diag([
            0.2,  # variance of location on x-axis
            0.2,  # variance of location on y-axis
            np.deg2rad(0.1),  # variance of yaw angle
            0.001  # variance of velocity
        ]) ** 2  # predict state covariance

        self.R = np.diag([0.5, 0.5, 0.2]) ** 2

        self.time_step = dt

        self.xEst = np.zeros((4, 1))
        self.PEst = np.eye(4)

    def motion_model(self, x, u):
        """
        Predict current position and yaw based on previous result
        (X = F * X_prev + B * u).

        Args:
            -x (np.array): [x_prev, y_prev, yaw_prev, v_prev], shape: (4, 1).
            -u (np.array): [v_current, imu_yaw_rate], shape:(2, 1).

        Returns:
          x (np.array): predicted state.
        """
        F = np.array([[1.0, 0, 0, 0],
                      [0, 1.0, 0, 0],
                      [0, 0, 1.0, 0],
                      [0, 0, 0, 0]])

        B = np.array([[self.time_step * math.cos(x[2, 0]), 0],
                      [self.time_step * math.sin(x[2, 0]), 0],
                      [0.0, self.time_step],
                      [1.0, 0.0]])

        x = F @ x + B @ u

        return x

    def jacob_f(self, x, u):
        """
        Jacobian of Motion Model motion model

        Args:
            -x (np.array): Input X array.
        Returns:
            -jF (np.array):  Jacobian of Motion Model motion model.

        """
        yaw = x[2, 0]
        v = u[0, 0]
        jF = np.array([
            [1.0, 0.0, -self.time_step * v * math.sin(yaw),
             self.time_step * math.cos(yaw)],
            [0.0, 1.0, self.time_step * v * math.cos(yaw),
             self.time_step * math.sin(yaw)],
            [0.0, 0.0, 1.0, 0.0],
            [0.0, 0.0, 0.0, 1.0]])

        return jF

    def observation_model(self, x):
        """
        Project the state.array to sensor measurement.array.

        Args:
            -x (np.array): [x, y, yaw, v], shape: (4. 1).

        Returns:
            -z (np.array): predicted measurement.

        """
        H = np.array([
            [1, 0, 0, 0],
            [0, 1, 0, 0],
            [0, 0, 1, 0]
        ])

        z = H @ x

        return z

    def run_step_init(self, x, y, heading, velocity):
        """
        Initalization for states.

        Args:
            -x (float): The X coordinate.
            -y (float): Tehe y coordinate.
            -heading (float): The heading direction.
            -velocity (float): The velocity.

        """
        self.xEst[0] = x
        self.xEst[1] = y
        self.xEst[2] = heading
        self.xEst[3] = velocity

    def run_step(self, x, y, heading, velocity, yaw_rate_imu):
        """
        Apply EKF on current measurement and previous prediction.

        Args:
            -x (float): x(esu) coordinate from
             gnss sensor at current timestamp.
            -y (float): y(esu) coordinate from
             gnss sensor at current timestamp.
            -heading (float): heading direction at current timestamp.
            -velocity (float): current speed.
            -yaw_rate_imu (float): yaw rate rad/s from IMU sensor.
        Returns:
            - xEST (np.array): The corrected x, y, heading,
              and velocity information.
        """

        # gps observation
        z = np.array([x, y, heading]).reshape(3, 1)
        # velocity and imu yaw rate
        u = np.array([velocity, yaw_rate_imu]).reshape(2, 1)

        # EKF starts
        xPred = self.motion_model(self.xEst, u)
        jF = self.jacob_f(self.xEst, u)
        PPred = jF @ self.PEst @ jF.T + self.Q

        # Jacobian of Observation Model
        jH = np.array([
            [1, 0, 0, 0],
            [0, 1, 0, 0],
            [0, 0, 1, 0]])
        zPred = self.observation_model(xPred)
        y = z - zPred
        S = jH @ PPred @ jH.T + self.R
        K = PPred @ jH.T @ np.linalg.inv(S)
        self.xEst = xPred + K @ y
        self.PEst = (np.eye(len(self.xEst)) - K @ jH) @ PPred

        return self.xEst[0][0], \
            self.xEst[1][0], \
            self.xEst[2][0], \
            self.xEst[3][0]


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 11 16:39:01 2018

@author: gotamist
"""
import numpy as np
from data_generator import AudioGenerator
#import re
from itertools import chain
#from textblob import TextBlob as tb
#import kenlm
import panphon.distance
from fuzzy import DMetaphone
import pyphen

def create_DMetaphone_list(wordset):
    dm = DMetaphone(5)
    codedict = {}
    codeset = set()
    for word in wordset:
        strings2 = [  str(code)[2:-1] for code in dm(word) if code is not None ]
        codedict[word] = strings2
        codeset.update( set(strings2) )
    return codedict, codeset

def code2words( code_dict, codeset ):
    '''  create a lookup table where for every Metaphone code, you store all
    the words that have that code.  The inputs, code_dict and codeset, come
    for a specific corpus and are created by create_DMetaphone_list()'''
    c2w_dict = {}
    for word, code_list in code_dict.items():
        for code in code_list:
            if code in c2w_dict.keys():
                c2w_dict[ code ].append(word)
            else:
                    c2w_dict[ code ] = [word]

    return c2w_dict

def num_syllables_in_word(word):
    dic = pyphen.Pyphen(lang='en')
    x=dic.inserted( word )
    return x.count('-')+1

def count_syllables_using_vowels( word ):
    vowels = ['a','e','i','o','u']
    count = 0
    for i in range(len(word)):
        if word[i] in vowels:
            count += 1
        if i > 2:
            if word[i] == 'y' and word[i-1] not in vowels:
                   count += 1
    return count

def sound_neighborhood( string, codeset, code2words_dict, edit_dist ):
    '''Find a phonological neighborhood around given string from a
    given codeset and code2words dictionary (both of which correspond to a corpus)'''
    dm = DMetaphone(5)
    dm_codes = [  str(code)[2:-1] for code in dm(string) if code is not None ]
    code_nbd = set()
    for code in dm_codes:
        code_nbd.update( get_neighborhood( code, codeset, edit_dist) )
    sound_nbd = []
    for similar_code in code_nbd:
        sound_nbd.extend( code2words_dict[similar_code] )
    return sound_nbd

def sound_bigram_predict(input_sentence, train_dictionary, predict_dictionary, lmodel, radius=1.5): #input is a string
    # assumes that the output of the DNN is of the right length
    codedict, codeset = create_DMetaphone_list(predict_dictionary)
    code2words_dict = code2words( codedict, codeset )

    inp = input_sentence.split()

    nbd0 = [ inp[0] ] if inp[0] in train_dictionary else sound_neighborhood( inp[0], codeset, code2words_dict, radius )
    nbd1 = sound_neighborhood( inp[1], codeset, code2words_dict, radius )
    tg={}

    for first_word in nbd0:
        for second_word in nbd1:
            bigram = first_word+' '+second_word #+' '+third_word
            tg[ bigram ]=lmodel.score( bigram, bos = True, eos = False)

    pred=max(tg, key=tg.get)
    output = pred.split()

    for i in range(2,len(inp)):
        phrases={}
        nbd = [ inp[i] ] if inp[i] in train_dictionary else sound_neighborhood( inp[i], codeset, code2words_dict, radius )

        for word in nbd:
            candidate=output[-1]+' '+word
            phrases[ word ]=lmodel.score( candidate, bos = False, eos = False)
        next_word=max(phrases, key=phrases.get)
        output.append( next_word )
        pred=pred+' '+next_word

    return pred

def sound_trigram_predict(input_sentence, train_dictionary, predict_dictionary, lmodel, radius=1.5): #input is a string
    #assumes that the output of the DNN is of the right length
    codedict, codeset = create_DMetaphone_list(predict_dictionary)
    code2words_dict = code2words( codedict, codeset )
    inp = input_sentence.split()

    #construct the first trigram
    #Note that the shortest sentence in this dataset has three words

    nbd0 = [ inp[0] ] if inp[0] in train_dictionary else sound_neighborhood( inp[0], codeset, code2words_dict, radius )
    nbd1 = sound_neighborhood( inp[1], codeset, code2words_dict, radius )
    nbd2 = sound_neighborhood( inp[2], codeset, code2words_dict, radius )
    tg={}

    for first_word in nbd0:
        for second_word in nbd1:
            for third_word in nbd2:
                trigram = first_word+' '+second_word+' '+third_word
                tg[ trigram ]=lmodel.score(trigram, bos = True, eos = False)

    pred=max(tg, key=tg.get)
    output = pred.split()

    for i in range(3,len(inp)):
        phrases={}
        nbd = [ inp[i] ] if inp[i] in train_dictionary else sound_neighborhood( inp[i], codeset, code2words_dict, radius )
#        nbd = get_neighborhood( inp[i], dictionary, 2)

        for word in nbd:
            candidate=output[-2]+' '+output[-1]+' '+word
            phrases[ word ]=lmodel.score( candidate, bos = False, eos = False)
        next_word=max(phrases, key=phrases.get)
        output.append( next_word )
        pred=pred+' '+next_word

    return pred

def levenshtein(seq1, seq2):
    # Thanks for this function to Frank Hoffman at
    # https://stackabuse.com/levenshtein-distance-and-text-similarity-in-python/
    size_x = len(seq1) + 1
    size_y = len(seq2) + 1
    matrix = np.zeros ((size_x, size_y))
    for x in range(size_x):
        matrix [x, 0] = x
    for y in range(size_y):
        matrix [0, y] = y

    for x in range(1, size_x):
        for y in range(1, size_y):
            if seq1[x-1] == seq2[y-1]:
                matrix [x,y] = min(
                    matrix[x-1, y] + 1,
                    matrix[x-1, y-1],
                    matrix[x, y-1] + 1
                )
            else:
                matrix [x,y] = min(
                    matrix[x-1,y] + 1,
                    matrix[x-1,y-1] + 1,
                    matrix[x,y-1] + 1
                )
#    print (matrix)
    return (matrix[size_x - 1, size_y - 1])

def generate_corpus(desc_file):
    #outputs a list of sentences
    data_sentences = AudioGenerator()
    data_sentences.load_train_data(desc_file=desc_file)
    sentences = data_sentences.train_texts
    return sentences

def wordset_from_corpus(sent_list):
    word_list = []
    for sent in sent_list:
        word_list.append(sent.split())
    long_word_list = [word for word in chain.from_iterable(word_list)]
#    words = re.findall('\w+', long_string)
    return set(long_word_list)

#st = generate_corpus("./train_corpus.json")
#train_words = wordset_from_corpus(st)
#valid_words = wordset_from_corpus(generate_corpus("./valid_corpus.json") )
#unseen_words =[word for word in valid_words if word not in train_words]
#hmmm...733 such unseen words (not many are proper nouns).
# Need a larger wordset than just the train_words (try nltk)

#st_small= generate_corpus("./small_train_corpus.json")
#with open('small_corpus_lines.txt', 'w') as filehandle:
#    filehandle.writelines("%s\n" % sentence for sentence in st_small)

def get_neighborhood(string, wordset, distance):
    """Finds all words from a set of words that are within a specified Levenshtein
    Distance from a given string"""
    nbd = [word for word in wordset if levenshtein(string, word) <= distance ]
    return set( nbd )

def dolgopolsky_neighborhood(string, wordset, distance):
    """Finds all words from a set of words that are within a specified Dolgopolsky
    Distance from a given string"""
    dst = panphon.distance.Distance()
    nbd = [word for word in wordset if dst.dogol_prime_distance(string, word) <= distance ]
    return set( nbd )

#nbd = get_neighborhood('helium', train_words, 2) #tested, 5 words found
#nbd = get_neighborhood('helium', english, 2) #tested, 43 words found
#sample_true = 'far up the lake eighteen miles above the town the eye of this cheerful camp follower of booms had spied out a graft'
#sample_input_sent = 'far ut the lake eightteen mils abo the town to ey of dis cherple can flolowor o bons had xpide ut a graft'
#sample_blob='far ut the lake eighteen miss ago the town to by of dis chere can follower o bons had side ut a graft'
#inp = input_sent.split()

#kenmodel = kenlm.Model('corpus_360_lines.arpa')
#ken5model = kenlm.Model('5_gram_corpus_360.binary')

def lm_predict(input_sentence, dictionary, lmodel): #input is a string
    #assumes that the output of the DNN is of the right length
    """this function keeps adding words that maximize the probability of sentence
    all the way from the beginning until the new word"""
    inp = input_sentence.split()

    #construct the first trigram
    #Note that the shortest sentence in this dataset has three words

    nbd0 = get_neighborhood( inp[0], dictionary, 2)
    nbd1 = get_neighborhood( inp[1], dictionary, 2)
    nbd2 = get_neighborhood( inp[2], dictionary, 2)
    tg={}

    for first_word in nbd0:
        for second_word in nbd1:
            for third_word in nbd2:
                trigram = first_word+' '+second_word+' '+third_word
                tg[ trigram ]=lmodel.score(trigram, bos = True, eos = False)

    pred=max(tg, key=tg.get)

    for i in range(3,len(inp)):
        phrases={}
        nbd = [ inp[i] ] if inp[i] in dictionary else get_neighborhood( inp[i], dictionary, 2)
        nbd = get_neighborhood( inp[i], dictionary, 2)

        for word in nbd:
            candidate=pred+' '+word
            phrases[ candidate ]=lmodel.score( candidate, bos = True, eos = False)
        pred=max(phrases, key=phrases.get)
    return pred
# 'far to the lake eighteen pins so the town to ku of dis cheaply can o fans had pile ku a graft'

def trigram_predict(input_sentence, train_dictionary, predict_dictionary, lmodel, radius=1.5): #input is a string
    #assumes that the output of the DNN is of the right length
    inp = input_sentence.split()

    #construct the first trigram
    #Note that the shortest sentence in this dataset has three words

    nbd0 = [ inp[0] ] if inp[0] in train_dictionary else get_neighborhood( inp[0], predict_dictionary, radius)
    nbd1 = get_neighborhood( inp[1], predict_dictionary, radius)
    nbd2 = get_neighborhood( inp[2], predict_dictionary, radius)
    tg={}

    for first_word in nbd0:
        for second_word in nbd1:
            for third_word in nbd2:
                trigram = first_word+' '+second_word+' '+third_word
                tg[ trigram ]=lmodel.score(trigram, bos = True, eos = False)

    pred=max(tg, key=tg.get)
    output = pred.split()

    for i in range(3,len(inp)):
        phrases={}
        nbd = [ inp[i] ] if inp[i] in train_dictionary else get_neighborhood( inp[i], predict_dictionary, radius)
#        nbd = get_neighborhood( inp[i], dictionary, 2)

        for word in nbd:
            candidate=output[-2]+' '+output[-1]+' '+word
            phrases[ word ]=lmodel.score( candidate, bos = False, eos = False)
        next_word=max(phrases, key=phrases.get)
        output.append( next_word )
        pred=pred+' '+next_word

    return pred

# 'far to the lake eighteen miles above the town to be of dis cheaply can can o one had side of a graft'
def cumul_sweep(input_sentence, intermediate, dictionary):
    inp = input_sentence.split()
    inter=intermediate.split()

    for i in range(3,len(inp)):
        phrases={}
        nbd = get_neighborhood( inter[i], dictionary, 2)
        u_nbd=nbd.union( get_neighborhood( inp[i], dictionary, 2) )
        for word in u_nbd:
            candidate=pred+' '+word
            phrases[ word ]=lmodel.score( candidate, bos = False, eos = False)
        next_word=max(phrases, key=phrases.get)
        pred=pred+' '+next_word
    return pred

def trigram_dolgopolsky_predict(input_sentence, train_dictionary, predict_dictionary, lmodel, radius=1.5): #input is a string
    #assumes that the output of the DNN is of the right length
    inp = input_sentence.split()

    #construct the first trigram
    #Note that the shortest sentence in this dataset has three words
    #for the second word, use bigram prob from kenlm

    nbd0 = [ inp[0] ] if inp[0] in train_dictionary else dolgopolsky_neighborhood( inp[0], predict_dictionary, radius)
    nbd1 = dolgopolsky_neighborhood( inp[1], predict_dictionary, radius)
    nbd2 = dolgopolsky_neighborhood( inp[2], predict_dictionary, radius)
    tg={}

    for first_word in nbd0:
        for second_word in nbd1:
            for third_word in nbd2:
                trigram = first_word+' '+second_word+' '+third_word
                tg[ trigram ]=lmodel.score(trigram, bos = True, eos = False)

    pred=max(tg, key=tg.get)
    output = pred.split()

    for i in range(3,len(inp)):
        phrases={}
        nbd = [ inp[i] ] if inp[i] in train_dictionary else dolgopolsky_neighborhood( inp[i], predict_dictionary, radius)
        for word in nbd:
            candidate=output[-2]+' '+output[-1]+' '+word
            phrases[ word ]=lmodel.score( candidate, bos = False, eos = False)
        next_word=max(phrases, key=phrases.get)
        output.append( next_word )
        pred=pred+' '+next_word
    return pred

#test_dolgo=trigram_dolgoposlky_predict(sample_input_sent, train_dictionary=train_words, predict_dictionary=english, radius=1.5)
#print( test_dolgo )

def bigram_predict(input_sentence, train_dictionary, predict_dictionary, lmodel, radius=1.5): #input is a string
    #assumes that the output of the DNN is of the right length
    inp = input_sentence.split()

    nbd0 = [ inp[0] ] if inp[0] in train_dictionary else get_neighborhood( inp[0], predict_dictionary, radius)
    nbd1 = get_neighborhood( inp[1], predict_dictionary, radius)
#    nbd2 = get_neighborhood( inp[2], predict_dictionary, radius)
    tg={}

    for first_word in nbd0:
        for second_word in nbd1:
#            for third_word in nbd2:
            bigram = first_word+' '+second_word #+' '+third_word
            tg[ bigram ]=lmodel.score( bigram, bos = True, eos = False)

    pred=max(tg, key=tg.get)
    output = pred.split()

    for i in range(2,len(inp)):
        phrases={}
        nbd = [ inp[i] ] if inp[i] in train_dictionary else get_neighborhood( inp[i], predict_dictionary, radius)

        for word in nbd:
            candidate=output[-1]+' '+word
            phrases[ word ]=lmodel.score( candidate, bos = False, eos = False)
        next_word=max(phrases, key=phrases.get)
        output.append( next_word )
        pred=pred+' '+next_word

    return pred

def bigram_dolgopolsky_predict(input_sentence, train_dictionary, predict_dictionary, lmodel, radius=1.5): #input is a string
    #assumes that the output of the DNN is of the right length
    inp = input_sentence.split()

    #construct the first trigram
    #Note that the shortest sentence in this dataset has three words
    #for the second word, use bigram prob from kenlm

    nbd0 = [ inp[0] ] if inp[0] in train_dictionary else dolgopolsky_neighborhood( inp[0], predict_dictionary, radius)
    nbd1 = dolgopolsky_neighborhood( inp[1], predict_dictionary, radius)
#    nbd2 = dolgopolsky_neighborhood( inp[2], predict_dictionary, radius)
    tg={}

    for first_word in nbd0:
        for second_word in nbd1:
#            for third_word in nbd2:
            bigram = first_word+' '+second_word
            tg[ bigram ]=lmodel.score(bigram, bos = True, eos = False)

    pred=max(tg, key=tg.get)
    output = pred.split()

    for i in range(3,len(inp)):
        phrases={}
        nbd = [ inp[i] ] if inp[i] in train_dictionary else dolgopolsky_neighborhood( inp[i], predict_dictionary, radius)

        for word in nbd:
            candidate=output[-1]+' '+word
            phrases[ word ]=lmodel.score( candidate, bos = False, eos = False)
        next_word=max(phrases, key=phrases.get)
        output.append( next_word )
        pred=pred+' '+next_word
    return pred

#for key, value in sorted(bg_scores.iteritems(), key=lambda (k,v): (v,k)):
#    print "%s: %s" % (key, value)
#newD = dict(sorted(bg.items(), key=operator.itemgetter(1), reverse=True)[:5])

#x = sorted(tg, key=tg.get, reverse=True)[:5]

## use the lines below to generate the txt on which to train kenlm
## the arpa file will be generated from this
#with open('corpus_360_lines.txt', 'w') as filehandle:
#    filehandle.writelines("%s\n" % sentence for sentence in st)

#Test kenlm using the python module contributed to kenlm by Victor Chahuneau.
# pip install https://github.com/kpu/kenlm/archive/master.zip
# see more here https://github.com/kpu/kenlm
#import kenlm
#model = kenlm.Model('corpus_360_lines.arpa')
# print(model.score('play the matter', bos = True, eos = True))

max_stars_repo_name: gotamist/nlp

max_issues_repo_name: gotamist/nlp

max_forks_repo_name: gotamist/nlp


text:
#!/usr/bin/env python
import cv2
import numpy as np
import sys

'''
based on http://docs.opencv.org/doc/tutorials/calib3d/camera_calibration/camera_calibration.html#cameracalibrationopencv
'''

def printnow(string):
   print string
   sys.stdout.flush()

class CameraCalibrator():
   def __init__(self):
      self.points_per_row = 9
      self.points_per_column = 7
      self.board_square_size_mm = 25.4 # The size of a square in millimeters

   def findCorners(self, cv_img):
      retval, corners = cv2.findChessboardCorners(cv_img, (self.points_per_row, self.points_per_column))
      if retval:
         # improve accuracy (might need to be tuned based on image size...
         gray = cv2.cvtColor(cv_img.copy(), cv2.COLOR_BGR2GRAY)
         cv2.cornerSubPix(gray, corners, (2,2), (-1,-1), (cv2.cv.CV_TERMCRIT_ITER | cv2.cv.CV_TERMCRIT_EPS, 20, 0.001))
      return corners

   def drawCorners(self, cv_img, corners):
      copy = cv_img.copy()
      cv2.drawChessboardCorners(copy, (self.points_per_row, self.points_per_column), corners, True)
      return copy

   def calibrate(self, image_points, image_size=(320,240)):
      objp = np.zeros((self.points_per_column*self.points_per_row,3), np.float32)
      objp[:,:2] = np.mgrid[0:self.points_per_row, 0:self.points_per_column].T.reshape(-1,2)
      objp *= self.board_square_size_mm
      object_points = []
      for i in range(len(image_points)):
         object_points.append(objp) # load up the object points with a clean grid

      try:
         retval, cameraMatrix, distCoeffs, rvecs, tvecs = cv2.calibrateCamera(object_points, image_points, image_size)

         # calculate how good the fit is..
         total_error_squared = 0
         total_num_points = 0
         for i in range(len(object_points)):
            projected_image_points, jacobian = cv2.projectPoints(object_points[i], rvecs[i], tvecs[i], cameraMatrix, distCoeffs)

            error_squared = 0
            num_points = 0
            for j in range(len(image_points[i])):
               dx = (image_points[i][j][0][0] - projected_image_points[j][0][0])
               dy = (image_points[i][j][0][1] - projected_image_points[j][0][1])
               error_squared += dx*dx + dy*dy
               num_points += 1
            printnow("error[%d]=%f" % ( i, np.sqrt(error_squared/num_points) ) )
            total_error_squared += error_squared
            total_num_points += num_points

         printnow("average error=%f" % np.sqrt(total_error_squared/total_num_points) )

         printnow((len(image_points), cameraMatrix[0,0], cameraMatrix[1,1], cameraMatrix[0,2], cameraMatrix[1,2], distCoeffs[0][0], distCoeffs[0][1], distCoeffs[0][2], distCoeffs[0][3], distCoeffs[0][4]))

         return cameraMatrix, distCoeffs

      except Exception, e:
         pass

if __name__ == '__main__':
   calibrator = CameraCalibrator()
   print "no unit test"