text:
#!/usr/bin/env python3
# Copyright 2019 Christian Henning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Train a resnet on images and time series data to predict below surface
temperatures.

Note, this implementation does not represent a Bayesian Neural Network yet (not
enough time).

The timeseries data is fed into the network via a fully-connected network that
produces the batch norm weights.
"""

from argparse import Namespace
import numpy as np
import random
from time import time
import torch
from torch.utils.data import DataLoader

from regression_dataset import PermaRegressionDataset
from resnet import ResNet
from simple_bn_generator import BNGenerator

if __name__ == "__main__":
    script_start = time()

    # FIXME
    ### User config
    args = Namespace()
    args.batch_size = 32
    args.lr = 0.01
    args.random_seed = 42
    args.local = False
    args.num_workers = 4

    ### Deterministic computation
    # Note, doesn't hold when using GPU or multiple workers that load the
    # dataset.
    torch.manual_seed(args.random_seed)
    torch.cuda.manual_seed_all(args.random_seed)
    np.random.seed(args.random_seed)
    random.seed(args.random_seed)

    ### Select device.
    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")
    print("Using cuda: " + str(use_cuda))

    ### Generate networks.
    # FIXME Downsample images to speed up computation.
    img_net = ResNet(
        in_shape=[536, 356, 3], num_outs=1, no_weights=False, use_batch_norm=True
    )

    # We include the following sensory information:
    # - surface temperature
    # - radiation
    # - month
    # - daytime
    n_in = 4

    # Shapes of batchnorm layer weights.
    # We will produce these weights with an auxiliary fully-connected network,
    # that takes the timeseries data as input.
    bn_shapes = []
    for l in img_net._batchnorm_layers:
        bn_shapes.extend(l.param_shapes)

    # FIXME Our current implementation doesn't allow efficient batch processing.
    # Neither the underlying hnet allows the usage of multiple task embeddings
    # nor does the batch norm layer support a batch of weights.
    ts_net = BNGenerator(bn_shapes, 1, layers=[100, 100], te_dim=n_in)

    ### Generate datasets.
    train_data = PermaRegressionDataset(
        args.local, time_slice={"start_time": "2017-01-01", "end_time": "2017-06-30"}
    )
    test_data = PermaRegressionDataset(
        args.local, time_slice={"start_time": "2017-07-01", "end_time": "2017-07-31"}
    )

    train_loader = DataLoader(
        train_data,
        batch_size=args.batch_size,
        shuffle=True,
        num_workers=args.num_workers,
    )
    test_loader = DataLoader(
        test_data,
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=args.num_workers,
    )

    ### Train and test.
    raise NotImplementedError("Training and testing not implemented yet.")

    print("Program finished successfully in %f sec." % (time() - script_start))

max_stars_repo_name: calemen/permafrostanalytics

max_issues_repo_name: elonag/permafrostanalytics

max_forks_repo_name: elonag/permafrostanalytics


text:
# Copyright (c) [2021] [wlicsnju]
# [HKMF-T] is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#         http://license.coscl.org.cn/MulanPSL2
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
import logging

import numpy as np

from callback import AbstractConvergeCallback
from collections import namedtuple
from typing import Iterable

from hankel_methods import get_hankel_result, find_max_continue_0, hankelization

HKMFTTrainParam = namedtuple('HKMFTTrainParam',
                             ['eta', 'lambda_s', 'lambda_o', 'lambda_e', 'stop_rate', 'random'])
HKMFT_MAX_EPOCH = 5000

class HKMFT(object):
    def __init__(self):
        self._p = 0
        self._Hpx = None
        self._Hpx_mask = None
        self._Hpx_tag = None
        self._U = None
        self._V = None
        self._E = None
        self._A_hat = None
        self._data = None

    def put_and_reset(self, data, mask, tag, r=None, p=None):
        if data.shape != tag.shape:
            logging.error(f'\'data\'{data.shape} must be same to \'tag\'{tag.shape}.', ValueError)
            return
        if len(mask.shape) != 1:
            logging.error(f'\'mask\'{mask.shape} must be 1-d array.', ValueError)
            return
        l_b = find_max_continue_0(mask)
        if p is None:
            p = l_b + 1
        if p < l_b + 1:
            logging.warning(f'\'p\' is less than length of blackouts {l_b}+1, set \'p\' to {l_b + 1}.')
            p = l_b + 1
        if r is None:
            r = p
        if r > p:
            logging.warning(f'\'r\' is great than \'p\' {p}, set \'r\' to {p}.')
            r = p
        if len(data.shape) == 1:
            self._data = data[np.newaxis, :]
            tag = tag[np.newaxis, :]
        elif len(data.shape) == 2:
            self._data = data
        elif len(data.shape) > 2:
            logging.error('Do not support data with dim > 2.', ValueError)
            return
        self._p = p
        data_d = self._data.shape[0]
        data_l = self._data.shape[1]
        self._U = np.random.rand(data_d * p, r)
        self._V = np.random.rand(r, data_l)
        self._E = np.random.rand(data_d * p, data_l)
        self._A_hat = (self._U @ self._V) + self._E
        self._Hpx, self._Hpx_mask, self._Hpx_tag = hankelization(data, mask, tag, self._p)

    def train(self, param: HKMFTTrainParam, callbacks: Iterable[AbstractConvergeCallback]):
        epoch = 0
        data_l = self._Hpx.shape[1]
        it_set = set()
        all_it_list = []
        for i, _ in enumerate(self._Hpx_mask):
            for t, x in enumerate(_):
                if x != 0:
                    it_set.add((i, t))
                    all_it_list.append((i, t))
        max_eliminate = int(0.005 * len(it_set))
        stop_it = set()
        lr = param.eta * 2.0
        while epoch < HKMFT_MAX_EPOCH:
            epoch += 1
            e = np.zeros(self._Hpx.shape, dtype=np.float)
            if param.random:
                it_tmp = list(it_set)
                np.random.shuffle(it_tmp)
            else:
                it_tmp = all_it_list
            for i, t in it_tmp:
                if (i, t) not in it_set:
                    continue
                err_it = self._Hpx[i, t] - ((self._U[i, :] @ self._V[:, t]) + self._E[i, t])
                e[i, t] = err_it
                new_U_i_V_t = self._U[i, :] @ self._V[:, t]
                new_U_i_V_t_s1 = self._U[i, :] @ self._V[:, t - 1]
                # if t > 0:
                #     new_U_i_V_t_s1 = self._U[i, :] @ self._V[:, t - 1]
                # else:
                #     new_U_i_V_t_s1 = 0.0
                new_U_i_V_t_p1 = self._U[i, :] @ self._V[:, (t + 1) % data_l]
                # if t < data_l - 1:
                #     new_U_i_V_t_p1 = self._U[i, :] @ self._V[:, t + 1]
                # else:
                #     new_U_i_V_t_p1 = 0.0
                # U_i
                U_i = self._U[i, :] + (lr * ((err_it * self._V[:, t]) -
                                             (param.lambda_o * self._U[i, :])))
                # if t > 0:
                U_i -= (lr * param.lambda_s *
                        (new_U_i_V_t - new_U_i_V_t_s1) *
                        (self._V[:, t] - self._V[:, t - 1]))
                # if t < data_l - 1:
                U_i -= (lr * param.lambda_s *
                        (new_U_i_V_t_p1 - new_U_i_V_t) *
                        (self._V[:, (t + 1) % data_l] - self._V[:, t]))
                # V_t
                V_t = self._V[:, t] + (lr * ((err_it * self._U[i, :]) -
                                             (param.lambda_o * self._V[:, t]) -
                                             (2.0 * param.lambda_s * new_U_i_V_t * self._U[i, :])))
                # if t > 0:
                V_t += (lr * param.lambda_s *
                        new_U_i_V_t_s1 * self._U[i, :])
                # if t < data_l - 1:
                V_t += (lr * param.lambda_s *
                        new_U_i_V_t_p1 * self._U[i, :])
                # E_ij
                E_bar = np.average(np.compress((self._Hpx_mask.ravel() != 0) &
                                               (self._Hpx_tag.ravel() == self._Hpx_tag[i, t]),
                                               self._E))
                # E_bar = np.average(self._E[(self._Hpx_mask != 0) & (self._Hpx_tag == self._Hpx_tag[i, t])])
                E_it = self._E[i, t] + (lr * (err_it -
                                              (param.lambda_o * self._E[i, t]) -
                                              (param.lambda_e * (self._E[i, t] - E_bar))))
                # for c in callbacks:
                #     if isinstance(c, AbstractConvergeCallback):
                #         if not c.on_one_update(i, t, self._U, self._V, self._E, U_i, V_t, E_it, E_bar):
                #             self._U[i, :] = U_i
                #             self._V[:, t] = V_t
                #             self._E[i, t] = E_it
                #             logging.info(f'Early stop in {c.__class__.__name__}.on_one_update().')
                #             return
                self._U[i, :] = U_i
                self._V[:, t] = V_t
                self._E[i, t] = E_it
            new_A_hat = (self._U @ self._V) + self._E
            for c in callbacks:
                if isinstance(c, AbstractConvergeCallback):
                    if not c.on_epoch(self._p, it_set, e, self._A_hat, new_A_hat, self._Hpx_mask, self._Hpx_tag):
                        self._A_hat = new_A_hat
                        logging.info(f'Early stop in {c.__class__.__name__}.on_epoch().')
                        return
            self._A_hat = new_A_hat
            if len(it_set) / (len(stop_it) + len(it_set)) <= param.stop_rate:
                continue
            # eliminate upper and lower 0.5% case
            error_sort_idx = np.unravel_index(e.argsort(axis=None), e.shape)
            for x_idx, y_idx in ((error_sort_idx[0], error_sort_idx[1]),
                                 (error_sort_idx[0][::-1], error_sort_idx[1][::-1])):
                eliminate_count = 0
                for i, t in zip(x_idx, y_idx):
                    if ((len(it_set) / (len(stop_it) + len(it_set)) <= param.stop_rate) or
                            (eliminate_count >= max_eliminate)):
                        break
                    if (i, t) in it_set:
                        it_set.remove((i, t))
                        stop_it.add((i, t))
                        eliminate_count += 1

    def get_result(self):
        tag_mean = {}
        for i, _ in enumerate(self._Hpx_mask):
            for t, x in enumerate(_):
                tag = self._Hpx_tag[i, t]
                if x == 0 and tag >= 0:
                    if tag not in tag_mean:
                        tag_mean[tag] = np.average(np.compress((self._Hpx_mask.ravel() != 0) &
                                                               (self._Hpx_tag.ravel() == tag),
                                                               self._E))
                        # tag_mean[tag] = np.average(self._E[(self._Hpx_mask != 0) & (self._Hpx_tag == tag)])
                    self._E[i, t] = tag_mean[tag]
        self._A_hat = (self._U @ self._V) + self._E
        return get_hankel_result(self._A_hat, self._Hpx_mask, self._p)

ext: py

max_stars_repo_name: wangliang-cs/hkmf-t

max_issues_repo_name: wangliang-cs/hkmf-t

max_forks_repo_name: wangliang-cs/hkmf-t

reason: import numpy


text:
import numpy as np
import unittest
import pytest
from desc.io import InputReader
from desc.equilibrium import Equilibrium
from desc.profiles import PowerSeriesProfile

class TestProfiles(unittest.TestCase):
    def test_same_result(self):
        input_path = "examples/DESC/SOLOVEV"
        ir = InputReader(input_path)

        eq1 = Equilibrium(**ir.inputs[-1])
        eq2 = eq1.copy()
        eq2.pressure = eq1.pressure.to_spline()
        eq2.iota = eq1.iota.to_spline()

        eq1.solve()
        eq2.solve()

        np.testing.assert_allclose(
            eq1.x,
            eq2.x,
            rtol=1e-05,
            atol=1e-08,
        )

    def test_close_values(self):

        pp = PowerSeriesProfile(modes=np.array([0, 2, 4]), params=np.array([1, -2, 1]))
        sp = pp.to_spline()
        with pytest.warns(UserWarning):
            mp = pp.to_mtanh(order=4, ftol=1e-12, xtol=1e-12)
        x = np.linspace(0, 0.8, 10)

        np.testing.assert_allclose(pp(x), sp(x), rtol=1e-5, atol=1e-3)
        np.testing.assert_allclose(pp(x, dr=2), mp(x, dr=2), rtol=1e-2, atol=1e-1)

        pp1 = sp.to_powerseries(order=4)
        np.testing.assert_allclose(pp.params, pp1.params, rtol=1e-5, atol=1e-2)
        pp2 = mp.to_powerseries(order=4)
        np.testing.assert_allclose(pp.params, pp2.params, rtol=1e-5, atol=1e-2)

        with pytest.warns(UserWarning):
            mp1 = sp.to_mtanh()
        np.testing.assert_allclose(mp1(x, dr=1), sp(x, dr=1), rtol=1e-5, atol=1e-3)

        pp3 = pp.to_powerseries()
        sp3 = sp.to_spline()
        np.testing.assert_allclose(sp3(x), pp3(x), rtol=1e-5, atol=1e-3)
        sp4 = mp.to_spline()
        np.testing.assert_allclose(sp3(x), sp4(x), rtol=1e-5, atol=1e-2)

    def test_repr(self):

        pp = PowerSeriesProfile(modes=np.array([0, 2, 4]), params=np.array([1, -2, 1]))
        sp = pp.to_spline()
        mp = pp.to_mtanh(order=4, ftol=1e-4, xtol=1e-4)

        assert "PowerSeriesProfile" in str(pp)
        assert "SplineProfile" in str(sp)
        assert "MTanhProfile" in str(mp)

    def test_get_set(self):

        pp = PowerSeriesProfile(modes=np.array([0, 2, 4]), params=np.array([1, -2, 1]))

        assert pp.get_params(2) == -2
        assert pp.get_idx(4) == 4
        pp.set_params(3, 22)

        assert pp.get_params(3) == 22
        pp.change_resolution(L=2)
        assert pp.params.size == 3

        sp = pp.to_spline()
        sp.params = sp.values + 1
        sp.values = sp.params

        np.testing.assert_allclose(sp.values, 1 + pp(sp._knots))

max_stars_repo_name: ddudt/DESC

max_issues_repo_name: PlasmaControl/DESC

max_forks_repo_name: ddudt/DESC


text:
import asyncio
import concurrent.futures
import os
import pickle
import sys
from time import time
from typing import List

import imageio
import moviepy.editor as mpy
from output_event import OutputEvent
import numpy as np
import torch
import yaml
from scipy.spatial import ConvexHull
from skimage import img_as_float, img_as_ubyte
from skimage.transform import resize
from tqdm import tqdm

from modules.generator import OcclusionAwareGenerator
from modules.keypoint_detector import KPDetector
from sync_batchnorm import DataParallelWithCallback
from task import Task
from tool.animate import normalize_kp
from tool.crop_video import crop_image, crop_video

def load_checkpoints(config_path, checkpoint_path, cpu=False):

    with open(config_path) as f:
        config = yaml.load(f)

    generator = OcclusionAwareGenerator(**config['model_params']['generator_params'],
                                        **config['model_params']['common_params'])
    if not cpu:
        generator.cuda()

    kp_detector = KPDetector(**config['model_params']['kp_detector_params'],
                             **config['model_params']['common_params'])
    if not cpu:
        kp_detector.cuda()

    if cpu:
        checkpoint = torch.load(
            checkpoint_path, map_location=torch.device('cpu'))
    else:
        checkpoint = torch.load(checkpoint_path)

    generator.load_state_dict(checkpoint['generator'])
    kp_detector.load_state_dict(checkpoint['kp_detector'])

    if not cpu:
        generator = DataParallelWithCallback(generator)
        kp_detector = DataParallelWithCallback(kp_detector)

    generator.eval()
    kp_detector.eval()

    return generator, kp_detector

def make_animation(source_image, driving_video, generator, kp_detector, relative=True, adapt_movement_scale=True, cpu=False, progress_bar=None):
    with torch.no_grad():
        predictions = []
        source = torch.tensor(source_image[np.newaxis].astype(
            np.float32)).permute(0, 3, 1, 2)
        if not cpu:
            source = source.cuda()
        driving = torch.tensor(np.array(driving_video)[np.newaxis].astype(
            np.float32)).permute(0, 4, 1, 2, 3)
        kp_source = kp_detector(source)
        kp_driving_initial = kp_detector(driving[:, :, 0])

        for frame_idx in range(driving.shape[2]):
            if progress_bar is not None:
                progress_bar.update(1)
            driving_frame = driving[:, :, frame_idx]
            if not cpu:
                driving_frame = driving_frame.cuda()
            kp_driving = kp_detector(driving_frame)
            kp_norm = normalize_kp(kp_source=kp_source, kp_driving=kp_driving,
                                   kp_driving_initial=kp_driving_initial, use_relative_movement=relative,
                                   use_relative_jacobian=relative, adapt_movement_scale=adapt_movement_scale)
            out = generator(source, kp_source=kp_source, kp_driving=kp_norm)

            predictions.append(np.transpose(
                out['prediction'].data.cpu().numpy(), [0, 2, 3, 1])[0])
    return predictions

def find_best_frame(source, driving, cpu=False, preprocessed_kp_driving=None):
    import face_alignment

    def normalize_kp(kp):
        kp = kp - kp.mean(axis=0, keepdims=True)
        area = ConvexHull(kp[:, :2]).volume
        area = np.sqrt(area)
        kp[:, :2] = kp[:, :2] / area
        return kp

    fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=True,
                                      device='cpu' if cpu else 'cuda')
    kp_source = fa.get_landmarks(255 * source)[0]
    kp_source = normalize_kp(kp_source)
    norm = float('inf')
    frame_num = 0
    for i, image in enumerate(driving):
        if preprocessed_kp_driving != None:
            kp_driving = preprocessed_kp_driving[i]
        else:
            kp_driving = fa.get_landmarks(255 * image)[0]
            kp_driving = normalize_kp(kp_driving)

        new_norm = (np.abs(kp_source - kp_driving) ** 2).sum()
        if new_norm < norm:
            norm = new_norm
            frame_num = i
    return frame_num

def process_frames(source_image, driving_frames, generator, kp_detector, from_best_frame=False, adapt_movement_scale=True, gpu=False, preprocessed_kp_driving=None, h_progress=True):
    if h_progress:
        frames_to_process = len(driving_frames) + \
            1 if from_best_frame else len(driving_frames)

        progress_bar = tqdm(total=frames_to_process)
    else:
        progress_bar = None

    if from_best_frame:
        i = find_best_frame(source_image, driving_frames, cpu=not gpu,
                            preprocessed_kp_driving=preprocessed_kp_driving)
        driving_forward = driving_frames[i:]
        driving_backward = driving_frames[:(i+1)][::-1]
        predictions_forward = make_animation(source_image, driving_forward, generator,
                                             kp_detector, relative=True, adapt_movement_scale=adapt_movement_scale, cpu=not gpu, progress_bar=progress_bar)
        predictions_backward = make_animation(source_image, driving_backward, generator,
                                              kp_detector, relative=True, adapt_movement_scale=adapt_movement_scale, cpu=not gpu, progress_bar=progress_bar)
        predictions = predictions_backward[::-1] + predictions_forward[1:]
    else:
        predictions = make_animation(source_image, driving_frames, generator, kp_detector,
                                     relative=True, adapt_movement_scale=adapt_movement_scale, cpu=not gpu, progress_bar=progress_bar)
    if h_progress:
        progress_bar.close()
    return predictions

def open_video(source_video, gpu=False, advanced_crop=False):
    with imageio.get_reader(source_video) as reader:
        driving_video = [frame for frame in reader]
        fps = reader.get_meta_data()['fps']

    if advanced_crop:
        driving_video = crop_video(driving_video, gpu=gpu, min_frames=1)
    else:
        driving_video = [resize(frame, (256, 256))[..., :3]
                         for frame in driving_video]

    return driving_video, fps

def open_image(source_image, gpu=False, advanced_crop=False, increase=0.2):
    source_image = imageio.imread(source_image)

    if advanced_crop:
        source_image = crop_image(source_image, gpu=gpu, increase=increase)
    else:
        source_image = resize(source_image, (256, 256))[..., :3]

    return source_image

def preprocess_kp_driving(driving: List, gpu=False, h_progress=True):
    import face_alignment

    def normalize_kp(kp):
        kp = kp - kp.mean(axis=0, keepdims=True)
        area = ConvexHull(kp[:, :2]).volume
        area = np.sqrt(area)
        kp[:, :2] = kp[:, :2] / area
        return kp

    fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=True,
                                      device='cuda' if gpu else 'cpu')

    preprocessed_kp_driving = []
    driving_enumerator = enumerate(
        tqdm(driving)) if h_progress else enumerate(driving)
    for i, image in driving_enumerator:
        kp_driving = fa.get_landmarks(255 * image)[0]
        kp_driving = normalize_kp(kp_driving)

        preprocessed_kp_driving.append(kp_driving)

    return preprocessed_kp_driving

async def process_task(task: Task, h_progress=True):
    start = time()
    yield OutputEvent(OutputEvent.Types.PROCESSING_STARTED, time()-start)

    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
        loop = asyncio.get_event_loop()

        video_name = os.path.splitext(task.driving_video)[0]

        should_preprocess_video = task.clean_build or not os.path.exists("temp/" + video_name) or not os.path.exists(
            "temp/" + video_name + "/driving_processed.mp4") or (task.find_best_frame and not os.path.exists("temp/" + video_name + "/kp_driving.npy"))

        preprocessed_kp_driving = None
        try:
            if should_preprocess_video:
                import shutil
                if os.path.exists("temp/" + video_name):
                    shutil.rmtree("temp/" + video_name)

                os.makedirs("temp/" + video_name)
                files_folder = "temp/" + video_name + "/"

                yield OutputEvent(OutputEvent.Types.OPENING_VIDEO, time()-start, task.driving_video)

                driving_video, fps = await loop.run_in_executor(
                    pool, open_video,
                    "videos/" + task.driving_video,
                    task.gpu,
                    task.crop)

                with imageio.get_writer(files_folder + "driving_processed.mp4", fps=fps) as writer:
                    [writer.append_data(img_as_ubyte(frame))
                     for frame in driving_video]

                if task.find_best_frame:
                    yield OutputEvent(OutputEvent.Types.PREPROCESSING_FIND_BEST_FRAME, time()-start)

                    preprocessed_kp_driving = await loop.run_in_executor(
                        pool, preprocess_kp_driving,
                        driving_video,
                        task.gpu,
                        h_progress)

                    with open(files_folder + "kp_driving.npy", "wb") as fp:
                        pickle.dump(preprocessed_kp_driving, fp)
            else:
                files_folder = "temp/" + video_name + "/"

                yield OutputEvent(OutputEvent.Types.OPENING_VIDEO_TEMP, time()-start)

                with imageio.get_reader(files_folder + "driving_processed.mp4") as reader:
                    driving_video = [img_as_float(frame) for frame in reader]

                if task.find_best_frame:
                    yield OutputEvent(OutputEvent.Types.PREPROCESSING_FIND_BEST_FRAME_TEMP, time()-start)

                    with open(files_folder + "kp_driving.npy", "rb") as fp:
                        preprocessed_kp_driving = pickle.load(fp)
        except:
            yield OutputEvent(OutputEvent.Types.ERROR_OPENING_VIDEO, time()-start, task.driving_video)
            return

        yield OutputEvent(OutputEvent.Types.VIDEO_OPENED, time()-start, task.driving_video)

        yield OutputEvent(OutputEvent.Types.OPENING_MODEL, time()-start, task.checkpoint)
        try:
            generator, kp_detector = await loop.run_in_executor(
                pool, load_checkpoints,
                task.config,
                task.checkpoint,
                not task.gpu)
        except:
            yield OutputEvent(OutputEvent.Types.ERROR_OPENING_MODEL, time()-start, task.checkpoint)
            return

        for i in range(len(task.source_images)):
            yield OutputEvent(OutputEvent.Types.PROCESSING_VIDEO_STARTED, time()-start, task.result_videos[i])

            try:
                source_image = await loop.run_in_executor(
                pool, open_image,
                "images/" + task.source_images[i],
                task.gpu,
                task.crop,
                task.image_padding)
            except:
                yield OutputEvent(OutputEvent.Types.ERROR_OPENING_IMAGE, time()-start, task.source_images[i])
                return

            predictions = await loop.run_in_executor(
                pool, process_frames,
                source_image,
                driving_video,
                generator,
                kp_detector,
                task.find_best_frame,
                False,
                task.gpu,
                preprocessed_kp_driving,
                h_progress)

            yield OutputEvent(OutputEvent.Types.SAVING_OUTPUT_VIDEO, time()-start, task.result_videos[i])
            with mpy.VideoFileClip("videos/" + task.driving_video) as clip:
                frames = iter(predictions)
                with mpy.VideoClip(lambda t: img_as_ubyte(
                        next(frames)), duration=len(predictions)/clip.fps) as output:

                    if task.audio:
                        output.audio = clip.audio

                    output.write_videofile(
                        "output/" + task.result_videos[i], fps=clip.fps, verbose=False, logger=None)
            yield OutputEvent(OutputEvent.Types.VIDEO_SAVED, time()-start, task.result_videos[i])

max_stars_repo_name: KMielnik/AlterEgo-core

max_issues_repo_name: KMielnik/AlterEgo-core

max_forks_repo_name: KMielnik/AlterEgo-core

reason: import numpy,from scipy


text:
import zmq
import argparse
import pdb
import numpy as np

import synth_bbone
from synth_commands import *

SYNTH_PINS = {'rf':synth_bbone.RF_SYNTH_PINS, 'demod':synth_bbone.DEMOD_SYNTH_PINS}

class zmq_synth_server:
    def __init__(self, context, synth, port):
        self.synth = synth
        self.context = context
        self.socket = context.socket(zmq.REP)
        self.socket.bind("tcp://*:{}".format(str(port)))

    def _set_freq(self, message):
        freq = float(message[1:])
        print('setting frequency to {} Hz'.format(freq))
        self.synth.set_freq(freq)
        return message[COMMAND_INDEX]

    def _set_filter_bank(self, message):
        freq = float(message[1:])
        print('setting filter bank for {} GHz'.format(freq/1e9))
        self.synth.set_filter_bank(freq)
        return message[COMMAND_INDEX]

    def _set_power_dac(self, message):
        pow_dac = int(message[1:])
        print('dac to set to '.format(pow_dac))
        self.synth.set_power_dac(pow_dac)
        return message[COMMAND_INDEX]

    def _set_power_lmx(self, message):
        pow_lmx = int(message[1:])
        print('lmx power level to {}'.format(pow_lmx))
        self.synth.set_power_lmx(pow_lmx)
        return message[COMMAND_INDEX]

    def _wait_for_lock(self, message):
        print('waiting for pll lock')
        self.synth.wait_for_lock()
        print('locked!')
        return message[COMMAND_INDEX]

    def run(self):
        self.command_handlers = {\
            FREQ_CMD : self._set_freq, \
            LMX_POW_CMD : self._set_power_lmx, \
            FILT_CMD : self._set_filter_bank, \
            DAC_POW_CMD : self._set_power_dac, \
            WAIT_CMD : self._wait_for_lock}

        print('entering run loop')
        while True:
            print('waiting for command..')
            message = self.socket.recv().decode('utf-8')
            command = message[COMMAND_INDEX]
            print('received {} command, message: {}'.format(command, message))

            response = ''

            if command in self.command_handlers:
                response = self.command_handlers[command](message)

            else:
                print('unrecognized command')
                pdb.set_trace()

            self.socket.send(response.encode('utf-8'))

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--synth', help = 'synth (rf/demod)', default = 'rf')
    args = parser.parse_args()

    context = zmq.Context()
    synth_name = args.synth

    pins = SYNTH_PINS[synth_name]
    port = SYNTH_PORTS[synth_name]

    synth = synth_bbone.synth_r1(pins)
    print('initializing synth')
    synth_server = zmq_synth_server(context, synth, port)
    print('running zmq server')
    synth_server.run()
    raw_input('press enter to continue')
    synth_server.close()

    context.destroy()

max_stars_repo_name: loxodes/Vector-Network-Analyzer

max_issues_repo_name: loxodes/Vector-Network-Analyzer

max_forks_repo_name: loxodes/Vector-Network-Analyzer


text:
from numpy.linalg import norm
from numpy import cos, arccos, sin, arctan2, vstack, clip, sum
from utilities.Constants import PI
import numpy as np
import warnings

def spherical_to_cartesian(spherical_vector):
    """Returns the cartesian form of the spherical vector."""
    spherical_vector = spherical_vector.reshape(-1, 3)
    output = vstack([(spherical_vector[:, 0]*sin(spherical_vector[:, 1])*cos(spherical_vector[:, 2])),
                    spherical_vector[:, 0]*sin(spherical_vector[:, 1])*sin(spherical_vector[:, 2]),
                    spherical_vector[:, 0]*cos(spherical_vector[:, 1])]).T
    if len(output) == 1:
        return output[0]
    else:
        return output

def cartesian_to_spherical(cartesian_vector):
    """Returns the cartesian form of the spherical vector."""
    cartesian_vector = cartesian_vector.reshape(-1, 3)
    r = norm(cartesian_vector, axis=1)
    output = vstack([r, arccos(cartesian_vector[:, 2]/r),
                     arctan2(cartesian_vector[:, 1], cartesian_vector[:, 0])]).T
    if len(output) == 1:
        return output[0]
    else:
        return output

def unit_vector(vector):
    """ Returns the unit vector of the vector.  """
    norms = norm(vector.reshape(-1, 3), axis=1)
    output = vector / norms.reshape(-1, 1)
    if len(output) == 1:
        return output[0]
    else:
        return output

def angle_between(s1, s2):
    """ Returns the angle in radians between vectors 'v1' and 'v2'::
    """
    v1_u, v2_u = unit_vector(s1), unit_vector(s2)
    output = arccos(clip(sum(v1_u.reshape((-1, 3))*v2_u.reshape((-1, 3)), axis=1), -1.0, 1.0))
    if len(output) == 1:
        return output[0]
    else:
        return output

def unit_radius(position):
    position = cartesian_to_spherical(position).reshape(-1, 3)
    sin_phi = sin(position[:, 2])
    unit_radii = vstack([cos(position[:, 1])*sin_phi,
                         sin(position[:, 1])*sin_phi,
                         cos(position[:, 2])]).T
    if len(unit_radii) == 1:
        return unit_radii[0]
    else:
        return unit_radii

def unit_theta(position):
    position = cartesian_to_spherical(position).reshape(-1, 3)
    cos_phi = cos(position[:, 2])
    unit_thetas_return = vstack([cos(position[:, 1])*cos_phi,
                                sin(position[:, 1])*cos_phi,
                                -sin(position[:, 2])]).T
    if len(unit_thetas_return) == 1:
        return unit_thetas_return[0]
    else:
        return unit_thetas_return

def latitude_to_spherical(vector):
    vector = vector.reshape(-1, 3)
    vector[:, 1] = 90 - vector[:, 1]
    vector[:, 1:] *= PI / 180.0

    if len(vector) == 1:
        return vector[0]
    else:
        return vector

def row_dot_product(a: np.ndarray, b: np.ndarray) -> np.ndarray:
    """
    Returns a vectorized dot product between the rows of a and b
    :param a: An array of shape (N, M) or (M, )
     (or a shape that can be broadcast to (N, M))
    :param b: An array of shape (N, M) or (M, )
    (or a shape that can be broadcast to (N, M))
    :return: A vector of shape (N, ) whose elements are the dot product of rows a, b
    """
    return np.einsum('ij,ij->i', np.atleast_2d(a), np.atleast_2d(b))

def normalize_last_axis(a: np.ndarray) -> np.ndarray:
    """
    Normalized the provided vector row-wise
    :param a: An array of shape (N, M)
    :return: A vector of shape (N, M), where each row is unit length and parallel to the row in the original vector
    """
    vec_norm = np.asarray(norm(a, axis=-1))
    zero_parts = vec_norm == 0

    if np.any(zero_parts):
        warnings.warn(RuntimeWarning(
            "Attempting to normalize a 0 vector. "
            "Returning 0 vector on rows instead of infinity"
        ))
        vec_norm[zero_parts] = 1

    return a / vec_norm[..., np.newaxis]

def angle_between_vector_collections(vec1: np.ndarray, vec2: np.ndarray) -> np.ndarray:
    """
    Internal utility. For speed, vec1 and vec2 need to be of shape (N, M)
    :param vec1: vector of shape (N, M)
    :param vec2: vector of shape (N, M)
    :return: angles between collections in array of shape (N, )
    """
    if vec1.ndim > 2 or vec2.ndim > 2:
        raise ValueError("Vectors need to have 1 or 2 dimensions")
    if vec1.shape[-1] != vec2.shape[-1] or vec1.shape[-1] < 2 or vec2.shape[-1] < 2:
        raise ValueError("Vectors have unequal shape in their last axes: {}, {}"
                         .format(vec1.shape[-1], vec2.shape[-1]))
    return np.arccos(np.clip(row_dot_product(vec1, vec2) / (
            norm(vec2, axis=-1) * norm(vec1, axis=-1)), -1.0, 1.0))

max_stars_repo_name: msdrigg/RayTracing

max_issues_repo_name: msdrigg/RayTracing

max_forks_repo_name: msdrigg/RayTracing

reason: import numpy,from numpy


text:
# -*- encoding: utf-8 -*-
# Module iatz

from ia870 import iasecross

# constants
MASK = -2
TIE_ZONE = 0

def iatz(f, Bc=iasecross()):
    from ipdp import se2offset

    offset = se2offset(Bc)
    return tieZone(f, offset)

def tieZone(im, offsets):

    import numpy as np
    from ipdp import wsImage
    from ipdp import findMinima
    from ipdp import wsHeapQueue

    # initialise variables
    ws = wsImage(im)
    N, im, lab, D = ws.begin(offsets)

    # find minima
    M = findMinima(im, N, D)

    # create the working images
    done = ws.makeWorkCopy(False)
    c1 = ws.makeWorkCopy(np.inf)
    c2 = ws.makeWorkCopy(0)
    par = ws.makeWorkCopy(MASK)
    lab[:] = MASK

    queue = wsHeapQueue()

    for m in xrange(len(M)):
        for p in M[m]:
            c1[p] = im[p]
            lab[p] = m+1
            par[p] = p
            queue.push(p, im[p])

    while not queue.empty():
        p = queue.pop()
        done[p] = True
        for q in N(p):
            if done[q]:
                continue

            c = max(c1[p], im[q])
            if c < c1[q]:
                if c1[q] < np.inf:
                    if queue.contains(q, c1[q]):
                        queue.remove(q, c1[q])
                c1[q] = c
                lab[q] = lab[p]
                par[q] = p
                queue.push(q, c1[q])
                if c == c1[p]:
                    c2[q] = c2[p] + 1
            elif c == c1[q] and lab[q] != lab[p]:
                if c == c1[p]:
                    if c2[q] == c2[p] + 1:
                        lab[q] = TIE_ZONE
                else:
                    lab[q] = TIE_ZONE

    return ws.end()


text:
# This file is part of the pyMOR project (http://www.pymor.org).
# Copyright 2013-2017 pyMOR developers and contributors. All rights reserved.
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

import numpy as np

from pymor.grids.interfaces import AffineGridInterface
from pymor.grids.referenceelements import triangle
from pymor.grids._unstructured import compute_edges

class UnstructuredTriangleGrid(AffineGridInterface):
    """A generic unstructured, triangular grid.

    Parameters
    ----------
    vertices
        A (num_vertices, 2)-shaped |array| containing the coordinates
        of all vertices in the grid. The row numbers in the array will
        be the global indices of the given vertices (codim 2 entities).
    faces
        A (num_faces, 3)-shaped |array| containing the global indices
        of the vertices which define a given triangle in the grid.
        The row numbers in the array will be the global indices of the
        given triangles (codim 0 entities).
    """

    dim = 2
    reference_element = triangle

    def __init__(self, vertices, faces):
        assert faces.shape[1] == 3
        assert np.min(faces) == 0
        assert np.max(faces) == len(vertices) - 1

        vertices = vertices.astype(np.float64, copy=False)
        faces = faces.astype(np.int32, copy=False)
        edges, num_edges = compute_edges(faces, len(vertices))

        COORDS = vertices[faces]
        SHIFTS = COORDS[:, 0, :]
        TRANS = COORDS[:, 1:, :] - SHIFTS[:, np.newaxis, :]
        TRANS = TRANS.swapaxes(1, 2)

        self.__embeddings = (TRANS, SHIFTS)
        self.__subentities = (np.arange(len(faces), dtype=np.int32).reshape(-1, 1), edges, faces)
        self.__sizes = (len(faces), num_edges, len(vertices))
        self.domain = np.array([[np.min(vertices[:, 0]), np.min(vertices[:, 1])],
                                [np.max(vertices[:, 0]), np.max(vertices[:, 1])]])

    def size(self, codim=0):
        assert 0 <= codim <= 2, 'Invalid codimension'
        return self.__sizes[codim]

    def subentities(self, codim=0, subentity_codim=None):
        assert 0 <= codim <= 2, 'Invalid codimension'
        if subentity_codim is None:
            subentity_codim = codim + 1
        assert codim <= subentity_codim <= self.dim, 'Invalid subentity codimensoin'
        if codim == 0:
            return self.__subentities[subentity_codim]
        else:
            return super().subentities(codim, subentity_codim)

    def embeddings(self, codim=0):
        if codim == 0:
            return self.__embeddings
        else:
            return super().embeddings(codim)

    def visualize(self, U, codim=2, **kwargs):
        """Visualize scalar data associated to the grid as a patch plot.

        Parameters
        ----------
        U
            |NumPy array| of the data to visualize. If `U.dim == 2 and len(U) > 1`, the
            data is visualized as a time series of plots. Alternatively, a tuple of
            |Numpy arrays| can be provided, in which case a subplot is created for
            each entry of the tuple. The lengths of all arrays have to agree.
        codim
            The codimension of the entities the data in `U` is attached to (either 0 or 2).
        kwargs
            See :func:`~pymor.gui.qt.visualize_patch`
        """
        from pymor.gui.qt import visualize_patch
        from pymor.vectorarrays.interfaces import VectorArrayInterface
        from pymor.vectorarrays.numpy import NumpyVectorSpace, NumpyVectorArray
        if isinstance(U, (np.ndarray, VectorArrayInterface)):
            U = (U,)
        assert all(isinstance(u, (np.ndarray, VectorArrayInterface)) for u in U)
        U = tuple(NumpyVectorSpace.make_array(u) if isinstance(u, np.ndarray) else
                  u if isinstance(u, NumpyVectorArray) else
                  NumpyVectorSpace.make_array(u.data)
                  for u in U)
        bounding_box = kwargs.pop('bounding_box', self.domain)
        visualize_patch(self, U, codim=codim, bounding_box=bounding_box, **kwargs)

    def __str__(self):
        return 'UnstructuredTriangleGrid with {} triangles, {} edges, {} vertices'.format(*self.__sizes)

max_stars_repo_name: mahgadalla/pymor

max_issues_repo_name: mahgadalla/pymor

max_forks_repo_name: mahgadalla/pymor


text:
#%%
import numpy as np
import pandas as pd
import tensorflow as tf
import scipy.io
import random
import xgboost as xgb
import seaborn as sns
from tensorflow import keras
from tensorflow.keras import layers
from sklearn.model_selection import train_test_split
from sklearn import metrics, decomposition

import plotly.graph_objects as go
import matplotlib.pyplot as plt
from tensorflow.python.keras.backend import std
from tensorflow.python.keras.layers import embeddings
from tensorflow.python.keras.layers.core import RepeatVector
from tensorflow.python.keras.losses import mean_squared_error
from tensorflow.python.util.nest import _IF_SHALLOW_IS_SEQ_INPUT_MUST_BE_SEQ

from load_data import *

#%%
SAMPLE_LEN = 5
channels = 8
latent_dim = 3

data, label = split_session(SAMPLE_LEN)

x_train, x_test, y_train, y_test = train_test_split(session_data_1, session_data_1, train_size = 0.85)
x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, train_size = 0.5)

#%%
new_batch = []
for i, batch in enumerate(session_data_1):
    print(batch.shape[1]//SAMPLE_LEN)
    new_batch.extend(np.array_split(batch, batch.shape[1]//SAMPLE_LEN, axis = 1)[1:batch.shape[1]//SAMPLE_LEN-2])

#%%
np.array(new_batch)

#%%

def create_model():

    input = keras.Input(shape = (sample_len,channels))  # Variable-length sequence of ints
    x = layers.LSTM(64, return_sequences=True)(input)
    x = layers.LSTM(64, return_sequences=True)(x)
    #x = layers.RepeatVector(sample_len)(x)
    #x = layers.LSTM(64, return_sequences=True)(x)
    x = layers.TimeDistributed(layers.Dense(16))(x)
    x = layers.Dense(latent_dim, name = 'embedding')(x)
    x = layers.Dense(16)(x)
    output = layers.Dense(channels, activation='linear')(x)

    model = keras.Model(inputs=[input],outputs=[output])

    return model

model = create_model()

keras.utils.plot_model(model, "keras_LSTM_autoencoder.png", show_shapes=True)

#%%
model.summary()

model.compile(optimizer=keras.optimizers.Adam(1e-3), loss='mse') #tf.keras.metrics.mean_squared_error

history = model.fit(x_train, y_train,
                    validation_data = (x_val, y_val),
                    epochs=100, batch_size=32)

plt.plot(history.history['loss'])
plt.yscale("log")
plt.show()

def get_embedding(sequences, model):

    encoder_model = keras.Model(inputs = model.input,
                                outputs = model.get_layer('embedding').output)
    try:
        emb = encoder_model.predict(sequences)[:,-1,:]
    except:
        sequences = list(sequences)
        emb = encoder_model.predict(sequences)[:,-1,:]
    return emb

def plot_3D_embedding(sequences, model):
    emb_vec = get_embedding(sequences, model)

    try:
        emb_vec = emb_vec.reshape(3,len(list(emb_vec)))
    except:
        'Embedding dim != 3'

    fig = go.Figure(data=[go.Scatter3d(x=emb_vec[0], y=emb_vec[1], z=emb_vec[2],
                                    mode='markers')])
    fig.show()

def plot_curve_comparison(sequences, model, n_sample = 1, title = ''):

    for i in range(n_sample):
        pred = model.predict(sequences[i])

        plt.plot(sequences[i], label = 'Truth')
        plt.plot(pred.reshape(sample_len), label = 'Pred')
        plt.title(title)
        plt.legend()
        plt.show()

def plot_reconstruction_error(sequences, model, n_sample = 1, title = ''):

    res = model.predict(sequences)

    try:
        s_len = len(sequences)
    except:
        s_len = sequences.shape[0]

    for i in np.random.randint(0, s_len - 1, size = n_sample):
        plt.plot(res[i] - sequences[i])
        plt.title(title)
        plt.show()

#%%

plot_reconstruction_error(x_train, model, 2, title = 'Train: Rec - Truth')
plot_reconstruction_error(x_test, model, 2, title = 'Test: Rec - Truth')
plot_reconstruction_error(x_val, model, 2, title = 'Val: Rec - Truth')

if channels == 1:
    plot_curve_comparison(x_train, model, title = 'Train')
    plot_curve_comparison(x_test, model, title = 'Test')
    plot_curve_comparison(x_val, model, title = 'Validation')

if latent_dim == 3:
    plot_3D_embedding(x_train, model)
    plot_3D_embedding(x_test, model)
    plot_3D_embedding(x_val, model)

#%%

if channels == 1:
    def stress_seq(rand = 'normal'):
        ampl = np.random.uniform(0.1, 1.)
        freq = np.random.uniform(0.01, 0.1)
        high_noise = np.random.uniform(0.1, 0.5)
        low_noise  = np.random.uniform(0.1, 0.5)
        std = np.random.uniform(0.1,0.3)

        if rand == 'normal':
            recorded_data = ampl*np.random.normal(0, std, size = (channels, sample_len))
        else:
            recorded_data = ampl*np.random.uniform(-low_noise, high_noise, size=( channels, sample_len))

        recorded_data += ampl*np.cos(freq*np.arange(0,sample_len))

        #recorded_data = np.array([np.cumsum(j) for j in recorded_data])
        data = recorded_data.reshape(1, sample_len, channels)
        return recorded_data, data

    stress_test = [stress_seq()[1] for i in range(10)]

    for i in stress_test:
        preds = model.predict(i)

        for j in range(channels):
            plt.plot(i.reshape(sample_len))

        plt.plot(preds[0])
        plt.show()

# %%
if channels == 1:
    for patient,p in zip(data, range(len(data))):
        print('Patient number %d ' % (p + 1))
        for channel,i in zip(patient, range(len(patient))):

            pred = model.predict(channel)

            rmse = np.sqrt(metrics.mean_squared_error(channel.reshape(-1), pred.reshape(-1)))

            print('\tPred on channel %d has RMSE of %f' % (i, rmse))

# %%

if channels == 1:
    # SINGLE CHANNEL
    for patient,p in zip(data_labeled, range(len(data_labeled))):
        print('\n\nPatient number %d ' % (p + 1))
        for channel,i in zip(patient, range(len(patient))):

            data_serie = list([list(i) for i in channel[:,0]])
            label_serie = list([list(i) for i in channel[:,1]])

            emb_serie = get_embedding(data_serie, model)

            x_b_train, x_b_test, y_b_train, y_b_test = train_test_split(emb_serie, label_serie, test_size = 0.2, stratify=label_serie)

            xgb_classifier = xgb.XGBClassifier(use_label_encoder=False)
            xgb_classifier.fit(x_b_train, y_b_train, eval_metric='error')

            xgb_preds = xgb_classifier.predict(x_b_test)

            rmse = np.sqrt(metrics.mean_squared_error(channel.reshape(-1), channel.reshape(-1)))

            print('\tChannel %d - Classifier ROC-AUC: %f' % (i,np.sqrt(metrics.roc_auc_score(y_b_test, xgb_preds))))

    ##################################
    # UNION OF CHANNELS

    for patient,p in zip(data_labeled, range(len(data_labeled))):
        print('\n\nPatient number %d ' % (p + 1))
        emb_serie = []
        for channel,i in zip(patient, range(len(patient))):

            data_serie = list([list(i) for i in channel[:,0]])
            label_serie = list([list(i) for i in channel[:,1]])

            emb = get_embedding(data_serie, model)
            emb = list(list(i) for i in emb)

            if i == 0:
                emb_serie = emb
            else:
                for j in range(len(channel)):
                    for k in emb[j]:
                        emb_serie[j].append(k)

        x_b_train, x_b_test, y_b_train, y_b_test = train_test_split(np.array(emb_serie), label_serie, test_size = 0.2, stratify=label_serie)

        xgb_classifier = xgb.XGBClassifier(use_label_encoder=False)
        xgb_classifier.fit(x_b_train, y_b_train, eval_metric='error')

        xgb_preds = xgb_classifier.predict(x_b_test)

        print('\tClassifier ROC-AUC: %f' % np.sqrt(metrics.roc_auc_score(y_b_test, xgb_preds)))

    ####################################
    # ONE CHANNEL ALONE

    labeled_seq = np.array([[serie[i: i + sample_len],matrixS1['trig'][i]] for i in range(len(serie)) if matrixS1['trig'][i] != 0][:-20])

    data_serie = list([list(i) for i in labeled_seq[:,0]])
    label_serie = list([list(i) for i in labeled_seq[:,1]])

    emb_serie = get_embedding(data_serie, model)

    x_b_train, x_b_test, y_b_train, y_b_test = train_test_split(emb_serie, label_serie, test_size = 0.2, stratify=label_serie)

    xgb_classifier = xgb.XGBClassifier(use_label_encoder=True)
    xgb_classifier.fit(x_b_train, y_b_train)

    xgb_preds = xgb_classifier.predict(x_b_test)
#%%

if channels != 1:
    def labeled_multichannel_data_factory():
        data = []
        labels = []
        for i in range(5):
            patient = []
            patient_lab = []
            matrixS = scipy.io.loadmat('./data/wetransfer-01a2dc/p300/S' + str(i+1) + '.mat')
            for serie in matrixS['y'].T:
                fs = matrixS['fs'][0,0]
                offset = int(0.3*fs)

                seq = np.array([serie[i + offset - int(sample_len/2): i + offset + int(sample_len/2)] for i in range(len(serie)) if matrixS['trig'][i] != 0][:-20])
                label = np.array([0.5*matrixS['trig'][i] + 0.5 for i in range(len(serie)) if matrixS['trig'][i] != 0][:-20])
                patient.append(seq)
                patient_lab.append(label)

            patient = np.array(patient).reshape(-1,sample_len,channels)
            label = np.array(patient_lab)
            data.append(patient)
            labels.append(label[0])
        return data, labels

    dat, lab = labeled_multichannel_data_factory()
    dat = dat/np.max(data[0])*10

    for patient,i in zip (dat, range(len(dat))):
        emb = get_embedding(patient, model)

        x_b_train, x_b_test, y_b_train, y_b_test = train_test_split(emb, lab[i], test_size = 0.2, stratify = lab[i])

        xgb_classifier = xgb.XGBClassifier(use_label_encoder=False)
        xgb_classifier.fit(x_b_train, y_b_train, eval_metric='error')

        xgb_preds = xgb_classifier.predict(x_b_test)

        print('\nPatient number %d' % i)
        print('Classifier ROC-AUC: %f' % np.sqrt(metrics.roc_auc_score(y_b_test, xgb_preds)))

        confusion_matrix = metrics.confusion_matrix(y_b_test, xgb_preds)
        cm_plot = metrics.ConfusionMatrixDisplay(confusion_matrix)
        cm_plot.plot()

        plt.show()
        #xgb.plot_importance(xgb_classifier)

# %%

plot_pairplot = True

if channels != 1:
    for patient,i in zip (dat, range(len(dat))):
        emb = get_embedding(patient, model)
        pd_emb = pd.DataFrame(emb)
        pd_emb['signal'] = lab[i]

        if plot_pairplot:
            g = sns.pairplot(pd_emb, corner = True, hue = 'signal')
            g.map_lower(sns.kdeplot, levels=3, color=".2")
            plt.show()

        pca = decomposition.PCA(n_components=2)
        emb_pca = pca.fit(emb).transform(emb)

        plt.scatter([emb_pca.T[0][j] for j in range(len(emb_pca)) if lab[i][j] == 0],
                    [emb_pca.T[1][j] for j in range(len(emb_pca)) if lab[i][j] == 0],
                    label = 'No signal')
        plt.scatter([emb_pca.T[0][j] for j in range(len(emb_pca)) if lab[i][j] == 1],
                    [emb_pca.T[1][j] for j in range(len(emb_pca)) if lab[i][j] == 1],
                    label = 'Signal')
        plt.legend()
        plt.title('PCA')
        plt.show()

# %%

reason: import numpy,import scipy


text:
# --------------
#Header files
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
#path of the data file- path
data=pd.read_csv(path)
#print(data.head())
#Code starts here
data['Gender'].replace('-','Agender', inplace=True)
gender_count=data.Gender.value_counts()
gender_count.plot(kind='bar')

# --------------
#Code starts here
alignment=data.Alignment.value_counts()
#pie_labels='Character Alignment'
explode=(0.1,0,0)
plt.pie(alignment, explode=explode, labels= alignment.index)
plt.title('Character Alignment')
plt.show()

# --------------
#Code starts here
sc_df=data[['Strength', 'Combat']]
#print(sc_df.head())
#mean_strength=round(data.Strength.mean(),2)
#mean_combat=round(data.Combat.mean(),2)
#diff_strength=round(data.Strength-mean_strength,2)
#diff_combat=round(data.Combat-mean_combat,2)
#summation=round((diff_strength*diff_combat).sum(),2)
sc_covariance=round(sc_df.cov().iloc[0,1],2)
#print(sc_covariance)
sc_strength=sc_df['Strength'].std()
sc_combat=sc_df['Combat'].std()

#sc_covariance=round(summation/len(sc_df),2)
#print(sc_covariance)
#now calculate the SD
#strength_Distance=(data.Strength-mean_strength)**2
#sc_strength=(strength_Distance.sum()/len(strength_Distance))**(1/2)
#similarly for combat
#combat_Distance=(data.Combat-mean_combat)**2
#sc_combat=(combat_Distance.sum()/len(combat_Distance))**(1/2)
sc_pearson=sc_covariance/(sc_strength*sc_combat)
#print(sc_pearson)
#now we will do the same for Intelligence and combat columns
ic_df=data[['Intelligence', 'Combat']]
#print(ic_df.head())
#mean_intelligence=data.Intelligence.mean()
#diff_Intelligence=data.Intelligence-mean_intelligence
#summation_ic=(diff_Intelligence*diff_combat).sum()
#ic_covariance=summation_ic/len(ic_df)
ic_covariance=round(ic_df.cov().iloc[0,1],2)
ic_intelligence=ic_df['Intelligence'].std()
ic_combat=ic_df['Combat'].std()
#print(ic_covariance)
#Intelligence_Distance=(data.Intelligence-mean_intelligence)**2
#ic_intelligence=(Intelligence_Distance.sum()/len(Intelligence_Distance))**(1/2)
ic_pearson=ic_covariance/(ic_intelligence*ic_combat)

# --------------
#Code starts here
total_high=data['Total'].quantile(q=0.99)
#print(total_high)
super_best=data[data['Total']>total_high]
super_best_names=list(super_best['Name'])

print(super_best_names)
#print(super_best.head())

# --------------
#Code starts here
fig, (ax_1, ax_2, ax_3)=plt.subplots(1,3, figsize=(20,8))
ax_1.boxplot(super_best['Intelligence'])
ax_1.set(title='Intelligence')
ax_2.boxplot(super_best['Speed'])
ax_2.set(title='Speed')
ax_3.boxplot(super_best['Power'])
ax_3.set(title='Power')

max_stars_repo_name: Nitish-Khairnar/ga-learner-dsmp-repo

max_issues_repo_name: Nitish-Khairnar/ga-learner-dsmp-repo

max_forks_repo_name: Nitish-Khairnar/ga-learner-dsmp-repo


text:
import pandas as pd
import numpy as np
import statsmodels.api as sm
import scipy.stats as st
import matplotlib.pyplot as plt
import seaborn as sn
from sklearn.metrics import confusion_matrix
import matplotlib.mlab as mlab

#read data
url = 'https://storage.googleapis.com/great_lakes/LakeIce_PhysicalProps.csv'
lake_df = pd.read_csv(url)
lake_df = lake_df.iloc[: , 1:]
#read data
lake_df.head()

#mission values
lake_df.isnull().sum()
count=0
for i in lake_df.isnull().sum(axis=1):
    if i>0:
        count=count+1
print('Total number of rows with missing values is ', count)
print(round((count/len(lake_df.index))*100), 'percent of the entire dataset the rows with missing values are excluded.')
lake_df.dropna(axis=0,inplace=True)

lake_df.describe()
lake_df['iceflag'] = np.where(lake_df['Ice_pct'] > 53.3 , 1, 0) #53.3% ice percentage https://research.noaa.gov/article/ArtMID/587/ArticleID/2706/NOAA-projects-30-percent-average-Great-Lakes-ice-cover-for-2021-winter
lake_df.iceflag.value_counts()

#Lakes
lake_df.Lake.unique()
lake_df.Lake.value_counts()

#Split data
from sklearn.model_selection import train_test_split
X = lake_df.drop(['Year','Day','id','Ice_pct'], axis=1) #feature vector and target variable
Y = lake_df['iceflag']

from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 0)
X_train.shape, X_test.shape # check the shape of X_train and X_test

#Categorical data
X_train = pd.concat([X_train.Elevation_meters,X_train.Length_km,X_train.Breadth_km,X_train.Avg_Depth_meters,
                     X_train.Max_Depth_meters,X_train.Volume_km3,X_train.Water_Area_km2,X_train.Land_Drain_Area_km2,
                     X_train.Total_Area_km2,X_train.Shore_Length_km,X_train.Retention_Time_years,X_train.Surface_Temp_C,
                     pd.get_dummies(X_train.Lake),], axis=1)
X_test = pd.concat([X_test.Elevation_meters,X_test.Length_km,X_test.Breadth_km,X_test.Avg_Depth_meters,
                     X_test.Max_Depth_meters,X_test.Volume_km3,X_test.Water_Area_km2,X_test.Land_Drain_Area_km2,
                     X_test.Total_Area_km2,X_test.Shore_Length_km,X_test.Retention_Time_years,X_test.Surface_Temp_C,
                     pd.get_dummies(X_test.Lake),], axis=1)

#Scaling
X_train.describe()
cols = X_train.columns
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
X_train = pd.DataFrame(X_train, columns=[cols])
X_test = pd.DataFrame(X_test, columns=[cols])
X_train.describe()

# train logit model training set
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression(random_state=0)
# fit the model
logreg.fit(X_train, Y_train)
THRESHOLD = 0.533

#Predict
#predict(X) Predict class labels for samples in X.
#predict_log_proba(X) Predict logarithm of probability estimates.
#predict_proba(X) Probability estimates
#Xnew = [[...], [...]]
#ynew = model.predict(Xnew)
Y_pred_test = logreg.predict(X_test)
Y_pred_test
# probability of getting output as 0 - no max ice cover
logreg.predict_proba(X_test)[:,0]
# probability of getting output as 1 - max ice cover
logreg.predict_proba(X_test)[:,1]

#accuracy
from sklearn.metrics import accuracy_score
#	for floating point numbers #0.9345
print('Model accuracy score: {0:0.4f}'. format(accuracy_score(Y_test, Y_pred_test))) #y_test are the true class labels and y_pred_test are the predicted class labels in the test-set
#Compare the train-set and test-set accuracy
Y_pred_train = logreg.predict(X_train)
Y_pred_train
print('Training set score: {:.4f}'.format(logreg.score(X_train, Y_train)))
print('Test set score: {:.4f}'.format(logreg.score(X_test, Y_test))) #no overfitting

# fit the Logsitic Regression model with C=100
# instantiate the model
logreg100 = LogisticRegression(C=100, random_state=0)
# fit the model
logreg100.fit(X_train, Y_train)
print('Training set score: {:.4f}'.format(logreg100.score(X_train, Y_train))) #0.9771
print('Test set score: {:.4f}'.format(logreg100.score(X_test, Y_test))) #0.9752

# fit the Logsitic Regression model with C=001
# instantiate the model
logreg001 = LogisticRegression(C=0.01, random_state=0)
# fit the model
logreg001.fit(X_train, Y_train)
# print the scores on training and test set
print('Training set score: {:.4f}'.format(logreg001.score(X_train, Y_train))) #0.8964
print('Test set score: {:.4f}'.format(logreg001.score(X_test, Y_test))) #0.8989

#Compare model accuracy with null accuracy
# check class distribution in test set
Y_test.value_counts()
null_accuracy = (1592/(1575+179))
print('Null accuracy score: {0:0.4f}'. format(null_accuracy))

#Confusion matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(Y_test, Y_pred_test)
print('Confusion matrix\n\n', cm)
print('\nTrue Positives(TP) = ', cm[0,0])
print('\nTrue Negatives(TN) = ', cm[1,1])
print('\nFalse Positives(FP) = ', cm[0,1])
print('\nFalse Negatives(FN) = ', cm[1,0])

# Classification
from sklearn.metrics import classification_report
print(classification_report(Y_test, Y_pred_test))

TP = cm[0,0]
TN = cm[1,1]
FP = cm[0,1]
FN = cm[1,0]

precision = TP / float(TP + FP)
print('Precision : {0:0.4f}'.format(precision))
Sensitivity = TP / float(TP + FN)
print('Sensitivity : {0:0.4f}'.format(Sensitivity))
specificity = TN / (TN + FP)
print('Specificity : {0:0.4f}'.format(specificity))

# plot ROC Curve
from sklearn.metrics import roc_curve
fpr, tpr, thresholds = roc_curve(Y_test, Y_pred_test)
plt.figure(figsize=(6,4))
plt.plot(fpr, tpr, linewidth=2)
plt.plot([0,1], [0,1], 'k--' )
plt.rcParams['font.size'] = 12
plt.title('ROC curve')
plt.xlabel('False Positive Rate (1 - Specificity)')
plt.ylabel('True Positive Rate (Sensitivity)')
plt.show()

from numpy import argmax
from numpy import sqrt
# calculate the g-mean for each threshold
gmeans = sqrt(tpr * (1-fpr))
# locate the index of the largest g-mean
ix = argmax(gmeans)
print('Best Threshold=%f, G-Mean=%.3f' % (thresholds[ix], gmeans[ix])) #Best Threshold=1.000000, G-Mean=0.637

# compute ROC AUC
from sklearn.metrics import roc_auc_score
ROC_AUC = roc_auc_score(Y_test, Y_pred_test)
print('ROC AUC : {:.4f}'.format(ROC_AUC)) #ROC AUC : 0.7008

# GUI IMPLEMENTATION
from tkinter import *
root = Tk()
root.title('logit Error Calculator')

e = Entry(root, width=35)
e.pack()

def myClick():
    user_input = eval(e.get())

    GUIclassifier = LogisticRegression(C=user_input)
    GUIclassifier.fit(X_train, Y_train)
    GUI_preds = GUIclassifier.predict(X_test)
    accuracy = accuracy_score(Y_test, GUI_preds) * 100
    GUIstring = "The accuracy percentage for your chosen k-value is " + str(round(accuracy, 2)) + "%."
    myLabel = Label(root, text=GUIstring)
    myLabel.pack()

myButton = Button(root, text= "Enter c value", command=myClick)
myButton.pack()

root.mainloop()

reason: import numpy,from numpy,import scipy,import statsmodels


text:
# 0=anger, 1=contempt, 2=disgust, 3=fear, 4=happy, 5=sadness, 6=surprise
# contains 45, 18, 59, 25, 69, 28, 83 images

import os
import sys
import numpy as np
import h5py
import cv2

def main():
    ck_path = 'CK+120'
    data_x = []
    data_y = []

    out_path = os.path.join('data', 'CK_data.h5')
    if not os.path.exists(os.path.dirname(out_path)):
        os.makedirs(os.path.dirname(out_path))

    cat = os.listdir(ck_path)
    cat.sort()
    for label in cat:
        cur_path = os.path.join(ck_path, label)
        items = os.listdir(cur_path)
        items.sort()
        for item in items:
            img = cv2.imread(os.path.join(cur_path, item), 0)
            data_x.append(img.tolist())
            data_y.append(int(label))

    print(np.shape(data_x))
    print(np.shape(data_y))

    datafile = h5py.File(out_path, 'w')
    datafile.create_dataset("data_pixel", dtype='uint8', data=data_x)
    datafile.create_dataset("data_label", dtype='int64', data=data_y)
    datafile.close()

    print("Save data finish!!!")

if __name__ == "__main__":
    sys.exit(main())


text:
import tensorflow as tf
import pandas as pd
import numpy as np
from tensorflow.python.tools import freeze_graph
import os

dir = os.path.dirname(os.path.realpath(__file__)) + '/data/'

seed = 1234
np.random.seed(seed)
tf.set_random_seed(seed)

dataset = pd.read_csv('dataset.csv')
dataset = pd.get_dummies(dataset, columns=['Species'])
values = list(dataset.columns.values)

y = dataset[values[-3:]]
y = np.array(y, dtype='float32')
X = dataset[values[1:-3]]
X = np.array(X, dtype='float32')

indices = np.random.choice(len(X), len(X), replace=False)
X_values = X[indices]
y_values = y[indices]

test_size = 10
X_test = X_values[-test_size:]
X_train = X_values[:-test_size]
y_test = y_values[-test_size:]
y_train = y_values[:-test_size]

sess = tf.Session()

interval = 50
epoch = 500

X_data = tf.placeholder(shape=[None, 4], dtype=tf.float32, name='input')
y_target = tf.placeholder(shape=[None, 3], dtype=tf.float32)

hidden_layer_nodes = 8

w1 = tf.Variable(tf.random_normal(shape=[4,hidden_layer_nodes]), name='weights1')
b1 = tf.Variable(tf.random_normal(shape=[hidden_layer_nodes]), name='biases1')
w2 = tf.Variable(tf.random_normal(shape=[hidden_layer_nodes,3]), name='weights2')
b2 = tf.Variable(tf.random_normal(shape=[3]),name='biases2')

hidden_output = tf.nn.relu(tf.add(tf.matmul(X_data, w1), b1))
final_output = tf.nn.softmax(tf.add(tf.matmul(hidden_output, w2), b2), name='output')

loss = tf.reduce_mean(-tf.reduce_sum(y_target * tf.log(final_output), axis=0))

optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(loss)

init = tf.global_variables_initializer()
sess.run(init)

saver = tf.train.Saver()

print('Training the model...')
for i in range(1, (epoch + 1)):
    sess.run(optimizer, feed_dict={X_data: X_train, y_target: y_train})
    if i % interval == 0:
        print('Epoch', i, '|', 'Loss:', sess.run(loss, feed_dict={X_data: X_train, y_target: y_train}))

tf.train.write_graph(sess.graph_def, dir, 'graph.pbtxt')
saver.save(sess, dir+'graph')
freeze_graph.freeze_graph(dir+'graph.pbtxt','', False, dir+'graph', 'output', 'save/restore_all', 'save/Const:0', 'data/graphdef.pb', True, '')

for i in range(len(X_test)):
    print('For datapoint: '+ str(X_test[i])+ ' Actual:', y_test[i], 'Predicted:', np.rint(sess.run(final_output, feed_dict={X_data: [X_test[i]]})))

max_stars_repo_name: smokinguns/tensorflex

max_issues_repo_name: velimir/tensorflex

max_forks_repo_name: velimir/tensorflex


text:
import abc
import logging
from io import BytesIO

import numpy as np
from PIL import Image
from pynput import keyboard

from .utils import WebWrapper

class BasicAPI(abc.ABC):
    def __init__(self, url):
        self.logger = logging.getLogger(__name__)
        self.url = url
        self.wrapper = WebWrapper(headless=False, window_size=(650, 650))

    def open(self):
        self.wrapper.driver.get(self.url)
        self.wrapper.wait_find(xpath='//*[@id="game_title"]')
        self.logger.info('Game loaded')
        self.wrapper.driver.switch_to_window(self.wrapper.driver.current_window_handle)

    @abc.abstractmethod
    def play(self):
        raise NotImplementedError

    @abc.abstractmethod
    def auto(self):
        raise NotImplementedError

    def close(self):
        self.wrapper.close()

    def ss(self):
        img_bytes = self.wrapper.driver.get_screenshot_as_png()
        return np.array(Image.open(BytesIO(img_bytes)))

    @abc.abstractmethod
    def gameover(self):
        raise NotImplementedError

    @staticmethod
    def user_stop(key):
        if key == keyboard.Key.esc:
            # Stop listener
            return False

max_stars_repo_name: vmgustavo/autotgame

max_issues_repo_name: vmgustavo/autotgame

max_forks_repo_name: vmgustavo/autotgame


text:
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
iterative_algorithms.py: Contains functions to build up the trajectory iteratively over time.

"""

import matplotlib.pylab as plt
import numpy as np
import pandas as pd

from solvers import trajectory_recovery

def verify_dimensions(D, anchors, basis, times):
    N, M = D.shape
    dim = anchors.shape[0]
    if anchors.shape[1] != M:
        raise ValueError(D.shape, anchors.shape, basis.shape, len(times))
    if basis.shape[1] != N:
        raise ValueError(D.shape, anchors.shape, basis.shape, len(times))
    if len(times) != N:
        raise ValueError(D.shape, anchors.shape, basis.shape, len(times))

def averaging_algorithm(D, anchors, basis, times, t_window=1.0, n_times=None, verbose=False):
    """ Iteratively compute estimates over fixed time window.

    :param D: measurement matrix with squared distances (N x M)
    :param anchors: anchor coordinates (dim x M)
    :param basis: basis vectors (K x N)
    :param times: list of measurement times (length N)

    :param t_window: width of fixed time window.

    """
    if type(times) == list:
        times = np.array(times)

    verify_dimensions(D, anchors, basis, times)

    D_k = np.empty((0, D.shape[1]))
    basis_k = np.empty((basis.shape[0], 0))
    C_list = []
    t_list = []

    # make sure we always have overlap
    if n_times is None:
        t_start = np.arange(times[0], times[-1], step=t_window / 2)
    else:
        t_start = np.linspace(times[0], times[-1], n_times)

    for t_s in t_start:
        tk = times[(times >= t_s) & (times < t_s + t_window)]
        for t_n in tk:
            idx = np.where(times == t_n)[0]

            d_mn_row = D[idx, :]
            f_n = basis[:, idx]

            D_k = np.r_[D_k, d_mn_row]
            basis_k = np.c_[basis_k, f_n]

        try:
            C_k = trajectory_recovery(D_k, anchors, basis_k)
            # We need this somehow for numercial reasons.
            # Otherwise sometimes the tests fail because -0. != 0.
            C_k[np.abs(C_k) <= 1e-10] = 0.0
            C_list.append(C_k)
        except AssertionError:
            if verbose:
                print('skipping {:.2f} because only {} measurements.'.format(t_s, len(np.array(tk))))
        except np.linalg.LinAlgError:
            if verbose:
                print('skipping {:.2f} because failed.'.format(t_s))
        except ValueError:
            raise

        D_k = np.empty((0, D.shape[1]))
        basis_k = np.empty((basis.shape[0], 0))
        t_list.append(tk)
    return C_list, t_list

def build_up_algorithm(D, anchors, basis, times, eps=1, verbose=False):
    """ Build-up algorithm for trajectory estimation.

    Build up different trajectories as long as measurements "fit". When they
    stop fitting (see eps parameter), start a new trajectory.

    :param D: measurement matrix with squared distances (N x M)
    :param anchors: anchor coordinates (dim x M)
    :param basis: basis vectors (K x N)
    :param times: list of measurement times (length N)

    :param eps: error threshold for starting new trajectory.

    """

    verify_dimensions(D, anchors, basis, times)

    C_k = None
    tk = []

    D_k = np.empty((0, D.shape[1]))
    basis_k = np.empty((basis.shape[0], 0))

    C_list = []
    t_list = []

    def g(C_k):
        r_n = C_k.dot(f_n).reshape((ams.shape[0], 1))
        dist_estimates = np.linalg.norm(ams - r_n, axis=0)
        distances = np.sqrt(d_mn_row.flatten()[d_mn_row.flatten() > 0])
        return np.sum(np.abs(dist_estimates - distances)) / len(distances)  # MAE

    for d_mn_row, t_n, f_n in zip(D, times, basis.T):
        d_mn_row = d_mn_row.reshape((1, -1))
        ams = anchors[:, d_mn_row[0] > 0]
        if C_k is None or g(C_k) <= eps:
            D_k = np.r_[D_k, d_mn_row]
            basis_k = np.c_[basis_k, f_n]
            tk.append(t_n)
            try:
                C_test = trajectory_recovery(D_k, anchors, basis_k)
                assert C_test is not None
                # TODO find a sensible general threshold here.
                assert g(C_test) < 2 * eps
                C_k = C_test
                # We need this somehow for numercial reasons.
                # Otherwise sometimes the tests fail because -0. != 0.
                C_k[np.abs(C_k) <= 1e-10] = 0.0
            except AssertionError as e:
                if verbose:
                    print('skipping {:.2f} because only {} measurements.'.format(t_n, len(np.array(tk))))
            except np.linalg.LinAlgError:
                if verbose:
                    print('skipping {:.2f} because failed'.format(t_n))

        elif (C_k is not None) and g(C_k) > eps:
            if verbose:
                print('changing to new trajectory, because error is {:.4f}'.format(g(C_k)))

            basis_k = f_n
            D_k = d_mn_row

            C_list.append(C_k)
            t_list.append(tk)

            tk = [t_n]
            C_k = None

    if C_k is not None:
        C_list.append(C_k)
        t_list.append(tk)
    return C_list, t_list

def get_smooth_points(C_list, t_list, traj):
    """ Average the obtained trajectories. """
    result_df = pd.DataFrame(columns=['px', 'py', 't'])
    for Chat, t in zip(C_list, t_list):
        traj.set_coeffs(coeffs=Chat)
        positions = traj.get_sampling_points(times=t)
        this_df = pd.DataFrame({'px': positions[0, :], 'py': positions[1, :], 't': t})
        result_df = pd.concat((this_df, result_df))
    result_df.sort_values('t', inplace=True)
    result_df.reindex()

    import datetime
    mean_window = 10
    datetimes = [datetime.datetime.fromtimestamp(t) for t in result_df.t]
    result_df.index = [pd.Timestamp(datetime) for datetime in datetimes]
    result_df.loc[:, 'px_median'] = result_df['px'].rolling('{}s'.format(mean_window), min_periods=1,
                                                            center=False).median()
    result_df.loc[:, 'py_median'] = result_df['py'].rolling('{}s'.format(mean_window), min_periods=1,
                                                            center=False).median()
    return result_df

def plot_individual(C_list, t_list, traj):
    fig, ax = plt.subplots()
    fig.set_size_inches(10, 7)

    for Chat, t in zip(C_list, t_list):
        traj.set_coeffs(coeffs=Chat)
        if len(t) > 0:
            traj.plot(ax=ax, times=t)
            #traj.plot(ax=ax, times=t, label='{:.1f}'.format(t[0]))
    return ax

def plot_smooth(result_df):
    fig, ax = plt.subplots()
    fig.set_size_inches(10, 7)
    #plt.scatter(result_df.px, result_df.py, s=1)
    plt.scatter(result_df.px_median, result_df.py_median, s=2, color='red')
    plt.plot(result_df.px_median, result_df.py_median, color='red')
    return ax

max_stars_repo_name: duembgen/continuous-localization

max_issues_repo_name: duembgen/continuous-localization

max_forks_repo_name: duembgen/continuous-localization


text:
import cv2
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow.keras.backend as K

# def scale(array, maxv=1):
#     return ((array - K.min(array))/(K.max(array) - K.min(array)))*maxv
def scale(array):
    return (array-np.min(array))/4095

def edge_detect(img):
    X_edges = []
    for i in range(len(img[:,0,0,0])):
        X_edges.append(
            cv2.medianBlur(
                cv2.bitwise_not(
                    cv2.adaptiveThreshold(
                        scale(img[i,:,:,0], 255).astype(np.uint8),
                        255,
                        cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
                        cv2.THRESH_BINARY,
                        15, 2
                    )
                )
            ,9)
        )
    return X_edges

def find_segmentation(y):
    y_slices=len(y[:,0,0])-1
    for slice in range(y_slices):
        if np.sum(y[slice,:,:])>0:
            begining_slice=slice
            break
    for slice in range(y_slices):
        if np.sum(y[(y_slices-slice),:,:])>0:
            ending_slice=y_slices-slice
            break
    return begining_slice, ending_slice

#CURRENTLY WINDOWING AND EDGE DETECTION TURNED OFF
def preprocess_X(volume):
    #load as array
    X = volume.get_fdata()
    X = scale(X)
    # X = tf.cast(X, dtype=tf.float16)
    X = np.expand_dims(X, 3)
    if not X.shape[1] == 512 or not X.shape[2] == 512:
        X = tf.image.resize(X, size=(512, 512))

    # X_window = scale(X_window)
    # edge detection
    # CURRENTLY TURNED OFF
    # X_edges = edge_detect(X)
    # X_edges = scale(X_edges, 1)
    # X_edges = np.expand_dims(X_edges, 3)

    # concat contrast with edges
    # X = np.concatenate((X_window, X_edges), axis=3).astype(np.float32)
    return X

def preprocess_y(segmentation):
    y = segmentation.get_fdata()
    begining, end = find_segmentation(y)
    # y = tf.cast(y, dtype=tf.float16)
    y = tf.keras.utils.to_categorical(y)
    if not y.shape[1] == 512 or not y.shape[2] == 512:
        y = tf.image.resize(y, size=(512, 512))

    return y, begining, end


text:
# -*- coding: utf-8 -*-
'''
Created on 2020年12月29日

@author: xiaoyw
'''
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from DataBase.Car_Info import Gas_Collection

class ComboData(object):
    def __init__(self,datas):
        self.df = datas
        # 统一量纲为小时
        self.df['fuel_time'] = round(self.df['fuel_time']/60,2)
        # 进出站间隔时间
        self.df['Plate_interva'] = self.df['Entry_interva'] + self.df['Dep_interva'] + self.df['fuel_time']
        self.df['Entry_pre'] = round(self.df['Entry_interva']/self.df['Plate_interva'],3)*100
        self.df['Dep_pre'] = round(self.df['Dep_interva']/self.df['Plate_interva'],3)*100

    def  set_datas(self,times=1):
        self.df.sort_values(by=['IC_ID','fuelle_date'],inplace=True,ascending=[True,True])
        # 类似SQL的having，筛选分组结果
        df_ID = self.df.groupby(['Flag','IC_ID', 'License_plate_R'], as_index=False).filter(lambda x:len(x)>times).groupby(['Flag','IC_ID', 'License_plate_R'], as_index=False).size()
        df_ID = df_ID[['Flag','IC_ID', 'License_plate_R']]
        df_feature = self.df[['Flag','IC_ID','License_plate_R','gas_station','year','month','day','weekday','IC_time','fuel_time','Entry_time','Shop_time','Dep_time']]
        #df_feature = df_feature.rename(columns={'IC_time':'fuel_time','License_plate_R':'Lic_plate'})
        df_feature = df_feature.rename(columns={'License_plate_R':'Lic_plate'})

        y_df = df_ID[['Flag']]
        self.y_=pd.DataFrame(columns=('TrueIC','FalseIC'))
        self.y_['TrueIC']=y_df['Flag'].apply(lambda x:0 if x==0 else 1)
        self.y_['FalseIC']=y_df['Flag'].apply(lambda x:1 if x==0 else 0)
        self.y_= self.y_.values

        # 取20个行数据，不足补零
        x_data = []
        #cols_name = ['IC_ID','Lic_plate','gas_station','year','month','day','weekday','fuel_time','Entry_time','Shop_time','Dep_time']
        cols_name = ['IC_ID','Lic_plate','gas_station','year','month','day','weekday','IC_time','fuel_time','Entry_time','Shop_time','Dep_time']
        for index,row in df_ID.iterrows():
            IC_ID = row['IC_ID']
            Lic_plate = row['License_plate_R']

            df_tmp = df_feature[(df_feature['IC_ID']==IC_ID) & (df_feature['Lic_plate']==Lic_plate)][cols_name]
            num = len(df_tmp)
            if num > 20:
                num = num - 1
                df_tmp = df_tmp.iloc[(num - 20):num]
            else:
                idx = [i for i in range(20-num)]
                df_null = pd.DataFrame(0,columns=cols_name,index=idx)
                df_null[['IC_ID','Lic_plate']] = IC_ID , Lic_plate
                df_tmp = pd.concat([df_tmp,df_null],axis=0,ignore_index=True).reset_index()
                df_tmp = df_tmp.drop('index',axis=1)

            a = df_tmp.values
            #print(df_tmp)
            a = a.reshape(a.shape[0]*a.shape[1],)
            x_data.append(a)

        x_data= np.array(x_data)

        #x_data.reshape(-1,220)
        print(x_data.shape)
        #
        self.X = x_data
        # 归一化处理
        MM = MinMaxScaler()
        self.X = MM.fit_transform(x_data)
        #print(self.X)
        return

    def  set_datas2(self,times=1):
        #self.df = self.df.drop(drop_cols_name,axis=1)
        #self.df = self.df[self.df['IC_ID_R']>=0].reset_index()
        self.df.sort_values(by=['IC_ID','fuelle_date'],inplace=True,ascending=[True,True])

        #df_ID = self.df.groupby(['Flag','IC_ID', 'License_plate'], as_index=False)
        # 类似SQL的having，筛选分组结果
        df_ID = self.df.groupby(['Flag','IC_ID', 'License_plate_R'], as_index=False).filter(lambda x:len(x)>times).groupby(['Flag','IC_ID', 'License_plate_R'], as_index=False).size()
        df_ID = df_ID[['Flag','IC_ID', 'License_plate_R']]
        df_feature = self.df[['Flag','IC_ID','License_plate_R','gas_station','year','month','day','weekday','IC_time','Entry_time','Shop_time','Dep_time']]
        df_feature = df_feature.rename(columns={'IC_time':'fuel_time','License_plate_R':'Lic_plate'})

        y_df = df_ID[['Flag']]
        self.y_= y_df

        # 取20个行数据，不足补零
        x_data = []
        cols_name = ['gas_station','year','month','day','weekday','fuel_time','Entry_time','Shop_time','Dep_time']
        for index,row in df_ID.iterrows():
            IC_ID = row['IC_ID']
            Lic_plate = row['License_plate_R']

            df_tmp = df_feature[(df_feature['IC_ID']==IC_ID) & (df_feature['Lic_plate']==Lic_plate)][cols_name]
            num = len(df_tmp)
            if num > 20:
                num = num - 1
                df_tmp = df_tmp.iloc[(num - 20):num]
            else:
                idx = [i for i in range(20-num)]
                df_null = pd.DataFrame(0,columns=cols_name,index=idx)
                #df_null = df_null.drop(index)
                #df_null.fillna(0)
                df_tmp = pd.concat([df_tmp,df_null],axis=0,ignore_index=True).reset_index()
                df_tmp = df_tmp.drop('index',axis=1)
                #df_tmp = df_tmp.append(df_null,ignore_index=True).reset_index()

            #x_data.append(df_tmp.values)
            a = df_tmp.values
            a = a.reshape(a.shape[0]*a.shape[1],)
            x_data.append(a) #a.tolist())

        x_data= np.array(x_data)

        #x_data.reshape(-1,220)
        print(x_data.shape)
        #
        self.X = pd.DataFrame(x_data)
        #print(self.X)
        return

    def get_datas(self):
        return self.X, self.y_

def generate_combo_data(times = 0):
    GC = Gas_Collection('study')

    df = GC.get_combo_data()

    # 'combos' 进出站时间内，发生IC加油的次数
    #drop_cols_name = ['Entry_date', 'Departure_date']
    print(df.columns)
    # 探索性数据分析
    EDA = ComboData(df)

    #EDA.correlation_analysis(cols_name)
    EDA.set_datas(times)

if __name__ == '__main__':
    # 1 生成数据并分析
    #generate_combo_data(times = 0)
    # 2 分析数据
    generate_combo_data(1)

    pass


text:
import sys
sys.path.append("..")
import numpy as np
import argparse
import tools
import os

def get_chrom_num(chrom):
	if chrom == "X":
		return 23
	else:
		return int(chrom)

def normalize(chrom1, chrom2, rawpath, krpath1, krpath2, res, outpath):
	kr1 = np.loadtxt(krpath1)
	if krpath2 is None:
		kr2 = kr1
	else:
		kr2 = np.loadtxt(krpath2)
	with open(rawpath) as raw:
		with open(outpath, "w") as out:
			for line in raw:
				line = line.split()
				loc1 = line[0]
				loc2 = line[1]
				norm1 = kr1[int(loc1)/res]
				norm2 = kr2[int(loc2)/res]
				if not np.isnan(norm1) and not np.isnan(norm2):
					out.write("\t".join((chrom1, loc1, str(int(loc1) + res), chrom2, loc2, str(int(loc2) + res), str(float(line[2])/(norm1 * norm2)))) + "\n")
		out.close()
	raw.close()

def normalize_inter(hic_id, res, chrom_a, chrom_b):
	res_string = tools.get_res_string(res)

	if get_chrom_num(chrom_a) < get_chrom_num(chrom_b):
		chrom1 = chrom_a
		chrom2 = chrom_b
	else:
		chrom1 = chrom_b
		chrom2 = chrom_a

	rawpath = "{}/{}_resolution_interchromosomal/chr{}_chr{}/MAPQGE30/chr{}_{}_{}.RAWobserved".format(hic_id, res_string, chrom1, chrom2, chrom1, chrom2, res_string)
	krpath1 = "{}/{}_resolution_interchromosomal/chr{}_chr{}/MAPQGE30/chr{}_{}.KRnorm".format(hic_id, res_string, chrom1, chrom2, chrom1, res_string)
	if not os.path.isfile(krpath1):
		krpath1 = "{}/{}_resolution_interchromosomal/chr{}_chr{}/MAPQGE30/chr{}_{}.VCnorm".format(hic_id, res_string, chrom1, chrom2, chrom1, res_string)
	krpath2 = "{}/{}_resolution_interchromosomal/chr{}_chr{}/MAPQGE30/chr{}_{}.KRnorm".format(hic_id, res_string, chrom1, chrom2, chrom2, res_string)
	if not os.path.isfile(krpath2):
		krpath2 = "{}/{}_resolution_interchromosomal/chr{}_chr{}/MAPQGE30/chr{}_{}VCnorm".format(hic_id, res_string, chrom1, chrom2, chrom2, res_string)
	outpath = "{}_{}_{}_{}.bed".format(hic_id, chrom1, chrom2, res_string)
	chromstring1 = "chr" + chrom1
	chromstring2 = "chr" + chrom2
	normalize(chromstring1, chromstring2, rawpath, krpath1, krpath2, res, outpath)

def normalize_intra(hic_id, res, chrom):
	res_string = tools.get_res_string(res)

	rawpath = "{}/{}_resolution_intrachromosomal/chr{}/MAPQGE30/chr{}_{}.RAWobserved".format(hic_id, res_string, chrom, chrom, res_string)
	krpath = "{}/{}_resolution_intrachromosomal/chr{}/MAPQGE30/chr{}_{}.KRnorm".format(hic_id, res_string, chrom, chrom, res_string)
	if not os.path.isfile(krpath):
		krpath = "{}/{}_resolution_intrachromosomal/chr{}/MAPQGE30/chr{}_{}.VCnorm".format(hic_id, res_string, chrom, chrom, res_string)
	outpath = "{}_{}_{}.bed".format(hic_id, chrom, res_string)
	chromstring = "chr" + chrom
	normalize(chromstring, chromstring, rawpath, krpath, None, res, outpath)

def main():
	parser = argparse.ArgumentParser(description="Normalize Hi-C files using Knight-Ruiz method.")
	parser.add_argument("hic_id", help="e.g. GM12878_combined")
	parser.add_argument("res", type=int, help="resolution (bp)")
	parser.add_argument("chrom1", help="first chromosome (e.g. 1)")
	parser.add_argument("--chrom2", help="second chromosome (e.g. 2)")
	args = parser.parse_args()

	if args.chrom2 is None:
		normalize_intra(args.hic_id, args.res, args.chrom1)
	else:
		normalize_inter(args.hic_id, args.res, args.chrom1, args.chrom2)

if __name__ == "__main__":
	main()


text:
from threading import Thread, Event, Timer
import os
import json
import numpy as np
import awscam
import cv2
import time
import greengrasssdk
import datetime
import boto3
import datetime

client = greengrasssdk.client('iot-data')
iotTopic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME'])
thingName = os.environ['AWS_IOT_THING_NAME'];

motionThreshold = 200
bucketName = 'bucket-name-here'

def load_config_values():
    try:
        ssm = boto3.client('ssm', region_name='us-east-1')
        response = ssm.get_parameters(
            Names=[
                '/Cameras/{cameraKey}/CameraBucket'.format(cameraKey=thingName),
                '/Cameras/{cameraKey}/MotionThreshold'.format(cameraKey=thingName)
            ],
            WithDecryption=False
        )
        global bucketName
        global motionThreshold
        bucketName = response['Parameters'][0]['Value']
        motionThreshold = int(response['Parameters'][1]['Value'])
    except Exception as e:
        log("Getting SSM parameters failed: " + str(e))

def function_handler(event, context):
    return

def log(message):
    client.publish(topic=iotTopic, payload=message)

def mse(imageA, imageB):
	err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2)
	err /= float(imageA.shape[0] * imageA.shape[1])
	return err

def push_to_s3(frame, movement):
    try:
        timestamp = 2000000000 - int(time.time())
        key = "frames/{}/{}.jpg".format(thingName, timestamp)
        s3 = boto3.client('s3')
        encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90]

        frameResize = cv2.resize(frame, (1344, 760))

        _, jpg_data = cv2.imencode('.jpg', frameResize, encode_param)
        response = s3.put_object(Body=jpg_data.tostring(),
                                 Bucket=bucketName,
                                 ContentType='image/jpeg',
                                 Key=key,
                                 Metadata={
                                    'movement': str(movement)
                                 })
    except Exception as e:
        log("Pushing to S3 failed: " + str(e))

def loop():
    try:
        nextFrameIndexTime = datetime.datetime.now() + datetime.timedelta(minutes = 1)
        log("Inside main loop.")
        ret, previousFrame = awscam.getLastFrame()
        doLoop = True
        while doLoop:
            ret, currentFrame = awscam.getLastFrame()
            err = mse(previousFrame, currentFrame)
            if (err > motionThreshold) or (datetime.datetime.now() >= nextFrameIndexTime):
                push_to_s3(currentFrame, err)
                nextFrameIndexTime = datetime.datetime.now() + datetime.timedelta(minutes = 1)
            previousFrame = currentFrame
    except Exception as e:
        log("Error: " + str(e))

    Timer(15, loop).start()

log("loading config values..")
load_config_values()

log("Starting program...")
loop()


text:
import os
from PIL import Image
import csv
import pandas as pd
import numpy as np
import glob
import re

path = "/home/pf/pfshare/data/MA_Rajanie/raw_labels/moritz_cam0_16-17_preds"
dates=[]
dictionary = {}
per_day_analysis = []

for file in os.listdir(path):
    im=Image.open(os.path.join(path, file))
    match = re.search(r'\d{4}_\d{4}', file).group()
    #print(match)

    water =snow=ice=extra=0

    pixels = list(im.getdata())
    for i in pixels:
        if i == 1:
            water+=1
        elif i == 2:
            ice+=1
        elif i==3:
            snow+=1
        else:
            extra ==1

    #print(water, ice, snow)
    lake= water+ice+snow
    coverage = round(float((ice+snow)/(ice+snow+0.001)), 2)
    if match in dictionary.keys():
        dictionary[match].append(coverage)
    else:
        dictionary[match] = [coverage]

for key, values in dictionary.items():
    dates.append(key)
    per_day_analysis.append(np.median(values))

#print(dictionary)

data=np.array([dates,per_day_analysis]).T
df=pd.DataFrame(columns=['date','day_analysis'],data=data)
df.to_csv('/home/pf/pfshare/data/MA_Rajanie/raw_labels/moritz_cam0_16-17_preds.csv',index=False)

max_stars_repo_name: czarmanu/deeplab-lakeice-webcams

max_issues_repo_name: czarmanu/deeplab-lakeice-webcams

max_forks_repo_name: czarmanu/deeplab-lakeice-webcams


text:
import numpy as np
import tensorprob as tp
import scipy.stats as st
from numpy.testing import assert_array_almost_equal, assert_array_equal

def make_normal():
    mu = tp.Parameter(name='mu')
    sigma = tp.Parameter(name='sigma', lower=0)
    X = tp.Normal(mu, sigma)
    return mu, sigma, X

#def test_init():
#    with tp.Model():
#        mu, sigma, X = make_normal()
#        assert(X.mu is mu)
#        assert(X.sigma is sigma)

def test_pdf():
    with tp.Model() as m:
        mu, sigma, X = make_normal()

    m.observed(X)

    xs = np.linspace(-5, 5, 100)
    out1 = st.norm.pdf(xs, 0, 1)
    m.initialize({
        mu: 0,
        sigma: 1
    })
    out2 = m.pdf(xs)
    assert_array_almost_equal(out1, out2, 16)

#def test_cdf():
#    xs = np.linspace(-5, 5, 100)
#    out1 = st.norm.cdf(xs, 0, 1)
#
#    with tp.Model() as model:
#        mu, sigma, X = make_normal()
#
#    out2 = model.session.run(X.cdf(xs), feed_dict={mu: 0, sigma: 1})
#
#    assert_array_almost_equal(out1, out2, 16)

max_stars_repo_name: ibab/tensorfit

max_issues_repo_name: ibab/tensorprob

max_forks_repo_name: ibab/tensorfit

reason: import numpy,from numpy,import scipy