text:
"""
==================
=== Deprecated ===
==================

As of 2018-04-16, this file is no longer used. Operational nowcasting is
implemented in nowcast_update.py.

===============
=== Purpose ===
===============

Nowcasting! Currently doing sensor fusion using eight digital surveillance
sources:
  - gft  (62) | gft-nat,gft-hhs,gft-state
  - ght  (52) | ght-nat,ght-state
  - twtr (71) | twtr-nat,twtr-hhs,twtr-cen,twtr-state
  - wiki (1)  | wiki-nat
  - cdc  (1)  | cdc-nat
  - epic (11) | epic-nat,epic-hhs
  - sar3 (20) | sar3-nat,sar3-hhs,sar3-cen
  - arch (20) | arch-nat,arch-hhs,arch-cen

Note that regions hhs1 and cen1 contain the same states, so their signals
*should* be equal. For this reason, cen1 is omitted when nowcasting; whenever
cen1 is available, hhs1 is also available. The nowcast does however produce an
estimate (identically) for both cen1 and hhs1.

To update sensors, run:
  python3 sensor_update.py <sensor name-loc>
For example, update wiki nationally and twtr at all locations:
  python3 sensor_update.py wiki-nat,twtr-nat,twtr-hhs,twtr-cen,twtr-state

See also: state_info.py, fusion.py, sensor_update.py

=======================
=== Data Dictionary ===
=======================

Nowcasts (value and standard deviation) are stored in the `nowcasts` table.
+----------+------------+------+-----+---------+----------------+
| Field    | Type       | Null | Key | Default | Extra          |
+----------+------------+------+-----+---------+----------------+
| id       | int(11)    | NO   | PRI | NULL    | auto_increment |
| epiweek  | int(11)    | NO   | MUL | NULL    |                |
| location | varchar(8) | NO   | MUL | NULL    |                |
| value    | float      | NO   |     | NULL    |                |
| std      | float      | NO   |     | NULL    |                |
+----------+------------+------+-----+---------+----------------+
id: unique identifier for each record
epiweek: the epiweek for which (w)ILI is being predicted
location: where the data was collected (nat, hhs, cen, and states)
value: nowcast point prediction
std: nowcast standard deviation

=================
=== Changelog ===
=================

2017-12-15
  + option to run nowcast with quidel
2016-12-13
  + use secrets
2016-11-28
  + store timestamp of last update (this undoubtedly takes the world record for
    ugliest hack ever written)
2016-06-03
  * updated precision matrix estimation `b` parameter
2016-04-27
  * assume that sensors are pre-centered (zero mean)
2016-04-15
  + experimental prefetch/cache for faster nowcasting
2016-04-14
  + gft state signal
2016-04-11
  + work in progess: use cached data for a significant speedup
  * more useful output and more documentation
  * fixed unstable wILI lookup (off by one)
2016-04-09
  + first "real" version (based on various PoC's)
  * stole the `nowcast` table from old/nowcast_20160409.py
"""

# standard library
import argparse
import sys
import time

# third party
import mysql.connector
import numpy as np

# first party
from delphi.nowcast.fusion0 import Fusion
from delphi.epidata.client.delphi_epidata import Epidata
import delphi.operations.secrets as secrets
from delphi.utils.epidate import EpiDate
import delphi.utils.epiweek as flu
from delphi.utils.state_info import StateInfo

# the first epiweek for which we have ground truth ILI in all locations
FIRST_DATA_EPIWEEK = 201040

def get_most_recent_issue():
  # search for FluView issues within the last 10 weeks
  ew2 = EpiDate.today().get_ew()
  ew1 = flu.add_epiweeks(ew2, -9)
  rows = Epidata.check(Epidata.fluview('nat', Epidata.range(ew1, ew2)))
  return max([row['issue'] for row in rows])

def get_all_sensors():
  si = StateInfo()
  all_names = ['gft', 'ght', 'twtr', 'wiki', 'cdc', 'epic', 'sar3', 'arch']
  # all_names = ['gft', 'ght', 'twtr', 'wiki', 'cdc', 'epic', 'sar3', 'arch', 'quid']
  all_loc = si.nat + si.hhs + si.cen[1:] + si.sta
  return (all_names, all_loc)

def nowcast(epiweek, epidata_cache=None):
  si = StateInfo()
  # all sensors and locations
  all_names, all_loc = get_all_sensors()
  # get sensors available on the target week
  rows = Epidata.check(Epidata.sensors(secrets.api.sensors, all_names, all_loc, epiweek))
  present = {}
  for row in rows:
    name, loc, value = row['name'], row['location'], row['value']
    if name not in present:
      present[name] = {}
    if loc not in present[name]:
      present[name][loc] = value
  # get the history of each available sensor (6 sec)
  past = {}
  sensor_locs = set()
  missing = set()
  past_weeks = Epidata.range(FIRST_DATA_EPIWEEK, flu.add_epiweeks(epiweek, -1))
  all_epiweeks = [w for w in flu.range_epiweeks(past_weeks['from'], past_weeks['to'], inclusive=True)]
  num_obs = len(all_epiweeks)
  for name in present.keys():
    past[name] = {}
    for loc in present[name].keys():
      past[name][loc] = {}
      sensor_locs |= set([loc])
      #print(name, loc)
      try:
        if epidata_cache is not None:
          rows = epidata_cache.sensors(name, loc, past_weeks)
        else:
          rows = Epidata.check(Epidata.sensors(secrets.api.sensors, name, loc, past_weeks))
        if len(rows) < 2:
          raise Exception()
        for row in rows:
          past[name][loc][row['epiweek']] = row['value']
      except:
        missing |= set([(name, loc)])
  # remove sensors with zero past data
  for (n, l) in missing:
    del present[n][l]
    if len(present[n]) == 0:
      del present[n]
    del past[n][l]
    if len(past[n]) == 0:
      del past[n]
    #print(n, l, 'is missing')
  # inventory
  all_sensors = []
  for n in all_names:
    for l in si.nat + si.hhs + si.cen + si.sta:
      if n in past and l in past[n]:
        all_sensors.append((n, l))
  #print(all_sensors)
  num_sensors = len(all_sensors)
  # get historical ground truth for each sensor (4 sec)
  truth = {}
  auth = secrets.api.fluview
  for loc in sensor_locs:
    truth[loc] = {}
    if epidata_cache is not None:
      srows = epidata_cache.fluview(loc, past_weeks)
    else:
      srows = Epidata.check(Epidata.fluview(loc, past_weeks, auth=auth))
    sdata = dict([(r['epiweek'], r) for r in srows])
    udata = {}
    try:
      i = past_weeks['to']
      result = Epidata.fluview(loc, past_weeks, issues=i, auth=auth)
      urows = Epidata.check(result)
      udata = dict([(r['epiweek'], r) for r in urows])
    except:
      pass
    rows = []
    for ew in all_epiweeks:
      if ew in udata:
        rows.append(udata[ew])
      else:
        rows.append(sdata[ew])
    for row in rows:
      truth[loc][row['epiweek']] = row['wili']
  # rows are epiweeks, cols are sensors
  X = np.zeros((num_obs, num_sensors)) * np.nan
  for (r, ew) in enumerate(all_epiweeks):
    for (c, (name, loc)) in enumerate(all_sensors):
      if name in past and loc in past[name] and ew in past[name][loc] and loc in truth and ew in truth[loc]:
        X[r, c] = past[name][loc][ew] - truth[loc][ew]
  # sparse precision matrix
  Ri = Fusion.precision(X, mean=np.zeros((1, num_sensors)), b=0.25)
  # prepare for sensor fusion
  inputs = all_sensors
  state = si.sta
  outputs = si.nat + si.hhs + si.cen + si.sta
  num_i, num_s, num_o = len(inputs), len(state), len(outputs)
  # input  (z): [ num_i  x    1   ]
  # state  (x): [ num_s  x    1   ]
  # output (y): [ num_o  x    1   ]
  # S->I   (H): [ num_i  x  num_s ]
  # S->O   (W): [ num_o  x  num_s ]
  z = np.array([present[n][l] for (n, l) in inputs]).reshape((num_i, 1))
  H = np.zeros((num_i, num_s))
  W = np.zeros((num_o, num_s))
  # populate H, given input signals
  for (row, (name, location)) in enumerate(inputs):
    for (col, loc) in enumerate(state):
      if loc in si.within[location]:
        H[row, col] = si.weight[location][loc]
  if np.linalg.matrix_rank(np.dot(H.T, H)) != num_s:
    raise Exception('H is singluar')
  if not np.allclose(np.sum(H, axis=1), 1):
    raise Exception('H rows do not sum to 1')
  # populate W, given output locations
  for (row, location) in enumerate(outputs):
    for (col, loc) in enumerate(state):
      if loc in si.within[location]:
        W[row, col] = si.weight[location][loc]
  if not np.allclose(np.sum(W, axis=1), 1):
    raise Exception('W rows do not sum to 1')
  # sensor fusion
  x, P = Fusion.fuse(z, Ri, H)
  y, S = Fusion.extract(x, P, W)
  print(num_obs, num_i, num_s, num_o)
  pt = [float(v) for v in y.flatten()]
  std = [float(v) for v in np.sqrt(S).flatten()]
  return (outputs, pt, std)

def update(ew1, ew2, test_mode=False, epidata_cache=None):
  # database setup
  u, p = secrets.db.epi
  cnx = mysql.connector.connect(user=u, password=p, database='epidata')
  cur = cnx.cursor()
  sql = """
    INSERT INTO `nowcasts`
      (`epiweek`, `location`, `value`, `std`)
    VALUES
      (%s, %s, %s, %s)
    ON DUPLICATE KEY UPDATE
      value = %s, std = %s
  """

  for ew in flu.range_epiweeks(ew1, ew2, inclusive=True):
    try:
      print(ew)
      locations, values, stds = nowcast(ew, epidata_cache)
      print(' ', locations[0], values[0], stds[0])
      for (l, v, s) in zip(locations, values, stds):
        cur.execute(sql, (ew, l, v, s, v, s))
    except Exception as ex:
      print('failed: ', ew, ex)
      #raise ex
    sys.stdout.flush()

  # the Ugliest Hack Ever Written lies below. turn back now, cannot be unseen.
  # please fix me
  # store the unix timestamp in a meta row representing the last update time
  # the key to this row is `epiweek`=0, `location`='updated'
  # the timestamp is stored across the `value` and `std` fields
  # these are 32-bit floats, so precision is limited (hence, using both fields)
  t = round(time.time())
  a, b = t // 100000, t % 100000
  cur.execute(sql, (0, 'updated', a, b, a, b))
  # /hack

  # database cleanup
  cur.close()
  if test_mode:
    print('test mode - nowcasts not saved')
  else:
    cnx.commit()
  cnx.close()

class Cache:

  def __init__(self, ew2):
    ew1 = FIRST_DATA_EPIWEEK
    print('prefetching %d--%d...' % (ew1, ew2))
    weeks = Epidata.range(ew1, ew2)
    si = StateInfo()
    all_names, all_loc = get_all_sensors()
    self._sensors = {}
    self._fluview = {}
    na, nc = 0, 0
    for loc in all_loc:
      for name in all_names:
        res = Epidata.sensors(secrets.api.sensors, name, loc, weeks)
        if res['result'] == 1:
          for row in res['epidata']:
            n, l = row['name'], row['location']
            if n not in self._sensors:
              self._sensors[n] = {}
            if l not in self._sensors[n]:
              self._sensors[n][l] = []
            self._sensors[n][l].append(row)
            na += 1
      res = Epidata.fluview(loc, weeks, auth=secrets.api.fluview)
      if res['result'] == 1:
        for row in res['epidata']:
          # Make sure that the case of the returned location is consistent with
          # the case used elsewhere; lower for regions and upper for states.
          if row['region'].lower() != loc.lower():
            raise Exception(row['region'], loc)
          row['region'] = loc
          if loc not in self._fluview:
            self._fluview[loc] = []
          self._fluview[loc].append(row)
          nc += 1
    print('done (%d|%d)' % (na, nc))

  def sensors(self, name, loc, past_weeks):
    ew1, ew2 = past_weeks['from'], past_weeks['to']
    if name in self._sensors and loc in self._sensors[name]:
      return [r for r in self._sensors[name][loc] if ew1 <= r['epiweek'] <= ew2]
    else:
      return []

  def fluview(self, loc, past_weeks):
    ew1, ew2 = past_weeks['from'], past_weeks['to']
    if loc in self._fluview:
      return [r for r in self._fluview[loc] if ew1 <= r['epiweek'] <= ew2]
    else:
      return []

if __name__ == '__main__':
  # args and usage
  parser = argparse.ArgumentParser()
  parser.add_argument('-w', '--epiweek', default=None, type=int, help='epiweek override')
  parser.add_argument('-f', '--first', default=None, type=int, help='epiweek range first (requires --last)')
  parser.add_argument('-l', '--last', default=None, type=int, help='epiweek range last (requires --first)')
  parser.add_argument('-t', '--test', default=False, action='store_true', help='test mode, nothing is stored')
  args = parser.parse_args()

  # sanity checks
  if (args.first is None) ^ (args.last is None):
    raise Exception('--first and --last must be used together')
  if args.epiweek is not None and args.first is not None:
    raise Exception('epiweek and range both given')
  if args.first is not None and args.last is not None and args.first > args.last:
    raise Exception('first > last')

  # check input
  for ew in [args.epiweek, args.first, args.last]:
    if ew is not None:
      flu.check_epiweek(ew)

  # figure out which weeks to update
  epidata_cache = None
  if args.first is not None and args.last is not None:
    ew1, ew2 = args.first, args.last
    epidata_cache = Cache(ew2)
  else:
    epiweek = args.epiweek
    if epiweek is None:
      epiweek = flu.add_epiweeks(get_most_recent_issue(), +1)
      print('assuming nowcast for next week: %d' % epiweek)
    ew1, ew2 = epiweek, epiweek

  # make nowcast for each week
  update(ew1, ew2, args.test, epidata_cache)


text:
import csv
import os
import pickle
import random
from pathlib import Path
from collections import deque

import gym
import gym_chrome_dino
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import optim

from network.mlp_pg import MLPTorch
from utils.show_img import show_img

class GameSession:

    def __init__(
            self, session_env
    ):

        self.session_env = session_env

        # Display the processed image on screen using openCV, implemented using python coroutine
        self._display = show_img()
        # Initialize the display coroutine
        self._display.__next__()

    def train(self):

        """
        1. run the game
        2. get the parameters(probs, rewards, entropy)
        3. sum up total rewards
        4. calculate loss
        5. append rewards and entropy in to list
        6. backpropagation

        """
        # model = DQN()
        # model_optim = torch.optim.Adam(model.parameters(), lr=4e-3)
        reward_history = []
        entropy_history = []
        loss = []
        save_iteration = 1
        file_path = './models/rl_pl/reward_history.csv'
        self.session_env.reset()

        s_t, r_t, done, _ = self.session_env.step(0)

        model = MLPTorch(s_t.size, 10, 2)
        model_optim = optim.Adam(model.parameters(), lr=4e-3)

        no_iterations = 1000
        for i in range(no_iterations):
            print('iteration: ', i)
            log_probs, rewards, entropy = self.run_complete_game(model, s_t)
            # print(log_probs)
            # print(rewards)
            total_rewards = np.sum(rewards)
            model_loss = torch.sum(torch.stack(log_probs, 0), 0) * torch.tensor(total_rewards)
            model_loss = -torch.mean(model_loss)
            loss.append(float(model_loss))
            reward_history.append(total_rewards)
            entropy_history.append(entropy)
            model_optim.zero_grad()
            model_loss.backward()
            model_optim.step()

            print(reward_history)
            if no_iterations % save_iteration == 0:
                # open the file in the write mode

                mode = 'a' if os.path.exists(file_path) else 'w+'

                f = open(file_path, mode, newline='')

                # create the csv writer
                writer = csv.writer(f)

                # write a row to the csv file
                # for re in rewardHistory:
                writer.writerow(reward_history)

                # close the file
                f.close()
                print('Done writing into file. Clear Reward History...')
                #reward_history = []

        # print(loss)
        # print(entropyHistory)
        plt.figure(0)
        plt.plot(entropy_history)
        plt.figure(1)
        plt.plot(loss)
        plt.figure(2)
        plt.plot(reward_history)
        # plt.plot(loss)
        print(np.max(rewardHistory))

        plt.show()

    def run_complete_game(self, model, initial_state):
        """
        1. initialize variable
        2. run the game:
            2.1 get state
            2.2 predict action
            2.3 take action
            2.4 input reward into list
            2.5 repeat until die
        """

        self.session_env.reset()
        # t = self.load_obj('time')
        # print('......', t)
        # t = self.load_obj('time')
        # print(t)
        done = False
        rewards = []
        log_probs = []
        entropies = []
        s_t = initial_state
        i = 0
        print('start game...', i)

        while not done:
            if i % 10 == 0:
                s_t = s_t.astype(np.float32)
                probs = model.forward(torch.from_numpy(s_t))
                m = torch.distributions.Categorical(probs)
                # print('Prob:', probs)
                entropy = m.entropy().detach().numpy()
                action = m.sample()
                log_prob = m.log_prob(action)
                # print('Action:', action):q
                x_t, r_t, done, _ = self.session_env.step(action)

                rewards.append(r_t)
                log_probs.append(log_prob)
                entropies.append(entropy)
            i += 1

        print('end game...', i)

        return log_probs, rewards, entropies

def create_required_folders():
    Path("models/rl_pl").mkdir(parents=True, exist_ok=True)

"""
    Main method definition
"""

# Read arguments from command line

if __name__ == '__main__':

    # Guarantee the creation of required folders
    create_required_folders()

    if __name__ == '__main__':

        # Guarantee the creation of required folders

        env = gym.make('ChromeDinoRLPo-v0')
        env.set_score_mode('normal')

        game_session = GameSession(
            session_env=env
        )

        try:
            game_session.train()
        except Exception as e:
            print('Closing environment due to exception')
            env.close()
            raise e

ext: py

reason: import numpy


text:
# Copyright 2022 Maximilien Le Clei.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import deque
import re

import cv2
import gym
from gym import spaces
import numpy as np

def seed(emulator, seed):
    return # emulator.seed(seed) creates unstable behaviour

def set_state(emulator, state):
    emulator.unwrapped.restore_full_state(state)

def get_state(emulator):
    return emulator.unwrapped.clone_full_state()

def get_task_name(task):
    return 'ALE/' + re.sub(r'(?:^|_)([a-z])',
                           lambda x: x.group(1).upper(),
                           task) + '-v5'

def get_score_tasks():

    return ['alien', 'amidar', 'assault', 'asterix', 'asteroids', 'atlantis',
            'bank_heist', 'battle_zone', 'beam_rider', 'berzerk', 'bowling',
            'boxing', 'breakout', 'centipede', 'chopper_command',
            'crazy_climber', 'defender', 'demon_attack', 'double_dunk',
            'enduro', 'fishing_derby', 'freeway', 'frostbite', 'gopher',
            'gravitar', 'hero', 'ice_hockey', 'jamesbond', 'kangaroo',
            'krull', 'kung_fu_master', 'montezuma_revenge', 'ms_pacman',
            'name_this_game', 'phoenix', 'pitfall', 'pong', 'private_eye',
            'qbert', 'riverraid', 'road_runner' ,'robotank', 'seaquest',
            'skiing', 'solaris',  'space_invaders', 'star_gunner', 'surround',
            'tennis', 'time_pilot', 'tutankham', 'up_n_down', 'venture',
            'video_pinball', 'wizard_of_wor', 'yars_revenge', 'zaxxon']

def get_all_potential_imitation_tasks():

    return ['asteroids', 'beam_rider', 'breakout', 'enduro', 'pong', 'qbert',
            'road_runner', 'seaquest', 'space_invaders']

def get_imitation_tasks():

    return ['breakout', 'enduro', 'pong']

def get_action(x):

    # FIRE, UP, DOWN, LEFT, RIGHT

    if x[0] == 0:                    #          ¬ FIRE   =  ∅
        f = ''
    else: # x[0] == 1:               #            FIRE   = FIRE
        f = 'f'

    if x[1] == 0 and x[2] == 0:      # ¬ UP   & ¬ DOWN   =  ∅
        ud = ''
    elif x[1] == 1 and x[2] == 0:    #   UP   & ¬ DOWN   =  UP
        ud = 'u'
    elif x[1] == 0 and x[2] == 1:    # ¬ UP   &   DOWN   =  DOWN
        ud = 'd'
    else: # x[1] == 1 and x[2] == 1: #   UP   &   DOWN   =  ∅
        ud = ''

    if x[3] == 0 and x[4] == 0:      # ¬ LEFT & ¬ RIGHT  =  ∅
        lr = ''
    elif x[3] == 1 and x[4] == 0:    #   LEFT & ¬ RIGHT  =  LEFT
        lr = 'l'
    elif x[3] == 0 and x[4] == 1:    # ¬ LEFT &   RIGHT  =  RIGHT
        lr = 'r'
    else: # x[3] == 1 and x[4] == 1: #   LEFT &   RIGHT  =  ∅
        lr = ''

    if f == '' and lr == '' and ud == '':
        x = 0 # NOOP

    elif f == 'f' and lr == '' and ud == '':
        x = 1 # FIRE

    elif f == '' and lr == '' and ud == 'u':
        x = 2 # UP

    elif f == '' and lr == 'r' and ud == '':
        x = 3 # RIGHT

    elif f == '' and lr == 'l' and ud == '':
        x = 4 # LEFT

    elif f == '' and lr == '' and ud == 'd':
        x = 5 # DOWN

    elif f == '' and lr == 'r' and ud == 'u':
        x = 6 # UPRIGHT

    elif f == '' and lr == 'l' and ud == 'u':
        x = 7 # UPLEFT

    elif f == '' and lr == 'r' and ud == 'd':
        x = 8 # DOWNRIGHT

    elif f == '' and lr == 'l' and ud == 'd':
        x = 9 # DOWNLEFT

    elif f == 'f' and lr == '' and ud == 'u':
        x = 10 # UPFIRE

    elif f == 'f' and lr == 'r' and ud == '':
        x = 11 # RIGHTFIRE

    elif f == 'f' and lr == 'l' and ud == '':
        x = 12 # LEFTFIRE

    elif f == 'f' and lr == '' and ud == 'd':
        x = 13 # DOWNFIRE

    elif f == 'f' and lr == 'r' and ud == 'u':
        x = 14 # UPRIGHTFIRE

    elif f == 'f' and lr == 'l' and ud == 'u':
        x = 15 # UPLEFTFIRE

    elif f == 'f' and lr == 'r' and ud == 'd':
        x = 16 # DOWNRIGHTFIRE

    else: # f == 'f' and lr == 'l' and ud == 'd':
        x = 17 # DOWNLEFTFIRE

    return x

def hide_score(obs, task):

    new_obs = np.array(obs, copy=True)

    if 'breakout' in task:
        new_obs[:10] = 0

    elif 'enduro' in task:
        new_obs[-21:] = 0

    elif 'pong' in task:
        new_obs[:12] = 0

    return new_obs

# From https://bit.ly/2Q4fAih
class NoopResetEnv(gym.Wrapper):
    def __init__(self, env, noop_max=30):
        """Sample initial states by taking random number of no-ops on reset.
        No-op is assumed to be action 0.
        """
        gym.Wrapper.__init__(self, env)
        self.noop_max = noop_max
        self.override_num_noops = None
        self.noop_action = 0
        assert env.unwrapped.get_action_meanings()[0] == 'NOOP'

    def reset(self, **kwargs):
        """ Do no-op action for a number of steps in [1, noop_max]."""
        self.env.reset(**kwargs)
        if self.override_num_noops is not None:
            noops = self.override_num_noops
        else:
            noops = self.unwrapped.np_random.randint(1, self.noop_max + 1)
        assert noops > 0
        obs = None
        for _ in range(noops):
            obs, _, done, _ = self.env.step(self.noop_action)
            if done:
                obs = self.env.reset(**kwargs)
        return obs

    def step(self, ac):
        return self.env.step(ac)

# From https://bit.ly/2Q4fAih
class MaxAndSkipEnv(gym.Wrapper):
    def __init__(self, env, skip=4):
        """Return only every `skip`-th frame"""
        gym.Wrapper.__init__(self, env)
        # most recent raw observations (for max pooling across time steps)
        self._obs_buffer = np.zeros(
            (2,)+env.observation_space.shape, dtype=np.uint8)
        self._skip       = skip

    def step(self, action):
        """Repeat action, sum reward, and max over last observations."""
        total_reward = 0.0
        done = None
        for i in range(self._skip):
            obs, reward, done, info = self.env.step(action)
            if i == self._skip - 2: self._obs_buffer[0] = obs
            if i == self._skip - 1: self._obs_buffer[1] = obs
            total_reward += reward
            if done:
                break
        # Note that the observation on the done=True frame
        # doesn't matter
        max_frame = self._obs_buffer.max(axis=0)

        return max_frame, total_reward, done, info

    def reset(self, **kwargs):
        return self.env.reset(**kwargs)

# From https://bit.ly/2Q4fAih
class FireResetEnv(gym.Wrapper):
    def __init__(self, env):
        """Take action on reset for environments that are fixed until firing.
        """
        gym.Wrapper.__init__(self, env)
        assert env.unwrapped.get_action_meanings()[1] == 'FIRE'
        assert len(env.unwrapped.get_action_meanings()) >= 3

    def reset(self, **kwargs):
        self.env.reset(**kwargs)
        obs, _, done, _ = self.env.step(1)
        if done:
            self.env.reset(**kwargs)
        obs, _, done, _ = self.env.step(2)
        if done:
            self.env.reset(**kwargs)
        return obs

    def step(self, ac):
        return self.env.step(ac)

# From https://bit.ly/2Q4fAih
class WarpFrame(gym.ObservationWrapper):
    def __init__(self,
                 env,
                 width=84,
                 height=84,
                 grayscale=True,
                 dict_space_key=None):
        """
        Warp frames to 84x84 as done in the Nature paper and later work.
        If the environment uses dictionary observations, `dict_space_key`
        can be specified which indicates which observation should be warped.
        """
        super().__init__(env)
        self._width = width
        self._height = height
        self._grayscale = grayscale
        self._key = dict_space_key
        if self._grayscale:
            num_colors = 1
        else:
            num_colors = 3

        new_space = gym.spaces.Box(
            low=0,
            high=255,
            shape=(self._height, self._width, num_colors),
            dtype=np.uint8,
        )
        if self._key is None:
            original_space = self.observation_space
            self.observation_space = new_space
        else:
            original_space = self.observation_space.spaces[self._key]
            self.observation_space.spaces[self._key] = new_space
        assert (original_space.dtype == np.uint8
                and len(original_space.shape) == 3)

    def observation(self, obs):
        if self._key is None:
            frame = obs
        else:
            frame = obs[self._key]

        if self._grayscale:
            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
        frame = cv2.resize(
            frame, (self._width, self._height), interpolation=cv2.INTER_AREA
        )
        if self._grayscale:
            frame = np.expand_dims(frame, -1)

        if self._key is None:
            obs = frame
        else:
            obs = obs.copy()
            obs[self._key] = frame
        return obs

# Modified from https://bit.ly/2Q4fAih
class FrameStack(gym.Wrapper):
    def __init__(self, env, k):
        """Stack k last frames"""
        gym.Wrapper.__init__(self, env)
        self.k = k
        self.frames = deque([], maxlen=k)
        shp = env.observation_space.shape
        self.observation_space = spaces.Box(
            low=0, high=255, shape=(shp[:-1] + (shp[-1] * k,)),
                             dtype=env.observation_space.dtype)

    def reset(self):
        ob = self.env.reset()
        for _ in range(self.k):
            self.frames.append(ob)
        return self._get_ob()

    def step(self, action):
        ob, reward, done, info = self.env.step(action)
        self.frames.append(ob)
        return self._get_ob(), reward, done, info

    def _get_ob(self):
        return np.array(self.frames).squeeze()

class SkiingNoFireEnv(gym.Wrapper):
    """No shooting while skiing"""
    def __init__(self, env):
        gym.Wrapper.__init__(self, env)
        assert env.spec._env_name == 'Skiing'

    def step(self, ac):
        if ac in range(1, 10):
            ac -= 1
        elif ac in range(10, 18):
            ac -= 9
        return self.env.step(ac)

    def reset(self, **kwargs):
        return self.env.reset(**kwargs)

def wrap(env):

    env = NoopResetEnv(env, noop_max=30)

    env = MaxAndSkipEnv(env, skip=4)

    if 'FIRE' in env.unwrapped.get_action_meanings():
        env = FireResetEnv(env)

    if 'Skiing' in env.spec.id:
        env = SkiingNoFireEnv(env)

    env = WarpFrame(env)
    env = FrameStack(env, k=4)

    return env

max_stars_repo_name: MaximilienLC/nevo

max_issues_repo_name: MaximilienLC/nevo

max_forks_repo_name: MaximilienLC/nevo


text:
from operator import itemgetter				# itemgetter used to sort as per list parameter
import copy									# copy used to copy list's column
import numpy as np							# numpy used to calculate std deviation and transposing of row
import sys									# sys used to give the sysmaxsize
def sjf(p):					  # p is a 2D list
    n = len(p)                # n stores the total no. of processes
    complete = 0 			  # no. of completed processes
    t = 0                     # intteger to represent time passed in sec
    minm = sys.maxsize         # minm used to store the shortest remaining time (r[t])
    shortest = 0	          # stores the index of the process with least remaining time
    check = False             # when no process has started execution and t is increementing
    art = [0]*n               # list to store the arrival time
    wt = [0]*n				  # list to store the waiting time
    wt = np.transpose(wt)  	  # wt converted to column for simplicity (it is being added with tmp[] to give tat[]  and temp is a column)
    p[0][1] = float(p[0][1])  # converting first arrival time to float
    idle = 1                  # flag to show when the cpu is idle (1 when cpu is idle )
    for i in range(n):
        if i!=0:                                             # as arriaval time is relative
            p[i][1] = float(p[i][1]) + float(p[i-1][1])      # we are cummulating the arrival time
    p = sorted(p,key=itemgetter(5))		                     # sorting the 2D list p , using priority (indexed - 5)
    x = input("Enter Choice : general or complex  ----- ")   # general case or comples case
    if x == "general" or x == "GENERAL":
        rt = [float(row[2]) for row in p]                    # rt copying the values of column 2 i.e burst time
    else :
        burst   = np.transpose([float(row[2])  for row in p])
        elapsed = np.transpose([float(row[3]) for row in p])
        w_p    =  np.transpose([float(row[4])  for row in p])
        rt = burst + elapsed + w_p                  # remaining time (r[t] ) = burst time + elapsed time + time spent waiting
    temp = copy.copy(rt)                            # temp is used to store the initial value of rt[]
    # Run until all processes gets
    # completed
    while (complete != n):
        # Find process with minimum
        # remaining time among the
        # processes that arrives till the
        # current time
        for j in range(n):
            if p[j][1]<= t and rt[j] < minm and rt[j] > 0:
                minm = rt[j]
                shortest = j
                idle = 0
                check = True

        if check == False:                  # when no process is started , increement t
            t = t + 1
            continue

        # Reduce remaining time by one
        if rt[shortest]>0 :
            rt[shortest] = rt[shortest] - 1
        else:
            idle = 1                          # if rt[shortest] is not decreementing then cpu is idle
        # Update minimum if not idle
        if not idle:
            minm = rt[shortest]
        if (minm == 0):                       # if minm = 0 then rt[shortest ]  = 0  then restore max value in minm
            minm = sys.maxsize
        # If a process gets completely
        # executed
        if (rt[shortest] == 0 and not idle):  # when rt[shortest ] is 0 and cpu is not idle - then a process has just completed
            # Increment complete
            complete = complete + 1
            # Find finish time of current
            # process
            finish_time = t + 1

            # Calculate waiting time
            wt[shortest] = finish_time - temp[shortest] - p[shortest][1]
            if wt[shortest] < 0:
                wt[shortest] = 0
        # Increment time
        t = t + 1
        idle = 0        # reset idle = 0
    tat = wt + temp     # turnaround time = waiting time + run time (run time is the original value of remaining time i.e temp)
    print("Process Id" ,"Waiting time","TurnAroundTime")
    if x == "general" or x == "GENERAL":                        #for general case
        for i in range(n):
            print(p[i][0],"         ",wt[i] ,"          ",tat[i])
    else:                                                         #for complex case
        for i in range(n):
            print(p[i][0],"         ",wt[i] + elapsed[i] + w_p[i] ,"          ",tat[i])
    avg = sum(tat)/n														 #average time of turnaround
    print("Average Completion Time  : ",avg)
    print("The Standard Deviation of completion is ",np.std(np.array(tat)))  # standard deviation of turnaround calculated using numpy

max_stars_repo_path: Python/Scheduling Algorithms/SJF.py

max_issues_repo_path: Python/Scheduling Algorithms/SJF.py

max_forks_repo_path: Python/Scheduling Algorithms/SJF.py


text:
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Copyright (c) Megvii, Inc. and its affiliates.

from loguru import logger

import cv2
import numpy as np
import torch

from yolox.data.data_augment import preproc
from yolox.data.datasets import COCO_CLASSES
from yolox.exp import get_exp
from yolox.utils import fuse_model, get_model_info, postprocess, vis

import argparse
import os
import time
from numpy import random
import cvzone

palette = (2 ** 11 - 1, 2 ** 15 - 1, 2 ** 20 - 1)

# Draws the disconnected Rectangle around the object
def draw_disconnected_rect( img, pt1, pt2, color, thickness):
    width = pt2[0] - pt1[0]
    height = pt2[1] - pt1[1]
    line_width = min(30, width // 3)
    line_height = min(30, height // 3)
    line_length = max(line_width, line_height)
    cv2.line(img, pt1, (pt1[0] + line_length, pt1[1]), color, thickness)
    cv2.line(img, pt1, (pt1[0], pt1[1] + line_length), color, thickness)
    cv2.line(
        img, (pt2[0] - line_length, pt1[1]), (pt2[0], pt1[1]), color, thickness
    )
    cv2.line(
        img, (pt2[0], pt1[1]), (pt2[0], pt1[1] + line_length), color, thickness
    )
    cv2.line(
        img, (pt1[0], pt2[1]), (pt1[0] + line_length, pt2[1]), color, thickness
    )
    cv2.line(
        img, (pt1[0], pt2[1] - line_length), (pt1[0], pt2[1]), color, thickness
    )
    cv2.line(img, pt2, (pt2[0] - line_length, pt2[1]), color, thickness)
    cv2.line(img, (pt2[0], pt2[1] - line_length), pt2, color, thickness)

# Draws the shape of label component
def draw_border(img, pt1, pt2, color, thickness, r, d):
    x1,y1 = pt1
    x2,y2 = pt2
    # Top left
    cv2.line(img, (x1 + r, y1), (x1 + r + d, y1), color, thickness)
    cv2.line(img, (x1, y1 + r), (x1, y1 + r + d), color, thickness)
    cv2.ellipse(img, (x1 + r, y1 + r), (r, r), 180, 0, 90, color, thickness)

    # Top right
    cv2.line(img, (x2 - r, y1), (x2 - r - d, y1), color, thickness)
    cv2.line(img, (x2, y1 + r), (x2, y1 + r + d), color, thickness)
    cv2.ellipse(img, (x2 - r, y1 + r), (r, r), 270, 0, 90, color, thickness)
    # Bottom left
    cv2.line(img, (x1 + r, y2), (x1 + r + d, y2), color, thickness)
    cv2.line(img, (x1, y2 - r), (x1, y2 - r - d), color, thickness)
    cv2.ellipse(img, (x1 + r, y2 - r), (r, r), 90, 0, 90, color, thickness)
    # Bottom right
    cv2.line(img, (x2 - r, y2), (x2 - r - d, y2), color, thickness)
    cv2.line(img, (x2, y2 - r), (x2, y2 - r - d), color, thickness)
    cv2.ellipse(img, (x2 - r, y2 - r), (r, r), 0, 0, 90, color, thickness)

    cv2.rectangle(img, (x1 + r, y1), (x2 - r, y2), color, -1, cv2.LINE_AA)
    cv2.rectangle(img, (x1, y1 + r), (x2, y2 - r - d), color, -1, cv2.LINE_AA)

    cv2.circle(img, (x1 +r, y1+r), 2, color, 12)
    cv2.circle(img, (x2 -r, y1+r), 2, color, 12)
    cv2.circle(img, (x1 +r, y2-r), 2, color, 12)
    cv2.circle(img, (x2 -r, y2-r), 2, color, 12)
    return img

# Implementing the label component
def UI_box2(x, img, color=None,label=None,line_thickness=None, boundingbox = True):
    # Plots one bounding box on image img
    tl = line_thickness or round(0.30 * (img.shape[0] + img.shape[1]) / 2) + 1  # line/font thickness
    color = color or [random.randint(0, 255) for _ in range(3)]
    c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))
    if boundingbox:
        # cv2.rectangle(img, c1, c2, color, 2)
        draw_disconnected_rect(img, c1, c2, color, tl)
    if label:
        tf = max(tl - 1, 1)  # font thickness
        t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]
        img = draw_border(img, (c1[0], c1[1] - t_size[1] -3), (c1[0] + t_size[0], c1[1]+3), color, tf, 8, 2)
        cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)
    # return img

# Converting emojis to Frames
def vid_to_frames(path):
    frames = []
    cap = cv2.VideoCapture(path)
    ret = True
    while ret:
        ret, img = cap.read() # read one frame from the 'capture' object; img is (H, W, C)
        if ret:
            frames.append(img)
    return frames

# Emoji Dict to add more emojis make sure they follow naming conventions as per your labels
# For this to wrok your custom trainig has to be done as mentioned in the lectures

emojidict = dict(
    happy = vid_to_frames('assets/happy.png'),
    sad = vid_to_frames('assets/sad.png'),
    surprised = vid_to_frames('assets/surprised.png')
    )

current = None
count = 0

# Overlaying Image and Emoji
def add_image(img, src2, x, y, ):
    # x=  x+90
    # y = y-10
    w = 80
    h = 80

    initial = img[y:y+h,x:x+w]
    src1 = initial
    src2 = cv2.resize(src2, src1.shape[1::-1])
    u_green = np.array([1, 1, 1])
    l_green = np.array([0, 0, 0])
    mask = cv2.inRange(src2, l_green, u_green)
    res = cv2.bitwise_and(src2, src2, mask = mask)
    f = src2 - res
    f = np.where(f == 0, src1, f)
    img[y:y+h,x:x+w] = f
    return img

def compute_color_for_labels(label):
    """
    Simple function that adds fixed color depending on the class
    """
    if label == 0:
        color = (85,45,255)
    elif label == 1:
        color = (222,82,175)
    elif label == 2:
        color = (0, 204, 255)
    else:
        color = [int((p * (label ** 2 - label + 1)) % 255) for p in palette]
    return tuple(color)

# Combining all the elements and returning the final image
def vis2(img, boxes, scores, cls_ids, conf=0.5, class_names=None):

    global current
    global count

    for i in range(len(boxes)):
        box = boxes[i]
        cls_id = int(cls_ids[i])
        score = scores[i]
        if score < conf:
            continue
        label = class_names[cls_id]

        UI_box2(box, img, color=compute_color_for_labels(cls_id),label=label,line_thickness=2)
        if count == len(emojidict[label]):
            count = 0
        if label == current:
            count +=1
        elif label != current:
            count = 0
        width = int(box[0]) - int(box[0])
        height = int(box[1]) - int(box[1])
        try:
            img = add_image(img, emojidict[label][count] ,int(box[0] + width/2), int(box[1]) )
        except:
            img = add_image(img, emojidict[label][0] ,int(box[0]), int(box[1]) )

        current = label

    return img

IMAGE_EXT = [".jpg", ".jpeg", ".webp", ".bmp", ".png"]

def make_parser():
    parser = argparse.ArgumentParser("YOLOX Demo!")
    parser.add_argument(
        "demo", default="image", help="demo type, eg. image, video and webcam"
    )
    parser.add_argument("-expn", "--experiment-name", type=str, default=None)
    parser.add_argument("-n", "--name", type=str, default=None, help="model name")

    parser.add_argument(
        "--path", default="./assets/dog.jpg", help="path to images or video"
    )
    parser.add_argument("--camid", type=int, default=0, help="webcam demo camera id")
    parser.add_argument(
        "--save_result",
        action="store_true",
        help="whether to save the inference result of image/video",
    )

    # exp file
    parser.add_argument(
        "-f",
        "--exp_file",
        default=None,
        type=str,
        help="pls input your expriment description file",
    )
    parser.add_argument("-c", "--ckpt", default=None, type=str, help="ckpt for eval")
    parser.add_argument(
        "--device",
        default="cpu",
        type=str,
        help="device to run our model, can either be cpu or gpu",
    )
    parser.add_argument("--conf", default=None, type=float, help="test conf")
    parser.add_argument("--nms", default=None, type=float, help="test nms threshold")
    parser.add_argument("--tsize", default=None, type=int, help="test img size")
    parser.add_argument(
        "--fp16",
        dest="fp16",
        default=False,
        action="store_true",
        help="Adopting mix precision evaluating.",
    )
    parser.add_argument(
        "--fuse",
        dest="fuse",
        default=False,
        action="store_true",
        help="Fuse conv and bn for testing.",
    )
    parser.add_argument(
        "--trt",
        dest="trt",
        default=False,
        action="store_true",
        help="Using TensorRT model for testing.",
    )
    return parser

def get_image_list(path):
    image_names = []
    for maindir, subdir, file_name_list in os.walk(path):
        for filename in file_name_list:
            apath = os.path.join(maindir, filename)
            ext = os.path.splitext(apath)[1]
            if ext in IMAGE_EXT:
                image_names.append(apath)
    return image_names

class Predictor(object):
    def __init__(
        self,
        model,
        exp,
        cls_names=COCO_CLASSES,
        trt_file=None,
        decoder=None,
        device="cpu",
    ):
        self.model = model
        self.cls_names = cls_names
        self.decoder = decoder
        self.num_classes = exp.num_classes
        self.confthre = exp.test_conf
        self.nmsthre = exp.nmsthre
        self.test_size = exp.test_size
        self.device = device
        if trt_file is not None:
            from torch2trt import TRTModule

            model_trt = TRTModule()
            model_trt.load_state_dict(torch.load(trt_file))

            x = torch.ones(1, 3, exp.test_size[0], exp.test_size[1]).cuda()
            self.model(x)
            self.model = model_trt
        self.rgb_means = (0.485, 0.456, 0.406)
        self.std = (0.229, 0.224, 0.225)

    def inference(self, img):
        img_info = {"id": 0}
        if isinstance(img, str):
            img_info["file_name"] = os.path.basename(img)
            img = cv2.imread(img)
        else:
            img_info["file_name"] = None

        height, width = img.shape[:2]
        img_info["height"] = height
        img_info["width"] = width
        img_info["raw_img"] = img

        img, ratio = preproc(img, self.test_size, self.rgb_means, self.std)
        img_info["ratio"] = ratio
        img = torch.from_numpy(img).unsqueeze(0)
        if self.device == "gpu":
            img = img.cuda()

        with torch.no_grad():
            t0 = time.time()
            outputs = self.model(img)
            if self.decoder is not None:
                outputs = self.decoder(outputs, dtype=outputs.type())
            outputs = postprocess(
                outputs, self.num_classes, self.confthre, self.nmsthre
            )
            logger.info("Infer time: {:.4f}s".format(time.time() - t0))
        return outputs, img_info

    def visual(self, output, img_info, cls_conf=0.35):
        ratio = img_info["ratio"]
        img = img_info["raw_img"]
        if output is None:
            return img
        output = output.cpu()

        bboxes = output[:, 0:4]

        # preprocessing: resize
        bboxes /= ratio

        cls = output[:, 6]
        scores = output[:, 4] * output[:, 5]

        vis_res = vis(img, bboxes, scores, cls, cls_conf, self.cls_names)

        return vis_res

def image_demo(predictor, vis_folder, path, current_time, save_result):
    if os.path.isdir(path):
        files = get_image_list(path)
    else:
        files = [path]
    files.sort()
    for image_name in files:
        outputs, img_info = predictor.inference(image_name)
        result_image = predictor.visual(outputs[0], img_info, predictor.confthre)
        if save_result:
            save_folder = os.path.join(
                vis_folder, time.strftime("%Y_%m_%d_%H_%M_%S", current_time)
            )
            os.makedirs(save_folder, exist_ok=True)
            save_file_name = os.path.join(save_folder, os.path.basename(image_name))
            logger.info("Saving detection result in {}".format(save_file_name))
            cv2.imwrite(save_file_name, result_image)
        ch = cv2.waitKey(0)
        if ch == 27 or ch == ord("q") or ch == ord("Q"):
            break

def imageflow_demo(predictor, vis_folder, current_time, args):
    cap = cv2.VideoCapture(args.path if args.demo == "video" else args.camid)
    width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)  # float
    height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)  # float
    fps = cap.get(cv2.CAP_PROP_FPS)
    save_folder = os.path.join(
        vis_folder, time.strftime("%Y_%m_%d_%H_%M_%S", current_time)
    )
    os.makedirs(save_folder, exist_ok=True)
    if args.demo == "video":
        save_path = os.path.join(save_folder, args.path.split("/")[-1])
    else:
        save_path = os.path.join(save_folder, "camera.mp4")
    logger.info(f"video save_path is {save_path}")
    vid_writer = cv2.VideoWriter(
        save_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (int(width), int(height))
    )
    while True:
        ret_val, frame = cap.read()
        if ret_val:
            outputs, img_info = predictor.inference(frame)
            result_frame = predictor.visual(outputs[0], img_info, predictor.confthre)
            if args.save_result:
                vid_writer.write(result_frame)
            ch = cv2.waitKey(1)
            if ch == 27 or ch == ord("q") or ch == ord("Q"):
                break
        else:
            break

def main(exp, args):
    if not args.experiment_name:
        args.experiment_name = exp.exp_name

    file_name = os.path.join(exp.output_dir, args.experiment_name)
    os.makedirs(file_name, exist_ok=True)

    if args.save_result:
        vis_folder = os.path.join(file_name, "vis_res")
        os.makedirs(vis_folder, exist_ok=True)

    if args.trt:
        args.device = "gpu"

    logger.info("Args: {}".format(args))

    if args.conf is not None:
        exp.test_conf = args.conf
    if args.nms is not None:
        exp.nmsthre = args.nms
    if args.tsize is not None:
        exp.test_size = (args.tsize, args.tsize)

    model = exp.get_model()
    logger.info("Model Summary: {}".format(get_model_info(model, exp.test_size)))

    if args.device == "gpu":
        model.cuda()
    model.eval()

    if not args.trt:
        if args.ckpt is None:
            ckpt_file = os.path.join(file_name, "best_ckpt.pth.tar")
        else:
            ckpt_file = args.ckpt
        logger.info("loading checkpoint")
        ckpt = torch.load(ckpt_file, map_location="cpu")
        # load the model state dict
        model.load_state_dict(ckpt["model"])
        logger.info("loaded checkpoint done.")

    if args.fuse:
        logger.info("\tFusing model...")
        model = fuse_model(model)

    if args.trt:
        assert not args.fuse, "TensorRT model is not support model fusing!"
        trt_file = os.path.join(file_name, "model_trt.pth")
        assert os.path.exists(
            trt_file
        ), "TensorRT model is not found!\n Run python3 tools/trt.py first!"
        model.head.decode_in_inference = False
        decoder = model.head.decode_outputs
        logger.info("Using TensorRT to inference")
    else:
        trt_file = None
        decoder = None

    predictor = Predictor(model, exp, COCO_CLASSES, trt_file, decoder, args.device)
    current_time = time.localtime()
    if args.demo == "image":
        image_demo(predictor, vis_folder, args.path, current_time, args.save_result)
    elif args.demo == "video" or args.demo == "webcam":
        imageflow_demo(predictor, vis_folder, current_time, args)

if __name__ == "__main__":
    args = make_parser().parse_args()
    exp = get_exp(args.exp_file, args.name)

    main(exp, args)

max_stars_repo_name: augmentedstartups/EmotionDetectionYoloX

max_issues_repo_name: augmentedstartups/EmotionDetectionYoloX

max_forks_repo_name: augmentedstartups/EmotionDetectionYoloX

reason: import numpy,from numpy


text:
from copy import copy

import numpy as np
from scipy.stats import rankdata

from . import smooth_atom, affine_smooth
from .cox_utils import (cox_objective,
                        cox_gradient,
                        cox_hessian)

class cox_loglike(smooth_atom):

    """
    A class for combining the logistic log-likelihood with a general seminorm
    """

    objective_template = r"""\ell^{\text{Cox}}\left(%(var)s\right)"""

    def __init__(self,
                 shape,
                 event_times,
                 censoring,
                 coef=1.,
                 offset=None,
                 quadratic=None,
                 initial=None,
                 case_weights=None):

        smooth_atom.__init__(self,
                             shape,
                             offset=offset,
                             quadratic=quadratic,
                             initial=initial,
                             coef=coef)

        self.data = np.asarray([event_times, censoring]).T

        self._ordering = np.argsort(self.event_times).astype(np.intp)
        self._rankmax = (rankdata(self.event_times, method='max') - 1).astype(np.intp)
        self._rankmin = (rankdata(self.event_times, method='min') - 1).astype(np.intp)

        if case_weights is not None:
            case_weights = np.asarray(case_weights).astype(np.float)
            if not np.all(case_weights >= 0):
                raise ValueError('case_weights should be non-negative')
            self.case_weights = np.asarray(case_weights)
        else:
            self.case_weights = np.ones(self.event_times.shape, np.float)

        # buffers to store results used by C code
        self._G = np.zeros(self.event_times.shape, np.float) # gradient
        self._exp_buffer = np.zeros(self.event_times.shape, np.float) # exp(eta)
        self._exp_accum = np.zeros(self.event_times.shape, np.float) # accum of exp(eta)
        self._expZ_accum = np.zeros(self.event_times.shape, np.float) # accum of Z*exp(eta)
        self._outer_1st = np.zeros(self.event_times.shape, np.float) # for log(W)
        self._outer_2nd = np.zeros(self.event_times.shape, np.float) # used in Hessian

    def smooth_objective(self,
                         natural_param,
                         mode='both',
                         check_feasibility=False,
                         case_weights=None):
        """

        Evaluate the smooth objective, computing its value, gradient or both.

        Parameters
        ----------

        natural_param : ndarray
            The current parameter values.

        mode : str
            One of ['func', 'grad', 'both'].

        check_feasibility : bool
            If True, return `np.inf` when
            point is not feasible, i.e. when `natural_param` is not
            in the domain.

        case_weights : ndarray
            Non-negative case weights

        Returns
        -------

        If `mode` is 'func' returns just the objective value
        at `natural_param`, else if `mode` is 'grad' returns the gradient
        else returns both.
        """

        eta = natural_param # shorthand

        eta = self.apply_offset(eta)

        if case_weights is None:
            case_weights = np.ones_like(natural_param)
        cw = case_weights
        if self.case_weights is not None:
            cw *= self.case_weights

        censoring = self.censoring

        if mode in ['both', 'grad']:
            G = cox_gradient(self._G,
                             eta,
                             self._exp_buffer,
                             self._exp_accum,
                             self._outer_1st,
                             cw,
                             censoring,
                             self._ordering,
                             self._rankmin,
                             self._rankmax,
                             eta.shape[0])

        if mode in ['both', 'func']:
            F = cox_objective(eta,
                              self._exp_buffer,
                              self._exp_accum,
                              self._outer_1st,
                              cw,
                              censoring,
                              self._ordering,
                              self._rankmin,
                              self._rankmax,
                              eta.shape[0])

        if mode == 'both':
            return self.scale(F), self.scale(G)
        elif mode == 'grad':
            return self.scale(G)
        elif mode == 'func':
            return self.scale(F)
        else:
            raise ValueError("mode incorrectly specified")

    # Begin loss API

    def hessian_mult(self,
                     natural_param,
                     right_vector,
                     case_weights=None):
        """
        Evaluate Hessian of the loss at a pair of vectors.

        Parameters
        ----------

        natural_param : ndarray
            Parameters where Hessian will be evaluated.

        left_vec : ndarray
            Vector on the left in Hessian evaluation.

        right_vec : ndarray
            Vector on the left in Hessian evaluation.

        case_weights : ndarray
            Non-negative case weights

        Returns
        -------

        value : float
            Hessian evaluated at this pair of left and right vectors
        """

        if case_weights is None:
            case_weights = np.ones_like(natural_param)
        cw = case_weights
        if self.case_weights is not None:
            cw *= self.case_weights

        eta = natural_param # shorthand
        eta = self.apply_offset(eta)
        censoring = self.censoring

        H = np.zeros(eta.shape, np.float)

        return cox_hessian(H,
                           eta,
                           right_vector,
                           self._exp_buffer,
                           self._exp_accum,
                           self._expZ_accum,
                           self._outer_1st,
                           self._outer_2nd,
                           cw,
                           censoring,
                           self._ordering,
                           self._rankmin,
                           self._rankmax,
                           eta.shape[0])

    def get_data(self):
        return np.array([self.event_times, self.censoring]).T

    def set_data(self, data):
        event_times, censoring = np.asarray(data).T
        self.event_times, self.censoring = (np.asarray(event_times),
                                            np.asarray(censoring).astype(np.intp))

    data = property(get_data, set_data)

    def __copy__(self):
        event_times, censoring = self.event_times, self.censoring
        return cox_loglike(self.shape,
                           copy(event_times),
                           copy(censoring),
                           coef=self.coef,
                           offset=copy(self.offset),
                           quadratic=copy(self.quadratic),
                           initial=copy(self.coefs),
                           case_weights=copy(self.case_weights))

    def subsample(self, case_idx):
        """
        Create a saturated loss using a subsample of the data.
        Makes a copy of the loss and
        multiplies case_weights by the indicator for
        `idx`.

        Parameters
        ----------

        idx : index
            Indices of np.arange(n) to keep.

        Returns
        -------

        subsample_loss : `smooth_atom`
            Loss after discarding all
            cases not in `idx.

        """
        loss_cp = copy(self)
        if loss_cp.case_weights is None:
            case_weights = loss_cp.case_weights = np.ones(self.shape[0])
        else:
            case_weights = loss_cp.case_weights

        idx_bool = np.zeros_like(case_weights, np.bool)
        idx_bool[case_idx] = 1

        case_weights *= idx_bool
        return loss_cp

    def latexify(self, var=None, idx=''):
        # a trick to get latex representation looking right
        # coxph should be written similar to logistic
        # composed with a linear transform (TODO)
        return smooth_atom.latexify(self, var=var, idx=idx)

max_stars_repo_name: vishalbelsare/regreg

max_issues_repo_name: vishalbelsare/regreg

max_forks_repo_name: vishalbelsare/regreg

reason: import numpy,from scipy


text:
from SimPEG.EM.Base import BaseEMProblem, EMPropMap
from SimPEG import Maps
from scipy.constants import mu_0
import numpy as np

class ColeColePropMap(Maps.PropMap):
    """
        Property Map for EM Problems. The electrical conductivity (\\(\\sigma\\)) is the default inversion property, and the default value of the magnetic permeability is that of free space (\\(\\mu = 4\\pi\\times 10^{-7} \\) H/m)
    """

    sigmaInf = Maps.Property("Electrical Conductivity", defaultInvProp=True, propertyLink=('rhoInf', Maps.ReciprocalMap))
    rhoInf   = Maps.Property("Electrical Resistivity", propertyLink=('sigmaInf', Maps.ReciprocalMap))

    eta = Maps.Property("Electrical Conductivity", defaultVal=0)
    tau = Maps.Property("Electrical Conductivity", defaultVal=0.001)
    c   = Maps.Property("Electrical Conductivity", defaultVal=0.5)

    mu  = Maps.Property("Inverse Magnetic Permeability", defaultVal=mu_0, propertyLink=('mui', Maps.ReciprocalMap))
    mui = Maps.Property("Inverse Magnetic Permeability", defaultVal=1./mu_0, propertyLink=('mu', Maps.ReciprocalMap))

class BaseAEMProblem(BaseEMProblem):

    def __init__(self, mesh, **kwargs):
        Problem.BaseProblem.__init__(self, mesh, **kwargs)

    @property
    def MeS(self):
        """
            Current term: MeS = Me(js)
        """
        if getattr(self, '_MeS', None) is None:
            self._MeS = np.zeros((self.mesh.nE,len(self.survey.srcList)))
            for isrc, src in enumerate(self.survey.srcList):
                self._MeS[:,isrc] = src.getMeS(self.mesh, self.MfMui)
        return self._MeS

# class ColeColeMap(IdentityMap):
#     """
#         Takes a vector of [sigmaInf, eta, tau, c]
#         and sets

#             - sigmaInf: Conductivity at infinite frequency (S/m)
#             - eta: Chargeability (V/V)
#             - tau: time constant (sec)
#             - c: Frequency dependency (Dimensionless)
#     """
#     sigmaInf = None
#     eta = None
#     tau = None
#     c = None

#     def __init__(self, mesh, nC=None):
#         self.mesh = mesh
#         self.nC = nC or mesh.nC

#     def _transform(self, m):
#         m = m.reshape((self.mesh.nC, 4), order = "F")
#         self.sigmaInf = m[:,0]
#         self.eta = m[:,1]
#         self.tau = m[:,2]
#         self.c = m[:,3]
#         return Utils.mkvc(m)

#     @property
#     def shape(self):
#         return (self.nP, self.nP)

#     @property
#     def nP(self):
#         """Number of parameters in the model."""
#         return self.nC*4

if __name__ == '__main__':
    from simpegAIP.FD.Utils import ColeColePelton, ColeColeSeigel

    maps = [("sigmaInf", Maps.IdentityMap(nP=5))]
    PM = ColeColePropMap(maps)

    m = PM(np.ones(5))
    print m.mu

    print ColeColePelton(1, m.sigmaInf, m.eta, m.tau, m.c)

max_stars_repo_name: sgkang/simpegAIP

max_issues_repo_name: sgkang/simpegAIP

max_forks_repo_name: sgkang/simpegAIP


text:
import numpy as np
import pytest
from component_vis import factor_tools, postprocessing
from pytest import approx

def test_factor_match_score(rng):
    A = rng.standard_normal((30, 3))
    B = rng.standard_normal((20, 3))

    assert factor_tools.factor_match_score((None, (A, B)), (None, (A, B))) == approx(1)
    assert factor_tools.factor_match_score(
        (None, (A, B)), (None, (2.0 * A, 0.5 * B)),
    ) == approx(1)
    assert factor_tools.factor_match_score(
        (None, (A, B)), (None, (0.5 * A, 0.5 * B)), consider_weights=False
    ) == approx(1)
    assert (
        factor_tools.factor_match_score(
            (None, (A, B)), (None, (0.1 * A, 0.1 * B)), consider_weights=True
        )
        < 0.5
    )

def test_factor_match_score_against_tensortoolbox_three_modes():
    A1 = np.array(
        [
            [-3.17188834, 0.12037674, -0.52441857],
            [0.77006048, -1.46279475, 1.15829157],
            [-1.18581816, 0.48408977, -1.09991184],
            [1.16631215, 1.04360917, 1.36915614],
        ]
    )

    A2 = np.array(
        [
            [-1.30262887, -0.4169056, -0.93705999],
            [0.6507304, -0.35287475, -1.3054718],
            [2.3391102, -1.62249815, -2.44364441],
            [0.03060683, 1.1635985, -0.96374088],
        ]
    )

    B1 = np.array(
        [
            [1.53796467, -0.63422707, 0.45490094],
            [-1.22797090, -1.00880186, -0.68148948],
            [0.99478163, 0.56521819, 1.77120808],
            [0.32201656, 0.38783583, -0.80020482],
            [0.57868839, -1.25683464, 0.45556118],
        ]
    )

    B2 = np.array(
        [
            [2.59245268, -2.27144787, -0.2187818],
            [-0.81465855, -0.35188988, -1.64408907],
            [-0.20094789, 1.10495721, 0.72345472],
            [0.43747334, 1.24612605, -0.94359963],
            [-0.32968311, 1.29134727, -0.73294861],
        ]
    )
    C1 = np.array(
        [
            [0.9931878, -0.95281053, -0.76501819],
            [0.36991261, -1.14379257, -1.25567542],
            [1.19267987, -0.34327929, 0.5883692],
            [-1.76280424, -0.04712039, 0.12633968],
            [-0.21826917, 0.4038183, -0.96475298],
            [0.45657526, 1.82831411, -0.49339521],
        ]
    )

    C2 = np.array(
        [
            [1.09267308, -1.05671575, 0.19105565],
            [-0.54742553, -0.18846809, 0.12229789],
            [0.75474572, 0.71870513, 0.73885282],
            [1.0305555, -0.02279294, 1.57613977],
            [-0.2184589, -0.48151848, 2.38306396],
            [1.85820322, -2.10472093, -0.02799594],
        ]
    )

    assert factor_tools.factor_match_score(
        (None, (A1, B1, C1)), (None, (A2, B2, C2)), consider_weights=False
    ) == pytest.approx(0.027696956568833, rel=1e-8, abs=1e-10)
    assert factor_tools.factor_match_score(
        (None, (A1, B1, C1)), (None, (A2, B2, C2)), consider_weights=True
    ) == pytest.approx(0.019965399034340, rel=1e-8, abs=1e-10)

def test_factor_match_score_against_tensortoolbox_four_modes():
    A1 = np.array(
        [
            [-1.25305822, 0.41480863, 0.67937122, -1.41094398],
            [-0.61751106, -1.07501907, 1.53552703, 1.73289436],
            [-1.10167628, 0.10916708, 0.15440049, 0.71191913],
            [-0.75397339, -0.53277329, -0.31800362, -1.39385172],
        ]
    )
    A2 = np.array(
        [
            [-0.91375871, 1.25690037, 2.02901623, 1.23133122],
            [0.21117567, -0.04855492, -0.33869225, 0.24722038],
            [-1.04180422, 1.43493527, -0.98951756, 1.30852824],
            [-0.88917527, -1.15392866, 1.38868925, 0.51818143],
        ]
    )

    B1 = np.array(
        [
            [0.55524731, 0.18657303, 0.56951858, -0.45689014],
            [-0.45996235, 0.46204667, -1.14346653, -1.17044652],
            [-0.43544214, 2.11642726, -2.53224555, 0.37626344],
            [0.62684971, -1.46490152, -0.72183586, -0.5028133],
            [1.07613603, 0.28500245, 0.59924702, -0.74853447],
        ]
    )

    B2 = np.array(
        [
            [0.11685685, 0.00346736, -1.14081191, 0.03363361],
            [0.06782851, -0.12775747, -0.54999814, -0.73082095],
            [1.99588507, 1.07894308, 0.58721622, 0.56918949],
            [1.40235112, 2.1899298, 0.62988048, -1.3539888],
            [1.8810379, 0.37515909, 1.25157966, 0.20167407],
        ]
    )
    C1 = np.array(
        [
            [-0.40255827, 1.66766184, 0.45864578, 0.74544796],
            [-0.74191489, 0.22902213, -0.03567836, 1.20862804],
            [-2.09713766, -1.62251038, 0.00501596, 0.76610473],
            [0.35926428, 0.55441067, 1.57895469, 0.14168695],
            [-0.02182733, -0.04726342, -0.53751347, 0.4963245],
            [1.26656772, -0.08609847, 1.49541366, 0.23812769],
        ]
    )
    C2 = np.array(
        [
            [-1.35248615, 0.28876374, -0.48831322, 1.30607284],
            [-1.01727243, 1.70950614, -0.6432707, 2.27651227],
            [-0.07297823, -0.0599663, -1.54827796, -0.90780148],
            [2.18938174, 0.6736972, -0.21463208, 0.84751937],
            [-0.80608285, 0.0103886, 1.39146945, -0.09375545],
            [-1.08319207, -0.65801888, -1.46349547, -0.40689636],
        ]
    )
    D1 = np.array(
        [
            [0.16535113, 0.53760458, 1.42702119, 0.82964388],
            [1.05943545, 0.62382289, 0.99240215, -0.61245248],
            [-0.03280214, -0.21058093, -1.23180961, -1.17422252],
            [-1.79503608, -0.61318651, 0.38374694, -0.70810762],
            [-0.96153239, 1.21283919, 0.3295456, -0.40590376],
            [-0.0443726, -0.62473263, -0.9840032, 0.66008108],
            [0.48558508, 1.7994932, 1.23246065, -0.02151854],
        ]
    )
    D2 = np.array(
        [
            [-1.74672431, 0.73026699, -0.25879295, -1.9992047],
            [-0.92546903, 0.13632921, -1.0511824, 2.06251471],
            [-0.36672464, -0.19523376, -1.0453565, -0.31365466],
            [-0.83074898, 0.50937367, -0.18895524, 0.57345223],
            [-1.43804184, 0.02874416, 0.02366292, 0.09117656],
            [-0.40793959, 0.6383231, -0.71778188, -0.62907123],
            [-0.22739631, 0.62798532, 0.87659409, 1.03663001],
        ]
    )

    assert factor_tools.factor_match_score(
        (None, (A1, B1, C1, D1)), (None, (A2, B2, C2, D2)), consider_weights=False
    ) == pytest.approx(0.015619878684950, rel=1e-8, abs=1e-10)
    assert factor_tools.factor_match_score(
        (None, (A1, B1, C1, D1)), (None, (A2, B2, C2, D2)), consider_weights=True
    ) == pytest.approx(0.007867447488467, rel=1e-8, abs=1e-10)

def test_factor_match_score_permutation(rng):
    num_components = 4
    A = rng.standard_normal((30, num_components))
    B = rng.standard_normal((20, num_components))
    permutation = rng.permutation(num_components)

    A_permuted = A[:, permutation]
    B_permuted = B[:, permutation]

    fms, p = factor_tools.factor_match_score(
        (None, (A_permuted, B_permuted)), (None, (A, B)), return_permutation=True
    )
    assert fms == approx(1)
    assert np.allclose(permutation, p)

def test_degeneracy_on_degenerate_components():
    A = np.array([[1, 1, 3], [-1, -1, 0], [1, 1, 2], [2, 2, 6],])
    B = np.array([[4, 4, 6], [-3, -3, 2], [0, 0, -8],])
    C = np.array([[1, -1, 3], [2, -2, 4], [-1, 1, 2], [2, -2, -3],])
    assert factor_tools.degeneracy_score((None, (A, B, C))) == pytest.approx(-1)

def test_degeneracy_on_orthogonal_components(rng):
    A = rng.standard_normal(size=(4, 4))
    A_orthogonal = np.linalg.qr(A)[0]
    B = rng.standard_normal(size=(4, 4))
    B_orthogonal = np.linalg.qr(B)[0]
    assert factor_tools.degeneracy_score((None, (A_orthogonal, B))) == pytest.approx(0)
    assert factor_tools.degeneracy_score((None, (A, B_orthogonal))) == pytest.approx(0)

def test_degeneracy_one_mode(rng):
    A = rng.standard_normal(size=(5, 3))
    min_crossproduct = (factor_tools.normalise(A).T @ factor_tools.normalise(A)).min()
    assert factor_tools.degeneracy_score((None, (A,))) == pytest.approx(
        min_crossproduct
    )

def test_cp_tensors_equals(rng):
    # Generate random decomposition
    A = rng.standard_normal((30, 3))
    B = rng.standard_normal((20, 3))
    C = rng.standard_normal((10, 3))
    w = rng.uniform(size=(3,))

    cp_tensor1 = (w, (A, B, C))
    cp_tensor2 = (w.copy(), (A.copy(), B.copy(), C.copy()))

    # Check that a decomposition is equal to its copy
    assert factor_tools.check_cp_tensors_equals(cp_tensor1, cp_tensor2)

    # Check that the decompositions are not equal if one of the factor matrices differ
    cp_tensor3 = (w.copy(), (A.copy(), B.copy(), rng.standard_normal((15, 3))))
    assert not factor_tools.check_cp_tensors_equals(cp_tensor1, cp_tensor3)

    # Check that two equivalent, but permuted decompositions are not equal
    permutation = [2, 1, 0]
    cp_tensor4 = (
        w[permutation],
        (A[:, permutation], B[:, permutation], C[:, permutation]),
    )
    assert not factor_tools.check_cp_tensors_equals(cp_tensor1, cp_tensor4)

    # Check that two equivalent decompositions with different weight distributions are not equal
    cp_tensor5 = postprocessing.distribute_weights_evenly(cp_tensor1)
    assert not factor_tools.check_cp_tensors_equals(cp_tensor1, cp_tensor5)

    # Check that two completely different CP tensors are not equal
    A2 = rng.standard_normal((30, 3))
    B2 = rng.standard_normal((20, 3))
    C2 = rng.standard_normal((10, 3))
    w2 = rng.uniform(size=(3,))

    cp_tensor6 = (w2, (A2, B2, C2))
    assert not factor_tools.check_cp_tensors_equals(cp_tensor1, cp_tensor6)

def test_cp_tensors_equivalent(rng):
    # Generate random decomposition
    A = rng.standard_normal((30, 3))
    B = rng.standard_normal((20, 3))
    C = rng.standard_normal((10, 3))
    w = rng.uniform(size=(3,))
    cp_tensor1 = (w, (A, B, C))

    # Check that a decomposition is equivalent to its copy
    cp_tensor2 = (w, (A.copy(), B.copy(), C.copy()))
    assert factor_tools.check_cp_tensors_equivalent(cp_tensor1, cp_tensor2)

    # Check that the decompositions are not equivalent if one of the factor matrices differ
    cp_tensor3 = (w, (A.copy(), B.copy(), rng.standard_normal((15, 3))))
    assert not factor_tools.check_cp_tensors_equivalent(cp_tensor1, cp_tensor3)

    # Check that two permuted decompositions are equivalent
    permutation = [2, 1, 0]
    cp_tensor4 = (
        w[permutation],
        (A[:, permutation], B[:, permutation], C[:, permutation]),
    )
    assert factor_tools.check_cp_tensors_equivalent(cp_tensor1, cp_tensor4)

    # Check that two decompositions with different weight distributions are equivalent
    cp_tensor5 = postprocessing.normalise_cp_tensor(cp_tensor1)
    assert factor_tools.check_cp_tensors_equivalent(cp_tensor1, cp_tensor5)

    cp_tensor6 = postprocessing.distribute_weights_evenly(cp_tensor1)
    assert factor_tools.check_cp_tensors_equivalent(cp_tensor1, cp_tensor6)

    cp_tensor7 = postprocessing.distribute_weights_in_one_mode(cp_tensor1, mode=1)
    assert factor_tools.check_cp_tensors_equivalent(cp_tensor1, cp_tensor7)

    # Check that two completely different cp decompositions are not equivalent
    A2 = rng.standard_normal((30, 3))
    B2 = rng.standard_normal((20, 3))
    C2 = rng.standard_normal((10, 3))
    w2 = rng.uniform(size=(3,))
    cp_tensor8 = (w2, (A2, B2, C2))
    assert not factor_tools.check_cp_tensors_equivalent(cp_tensor1, cp_tensor8)

max_stars_repo_name: MarieRoald/component-viz

max_issues_repo_name: MarieRoald/component-viz

max_forks_repo_name: MarieRoald/component-viz


text:
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

from typing import Tuple, List, Any, Union

import detectron2.data.transforms.augmentation as aug
import numpy as np
import torch
from detectron2.config import CfgNode
from detectron2.data.transforms.transform import Transform
from detectron2.structures.boxes import Boxes

from .build import TRANSFORM_OP_REGISTRY, _json_load

def get_box_union(boxes: Boxes):
    """Merge all boxes into a single box"""
    if len(boxes) == 0:
        return boxes
    bt = boxes.tensor
    union_bt = torch.cat(
        (torch.min(bt[:, :2], 0).values, torch.max(bt[:, 2:], 0).values)
    ).reshape(1, -1)
    return Boxes(union_bt)

def get_box_from_mask(mask: torch.Tensor) -> Tuple[int, int, int, int]:
    """Find if there are non-zero elements per row/column first and then find
    min/max position of those elements.
    Only support 2d image (h x w)
    Return (x1, y1, w, h) if bbox found, otherwise None
    """
    assert len(mask.shape) == 2, f"Invalid shape {mask.shape}"
    rows = np.any(mask, axis=1)
    cols = np.any(mask, axis=0)
    if bool(np.any(rows)) is False or bool(np.any(cols)) is False:
        return None
    rmin, rmax = np.where(rows)[0][[0, -1]]
    cmin, cmax = np.where(cols)[0][[0, -1]]

    assert cmax >= cmin, f"cmax={cmax}, cmin={cmin}"
    assert rmax >= rmin, f"rmax={rmax}, rmin={rmin}"

    # x1, y1, w, h
    return cmin, rmin, cmax - cmin + 1, rmax - rmin + 1

def get_min_box_aspect_ratio(
    bbox_xywh: torch.Tensor, target_aspect_ratio: float
) -> torch.Tensor:
    """Get a minimal bbox that matches the target_aspect_ratio
    target_aspect_ratio is representation by w/h
    bbox are represented by pixel coordinates"""
    bbox_xywh = torch.Tensor(bbox_xywh)
    box_w, box_h = bbox_xywh[2:]
    box_ar = float(box_w) / box_h
    if box_ar >= target_aspect_ratio:
        new_w = box_w
        new_h = float(new_w) / target_aspect_ratio
    else:
        new_h = box_h
        new_w = new_h * target_aspect_ratio
    new_wh = torch.Tensor([new_w, new_h])
    bbox_center = bbox_xywh[:2] + bbox_xywh[2:] / 2.0
    new_xy = bbox_center - new_wh / 2.0
    return torch.cat([new_xy, new_wh])

def get_box_center(bbox_xywh: torch.Tensor) -> torch.Tensor:
    """Get the center of the bbox"""
    return torch.Tensor(bbox_xywh[:2]) + torch.Tensor(bbox_xywh[2:]) / 2.0

def get_bbox_xywh_from_center_wh(
    bbox_center: torch.Tensor, bbox_wh: torch.Tensor
) -> torch.Tensor:
    """Get a bbox from bbox center and the width and height"""
    bbox_wh = torch.Tensor(bbox_wh)
    bbox_xy = torch.Tensor(bbox_center) - bbox_wh / 2.0
    return torch.cat([bbox_xy, bbox_wh])

def get_bbox_xyxy_from_xywh(bbox_xywh: torch.Tensor) -> torch.Tensor:
    """Convert the bbox from xywh format to xyxy format
    bbox are represented by pixel coordinates,
    the center of pixels are (x + 0.5, y + 0.5)
    """
    return torch.Tensor(
        [
            bbox_xywh[0],
            bbox_xywh[1],
            bbox_xywh[0] + bbox_xywh[2],
            bbox_xywh[1] + bbox_xywh[3],
        ]
    )

def get_bbox_xywh_from_xyxy(bbox_xyxy: torch.Tensor) -> torch.Tensor:
    """Convert the bbox from xyxy format to xywh format"""
    return torch.Tensor(
        [
            bbox_xyxy[0],
            bbox_xyxy[1],
            bbox_xyxy[2] - bbox_xyxy[0],
            bbox_xyxy[3] - bbox_xyxy[1],
        ]
    )

def to_boxes_from_xywh(bbox_xywh: torch.Tensor) -> torch.Tensor:
    return Boxes(get_bbox_xyxy_from_xywh(bbox_xywh).unsqueeze(0))

def scale_bbox_center(bbox_xywh: torch.Tensor, target_scale: float) -> torch.Tensor:
    """Scale the bbox around the center of the bbox"""
    box_center = get_box_center(bbox_xywh)
    box_wh = torch.Tensor(bbox_xywh[2:]) * target_scale
    return get_bbox_xywh_from_center_wh(box_center, box_wh)

def offset_bbox(bbox_xywh: torch.Tensor, target_offset: float) -> torch.Tensor:
    """Offset the bbox based on target_offset"""
    box_center = get_box_center(bbox_xywh)
    new_center = box_center + torch.Tensor(target_offset)
    return get_bbox_xywh_from_center_wh(new_center, bbox_xywh[2:])

def clip_box_xywh(bbox_xywh: torch.Tensor, image_size_hw: List[int]):
    """Clip the bbox based on image_size_hw"""
    h, w = image_size_hw
    bbox_xyxy = get_bbox_xyxy_from_xywh(bbox_xywh)
    bbox_xyxy[0] = max(bbox_xyxy[0], 0)
    bbox_xyxy[1] = max(bbox_xyxy[1], 0)
    bbox_xyxy[2] = min(bbox_xyxy[2], w)
    bbox_xyxy[3] = min(bbox_xyxy[3], h)
    return get_bbox_xywh_from_xyxy(bbox_xyxy)

class EnlargeBoundingBox(Transform):
    """Enlarge bounding box based on fixed padding or percentage"""

    def __init__(
        self, percentage: float = None, fixed_pad: int = None, box_only: bool = False
    ):
        super().__init__()
        assert percentage is not None or fixed_pad is not None
        assert percentage is None or fixed_pad is None

        if percentage is not None:

            def xfn(x, c):
                return [((a - b) * percentage + a) for a, b in zip(x, c)]

        elif fixed_pad is not None:

            def xfn(x, c):
                return [(np.sign(a - b) * fixed_pad + a) for a, b in zip(x, c)]

        self.box_only = box_only
        self.xfm_fn = xfn

    def apply_image(self, img: torch.Tensor) -> np.ndarray:
        return img

    def apply_box(self, coords: Any) -> Any:
        # Takes boxes_xyxy
        center = (np.array(coords[0, 0:2]) + np.array(coords[0, 2:])) / 2
        new_coords = np.zeros_like(coords)
        new_coords[0, 0:2] = self.xfm_fn(coords[0, 0:2], center)
        new_coords[0, 2:] = self.xfm_fn(coords[0, 2:], center)
        return new_coords

    def apply_coords(self, coords: Any) -> Any:
        if self.box_only:
            return coords
        assert coords.shape[1] == 2, "Supported 2d inputs only"
        center = np.mean(coords, axis=0)
        for index in range(coords.shape[0]):
            coords[index] = self.xfm_fn(coords[index], center)
        return coords

@TRANSFORM_OP_REGISTRY.register()
def EnlargeBoundingBoxOp(
    cfg: CfgNode, arg_str: str, is_train: bool
) -> List[Union[aug.Augmentation, Transform]]:
    kwargs = _json_load(arg_str) if arg_str is not None else {}
    assert isinstance(kwargs, dict)
    return [EnlargeBoundingBox(**kwargs)]


text:
# -*- coding: utf-8 -*-
"""a function used to compute the loss."""

import numpy as np

def calculate_mse(e):
    """Calculate the mse for vector e."""
    return 1/2*np.mean(e**2)

def calculate_mae(e):
    """Calculate the mae for vector e."""
    return np.mean(np.abs(e))

def compute_loss(y, tx, w):
    """Calculate the loss.

    You can calculate the loss using mse or mae.
    """
    e = y - tx.dot(w)
    return calculate_mse(e)
    # return calculate_mae(e)


text:
# import
import numpy as np
import tensor.tensor_product_wrapper as tp
from utils.plotting_utils import montage_array, slice_subplots, classification_plots
import matplotlib.pyplot as plt
import similarity_metrics as sm
import scipy.io
import utils.starplus_utils as starp
from numpy.linalg import norm
from sklearn.model_selection import train_test_split
import transform_matrices as tm

import os
import pickle

# ==================================================================================================================== #
# saving options
save = False
filename = 'local_tsvdm'

# plotting options
plot = False

# ==================================================================================================================== #
# choose product type {'m'}
prod_type = 'm'
ortho = True

# ==================================================================================================================== #
# define projection
def projection(A, U, prod_type='m', M=None, dim_list=(), ortho=True):
    training_coeff = tp.ten_prod(tp.ten_tran(U, prod_type=prod_type), A, prod_type=prod_type, M=M, dim_list=dim_list,
                                 ortho=ortho)
    return tp.ten_prod(U, training_coeff, prod_type=prod_type, M=M, dim_list=dim_list, ortho=ortho)

# ==================================================================================================================== #
#%%
# for reproducibility
np.random.seed(20)

# load data
# subject_ids = ['04847', '04799', '04820', '05675', '05710']
subject_ids = ['04847']
tensor_PS = np.empty([64, 64, 8, 16, 80])
labels = np.empty(0)
for id in subject_ids:
    star_plus_data = scipy.io.loadmat('data/data-starplus-' + id + '-v7.mat')
    tensor_PS_id, labels_id = starp.get_labels(star_plus_data)
    tensor_PS = np.concatenate((tensor_PS, tensor_PS_id), axis=-1)
    labels = np.concatenate((labels, labels_id.reshape(-1)))

    star_plus_data = scipy.io.loadmat('data/data-starplus-' + id + '-v7.mat')
    roi_tensor, _, _ = starp.visualize_roi(star_plus_data)
    roi_tensor = np.repeat()

tensor_PS = tensor_PS[:, :, :, :, 80:] / norm(tensor_PS)

# split
training_data, test_data, training_labels, test_labels = train_test_split(np.moveaxis(tensor_PS, -1, 0), labels,
                                                                          test_size=0.33, random_state=42)
training_data = np.moveaxis(training_data, 0, 1)
test_data = np.moveaxis(test_data, 0, 1)

print('Loaded data...')

# ==================================================================================================================== #
#%%
# choose transformations for M
M = []
for i in range(2, training_data.ndim):
    M.append(tm.haar_matrix(training_data.shape[i], normalized=True))
    # M.append(tm.banded_matrix(training_data.shape[i], 2))

print('Formed M...')
# ==================================================================================================================== #
# form local t-svd
num_classes = len(np.unique(training_labels))
range_k = (2, 3)

U = []
for i in range(num_classes):
    u, _, _, _ = tp.ten_svd(training_data[:, training_labels == i], k=max(range_k), prod_type='m', M=M, ortho=ortho)
    U.append(u)

print('Formed local t-SVDM...')
# ==================================================================================================================== #
# compute results on training and test data
training_error = np.zeros([num_classes, training_data.shape[1], len(range_k)])
test_error = np.zeros([num_classes, test_data.shape[1], len(range_k)])
for j, k in enumerate(range_k):
    for i in range(num_classes):
        training_projection = projection(training_data, U[i][:, :k], prod_type='m', M=M)
        training_error[i, :, j] = sm.frobenius_metric(training_data, training_projection, axis=1)

        test_projection = projection(test_data, U[i][:, :k], prod_type='m', M=M)
        test_error[i, :, j] = sm.frobenius_metric(test_data, test_projection, axis=1)

# classification
training_predicted_classes = np.argmin(training_error, axis=0).reshape(-1, len(range_k))
test_predicted_classes = np.argmin(test_error, axis=0).reshape(-1, len(range_k))

# results
training_num_correct = np.sum(training_predicted_classes == training_labels.reshape(-1, 1), axis=0)
training_accuracy = training_num_correct / training_data.shape[1]

test_num_correct = np.sum(test_predicted_classes == test_labels.reshape(-1, 1), axis=0)
test_accuracy = test_num_correct / test_data.shape[1]

for j, k in enumerate(range_k):
    print('k = %d: train accuracy = %0.2f\ttest accuracy = %0.2f' %
          (k, 100 * training_accuracy[j], 100 * test_accuracy[j]))

if save:
    if not os.path.exists('results/'):
        os.makedirs('results/')
    stored_results = {'training_error' : training_error,
                      'test_error' : test_error,
                      'training_accuracy' : training_accuracy,
                      'test_accuracy' : test_accuracy}
    pickle.dump(stored_results, open('results/' + filename + '-' + prod_type + '.py', 'wb'))

# plot results
if plot:
    plt.figure()
    classification_plots(training_error[:, :, 0], training_labels)
    plt.show()

max_stars_repo_name: elizabethnewman/tensor-fmri

max_issues_repo_name: elizabethnewman/tensor-fmri

max_forks_repo_name: elizabethnewman/tensor-fmri

reason: import numpy,from numpy,import scipy


text:
#!/usr/bin/python3

import csv
import os

import sklearn_crfsuite
from sklearn_crfsuite import metrics
import scipy.stats
from sklearn.metrics import make_scorer
from sklearn.model_selection import RandomizedSearchCV

def evaluate_model(crf, X_test, y_test):
    '''
    Evaluates the trained model on f1-score, recall and precision.
    '''
    labels = list(crf.classes_)
    #labels.remove('O')
    y_pred = crf.predict(X_test)
    sorted_labels = sorted(labels, key=lambda name: (name[1:], name[0]))
    print(metrics.flat_classification_report(
        y_true=y_test, y_pred=y_pred, labels=sorted_labels, digits=3))

def label_transitions(transition_features):
    '''
    Puts all the most likely label transitions in a .csv file
    with the weight of the transitions.
    '''
    directory = os.path.dirname(__file__)
    file = os.path.join(directory, 'output/label_transitions.csv')
    with open(file, 'w') as file:
        csv_writer = csv.writer(file, delimiter='\t', lineterminator='\n')
        csv_writer.writerow(('Label', 'Next label', 'Weight'))
        for (label, next_label), weight in transition_features:
            row = label, next_label, weight
            csv_writer.writerow(row)

def feature_transitions(state_features):
    '''
    Puts all the feature transitions in a .csv file with the weight
    of the specific feature.
    '''
    directory = os.path.dirname(__file__)
    file = os.path.join(directory, 'output/feature_transitions.csv')
    with open(file, 'w') as file:
        csv_writer = csv.writer(file, delimiter='\t', lineterminator='\n')
        csv_writer.writerow(('Attribute', 'Label', 'Weight'))
        for (attribute, label), weight in state_features:
            row = attribute, label, weight
            csv_writer.writerow(row)

def hyperparameter_optimization(X_train, y_train, labels):
    '''
    Finetunes the model by conducting hyperparameter optimization
    with 3-fold cross validation. This means the best parameters
    are chosen for the best scores.
    '''
    crf = sklearn_crfsuite.CRF(
        algorithm = 'lbfgs',
        max_iterations = 100,
        all_possible_transitions = True
        )

    params_space = {
        'c1': scipy.stats.expon(scale=0.5),
        'c2': scipy.stats.expon(scale=0.5),
        }

    f1_scorer = make_scorer(metrics.flat_f1_score, average = 'weighted', labels = labels.remove('O'))

    rs = RandomizedSearchCV(crf, params_space,
                            cv = 3,
                            verbose = 1,
                            n_jobs = -1,
                            n_iter = 50,
                            scoring = f1_scorer)
    rs.fit(X_train, y_train)
    print('best params: ', rs.best_params_)
    print('best CV score: ', rs.best_score_)
    print('model size: ', rs.best_estimator_.size_)

    crf = rs.best_estimator_
    return crf

max_stars_repo_name: robertjvt/CRF-NER-Gronings

max_issues_repo_name: robertjvt/CRF-NER-Gronings

max_forks_repo_name: robertjvt/CRF-NER-Gronings

reason: import scipy


text:
from setuptools import setup, Extension

import numpy

setup(
    name='pyTMHMM',
    version='1.3.2',
    author='Brian Osborne',
    author_email='bosborne@alum.mit.edu',
    description='A transmembrane helix finder.',
    long_description=open('README.md').read(),
    long_description_content_type='text/markdown',
    url='https://github.com/bosborne/pyTMHMM/',
    packages=['pyTMHMM'],
    package_data={'pyTMHMM': ['TMHMM2.0.model']},
    zip_safe=False,
    python_version='>=3.5',
    setup_requires=['setuptools>=18.0', 'numpy>=1.9', 'cython'],
    install_requires=['numpy>=1.9'],
    extras_require={
        'plotting':  ['matplotlib'],
    },
    entry_points={
        'console_scripts': ['pyTMHMM=pyTMHMM.cli:cli'],
    },
    ext_modules=[
        Extension(
            'pyTMHMM.hmm',
            sources=['pyTMHMM/hmm.pyx'],
            include_dirs=[numpy.get_include()],
        ),
    ],
    classifiers=[
        "Programming Language :: Python :: 3",
        "Topic :: Scientific/Engineering :: Bio-Informatics",
        "License :: OSI Approved :: Apache Software License"
    ]
)

max_stars_repo_name: bosborne/pyTMHMM

max_issues_repo_name: bosborne/pyTMHMM

max_forks_repo_name: bosborne/pyTMHMM


text:
from datetime import datetime
import pandas as pd
import os
import util
import modeling.feature_extraction as fe
from modeling import common
from scipy import stats as st
# from sklearn import metrics
import numpy as np
import keras
from sklearn.linear_model import RidgeCV, LogisticRegressionCV
from sklearn.model_selection import ShuffleSplit
import glob

# def f1_score(true, pred):
#     pred = np.where(pred.flatten() > .5, 1, 0)
#     result = metrics.precision_recall_fscore_support(y_true=true,
#                                                      y_pred=pred,
#                                                      average='micro')
#     return result[2]

def correlation(true, pred):
    pred = pred.flatten()
    result = st.pearsonr(true, pred)
    return result[0]

## extra imports to set GPU options
import tensorflow as tf
from keras import backend as k

###################################
# TensorFlow wizardry
config = tf.ConfigProto()

# Don't pre-allocate memory; allocate as-needed
config.gpu_options.allow_growth = True

# Only allow a total of half the GPU memory to be allocated
config.gpu_options.per_process_gpu_memory_fraction = 0.5

# Create a session with the above options specified.
k.tensorflow_backend.set_session(tf.Session(config=config))
###################################

#######		Setting up data		########
train, dev, test = util.train_dev_test_split(util.get_messages())
data = pd.concat([train, test], axis=0)  #excluding dev set from CV
data = data.reset_index(drop=True)
# print(data.head())
########################################

# TODO: this takes a long time --> maybe use shelve?
# https://www.quora.com/What-is-a-fast-efficient-way-to-load-word-embeddings-At-present-a-crude-custom-function-takes-about-3-minutes-to-load-the-largest-GloVe-embedding
embs = common.get_facebook_fasttext_common_crawl(vocab_limit=1_000)

TARGETS = ['empathy', 'distress']

early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss',
                                               min_delta=0,
                                               patience=20,
                                               verbose=0,
                                               mode='auto')

# features_train_centroid=fe.embedding_centroid(train.essay, embs)
# features_train_matrix=fe.embedding_matrix(train.essay, embs, common.TIMESTEPS)

# features_test_centroid=fe.embedding_centroid(test.essay, embs)
# features_test_matrix=fe.embedding_matrix(test.essay, embs, common.TIMESTEPS)

FEATURES_MATRIX = fe.embedding_matrix(data.essay, embs, common.TIMESTEPS)
FEATURES_CENTROID = fe.embedding_centroid(data.essay, embs)

def train_ffn(target):
    print(f'training ffn for {target}')
    rs = ShuffleSplit(n_splits=1, train_size=.8)

    ffn = common.get_ffn(units=[300, 256, 128, 1],
                         dropout_hidden=.5,
                         dropout_embedding=.2,
                         learning_rate=1e-3,
                         problem='regression')

    for i, splits in enumerate(rs.split(data)):
        train, test = splits
        # print(len(train), len(test))

        # for target in TARGETS:
        # print(target)

        labels_train = data[target][train]
        labels_test = data[target][test]

        features_train_centroid = FEATURES_CENTROID[train]
        # features_train_matrix = FEATURES_MATRIX[train]

        features_test_centroid = FEATURES_CENTROID[test]
        # features_test_matrix = FEATURES_MATRIX[test]

        # print(labels_train)
        # print(features_train_matrix)

        ffn.fit(features_train_centroid,
                labels_train,
                epochs=200,
                validation_split=.1,
                batch_size=32,
                callbacks=[early_stopping])

        #	PREDICTION
        pred = ffn.predict(features_test_centroid)

        #	SCORING
        result = correlation(true=labels_test, pred=pred)

        #	RECORD
        print(result)

    timestamp = datetime.now().strftime('%Y-%m-%d_%H.%M')
    print(f'saving ffn for {target}')
    ffn.save(f'./models/ffn_{target}_{timestamp}.h5')

    return ffn

def load_model(target):
    list_of_files = glob.glob(
        f'./models/*{target}*.h5'
    )  # * means all if need specific format then *.csv

    if list_of_files:
        latest_file = max(list_of_files, key=os.path.getctime)

        if latest_file:
            print(f'loading latest saved model: {latest_file}')
            ffn = keras.models.load_model(latest_file)

    else:
        print('training model from scratch')
        ffn = train_ffn(target)

    return ffn

if __name__ == '__main__':

    words = pd.Series([
        "helps",
        "uncommon",
        "blank",
        "iraqis",
        "explored",
        "concentrate",
        "fabrication",
        "leukemia",
        "lakota",
        "healing",
        "inhumane",
        "dehumanizes",
        "mistreating"
    ]).rename('words')
    word_centroids = fe.embedding_centroid(words, embs)

    # mark all out of vocab words
    oov = np.where(~word_centroids.any(axis=1))[0]
    words[oov] = '<OOV> ' + words[oov].astype(str) + ' </OOV>'

    for target in TARGETS:
        ffn = load_model(target)
        pred = ffn.predict(word_centroids)

        pred = pd.DataFrame(pred).join(words)

        print(f'predictions for {target}')
        print(pred)


text:
import numpy as np
import math
from math import sin, cos, tan

class ObjectInThreeD:

    def __init__(self,
                dt
                ):
        '''
        Initialize the object with zero state space and the gravity constant
        '''

        self.dt = dt
        self.X = np.zeros((8, 1))
        self.g = 9.81 * np.array([[0], [0], [-1]])# The opposite of the gravity thus directed opposite of the z-axis.

    def rotation_matrix(self, phi, theta):
        '''
        Generates the rotation matrix for the given roll and pitch angles
        The yaw angle is assumed to be equal to zero.
        '''

        psi = 0.0
        r_x = np.array([[1, 0, 0],
                        [0, np.cos(phi), -np.sin(phi)],
                        [0, np.sin(phi), np.cos(phi)]])

        r_y = np.array([[np.cos(theta), 0, np.sin(theta)],
                        [0, 1, 0],
                        [-np.sin(theta), 0, np.cos(theta)]])

        r_z = np.array([[np.cos(psi), -np.sin(psi), 0],
                        [np.sin(psi), np.cos(psi), 0],
                        [0, 0, 1]])

        r = np.matmul(r_z, np.matmul(r_y, r_x))

        return r

    def get_euler_derivatives(self, phi, theta, p, q):
        '''
        Converts the measured body rates into the Euler angle derivatives using the estimated pitch and roll angles.
        '''
        # TODO: Calculate the Derivatives for the Euler angles in the inertial frame
        euler_rot_mat= np.array([[1, np.sin(phi) * np.tan(theta)],
                                 [0, np.cos(phi)]])

        derivatives_in_bodyframe = np.array([p, q])

        euler_dot = np.matmul(euler_rot_mat, derivatives_in_bodyframe)

        return euler_dot

    def linear_acceleration(self, measured_acceleration, phi, theta):
        '''
        Calculates the true accelerations in the inertial frame of reference based
        on the measured values and the estimated roll and pitch angles.
        '''
        # TODO: Calculate the true acceleration in body frame by removing the gravity component
        a_body_frame = measured_acceleration - np.matmul(self.rotation_matrix(phi, theta), self.g)

        # TODO: Convert the true acceleration back to the inertial frame
        a_inertial_frame = np.matmul(np.linalg.inv(self.rotation_matrix(phi, theta)), a_body_frame)

        return a_inertial_frame

    def dead_reckoning_orientation(self, p, q):
        '''
        Updates the orientation of the drone for the measured body rates
        '''
        # TODO: Update the state vector component of roll and pitch angles
        phi = self.X[4]
        theta = self.X[3]

        euler_dot = self.get_euler_derivatives(phi, theta, p, q)
        self.X[3] = self.X[3] + euler_dot[1] * self.dt         # integrating the roll angle
        self.X[4] = self.X[4] + euler_dot[0] * self.dt         # integrating the roll angle

    def dead_reckoning_position(self, measured_acceleration):
        '''
        Updates the position and the linear velocity in the inertial frame based on the measured accelerations
        '''
        perceived_phi = self.X[4]
        perceived_theta = self.X[3]

        # TODO: Conver the body frame acceleration measurements back to the inertial frame measurements
        a = self.linear_acceleration(measured_acceleration, perceived_phi, perceived_theta)

        # TODO:
        # Use the velocity components of the state vector to update the position components of the state vector
        # Use the linear acceleration in the inertial frame to update the velocity components of the state vector

        self.X[0] = self.X[0] + self.X[5] * self.dt   # x coordianate x= x + \dot{x} * dt
        self.X[1] = self.X[1] + self.X[6] * self.dt   # y coordianate y= y + \dot{y} * dt
        self.X[2] = self.X[2] + self.X[7] * self.dt   # z coordianate z= z + \dot{z} * dt
        self.X[5] = self.X[5] + a[0] * self.dt        # change in velocity along the x is a_x * dt
        self.X[6] = self.X[6] + a[1] * self.dt        # change in velocity along the y is a_y * dt
        self.X[7] = self.X[7] + a[2] * self.dt        # change in velocity along the z is a_z * dt

    def advance_state(self, measured_acceleration, p, q, dt):
        '''
        The function of the advance state updated the position of the drone
        in the inertial frame and then the drone attitude.
        '''

        # Advance the position
        self.dead_reckoning_position(measured_acceleration)

        # Advance the attitude
        self.dead_reckoning_orientation(p, q)

        return self.X

max_stars_repo_name: thhuang/notes-fcnd

max_issues_repo_name: thhuang/notes-fcnd

max_forks_repo_name: thhuang/notes-fcnd