text:
#!/usr/bin/env python
# coding: utf-8

# In[3]:

import urllib.request
import xarray as xr
import numpy as np
from datetime import datetime, date, time, timedelta
import urllib
import requests
import json
import smtplib

# In[4]:

#not useing this right now but consider putting instance here
def get_key(file_name):
    myvars = {}
    with open(file_name) as myfile:
        for line in myfile:
            name, var = line.partition("=")[::2]
            myvars[name.strip()] = str(var).rstrip()
    return myvars

file_key = "C:/Users/gentemann/Google Drive/f_drive/secret_keys/saildrone.txt"
saildrone_key = get_key(file_key)
file_key = "C:/Users/gentemann/Google Drive/f_drive/secret_keys/gmail_login.txt"
email_key = get_key(file_key)

# ## Use restful API to get USV locations

endtime = datetime.today().strftime('%Y-%m-%d')
starttime = (datetime.today() + timedelta(days=-5)).strftime('%Y-%m-%d')
#all_usv = ['1041','1033','1034','1035','1036','1037']
all_usv = ['1034','1035','1036','1037']
#get token
payload={'key': saildrone_key['key'], 'secret':saildrone_key['secret']}
headers={'Content-Type':'application/json', 'Accept':'application/json'}
url = 'https://developer-mission.saildrone.com/v1/auth'
res = requests.post(url, json=payload, headers=headers)
json_data = json.loads(res.text)
names=[]
inum_usv = len(all_usv)
ilen = 500 #len(usv_data['data'])
usv_lats = np.empty((ilen,inum_usv))*np.nan
usv_lons = np.empty((ilen,inum_usv))*np.nan
usv_time = np.empty((ilen,inum_usv))*np.nan
for iusv in range(inum_usv):
    str_usv = all_usv[iusv]
    url = 'https://developer-mission.saildrone.com/v1/timeseries/'+str_usv+'?data_set=vehicle&interval=5&start_date='+starttime+'&end_date='+endtime+'&order_by=desc&limit=500&offset=0'
    payload = {}
    headers = {'Accept':'application/json','authorization':json_data['token']}
    res = requests.get(url, json=payload, headers=headers)
    usv_data = json.loads(res.text)
    #print(usv_data.data)
    for i in range(ilen):
        usv_lons[i,iusv]=usv_data['data'][i]['gps_lng']
        usv_lats[i,iusv]=usv_data['data'][i]['gps_lat']
        usv_time[i,iusv]=usv_data['data'][i]['gps_time']
    names.append(str_usv)
xlons = xr.DataArray(usv_lons,coords={'time':usv_time[:,0],'trajectory':names},dims=('time','trajectory'))
xlats = xr.DataArray(usv_lats,coords={'time':usv_time[:,0],'trajectory':names},dims=('time','trajectory'))
ds_usv = xr.Dataset({'lon': xlons,'lat':xlats})

# In[5]:

msg_body=[]
for i in range(1):
    for j in range(inum_usv):
        dt = datetime.fromtimestamp(ds_usv.time[i].data)
        s = dt.strftime('%Y-%m-%d %H:%M:%S')
        msg = all_usv[j]+" "+s+" lon :{0:5.2f}, lat :{1:5.2f}    ".format(ds_usv.lon[i,j].data,ds_usv.lat[i,j].data)
        msg_body.append(msg)

# In[6]:

try:
    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.ehlo()
except:
    print ('Something went wrong...')
sent_from = email_key['key']
to = ['cgentemann@gmail.com', 'andy.chiodi@noaa.gov']
subject = 'Daily Saildrone Position Update'
body = msg_body
email_text = """\
From: %s
To: %s
Subject: %s

%s
""" % (sent_from, ", ".join(to), subject, body)
try:
    server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
    server.ehlo()
    server.login(email_key['key'], email_key['secret'])
    server.sendmail(sent_from, to, email_text)
    server.close()

    print('Email sent!')
except:
    print('Something went wrong...')

# In[ ]:

max_stars_repo_name: agonmer/Saildrone

max_issues_repo_name: agonmer/Saildrone

max_forks_repo_name: agonmer/Saildrone


text:
from __future__ import print_function
import numpy as np
import pandas as pd
import quandl, math

from sklearn import preprocessing, cross_validation, svm
from sklearn.linear_model import LinearRegression
from time import time

start = time()
'''
`preprocessing` will be used to 'scale' the labels from 0 to 1
'''

df0  = quandl.get('WIKI/GOOG')

print("Downloading the DATA took {0:.2e} seconds".format(time() - start))
start = time()
df  = pd.DataFrame.copy(df0[['Adj. Open', 'Adj. High', 'Adj. Low', 'Adj. Close', 'Adj. Volume']])

df['HL_PCT']    = (df['Adj. High'] - df['Adj. Low'])   / df['Adj. Close'] * 100.0
df['PCT_change']= (df['Adj. Close'] - df['Adj. Open']) / df['Adj. Open']  * 100.0

df  = df[['Adj. Close', 'HL_PCT', 'PCT_change', 'Adj. Volume']]

forecast_col    = 'Adj. Close'
df.fillna(-99999, inplace=True) # fill "Not Applicable" (usually NaNs) with "-99999" in place

percentchange   = 0.01 # predict 1% forward from current data set
forecast_out    = int(math.ceil(percentchange*len(df)))

df['label']     = df[forecast_col].shift(-forecast_out)

df.dropna(inplace=True)

''' Define X and Y '''
# df.dropna(inplace=True)
X   = np.array(df.drop(['label'], 1))   # features are everything except for the `labels`
y   = np.array(df['label'])             # labels are the `labels`

''' Preprocessing can add to processing time
        because you have to rescale testing data and prediction data
        alongside training data for a proper scaling

    With HF Trading, you would definitely skip this step
'''
start = time()

X   = preprocessing.scale(X)
# X   = X[:-forecast_out+1]

X_train, X_test, y_train, y_test    = cross_validation.train_test_split(X, y, test_size=0.2)

print("Setting up the DATA took {0:.2e} seconds".format(time() - start))

''' LinearRegression '''
start = time()

clf = LinearRegression() # default
clf.fit(X_train, y_train)

accuracy    = clf.score(X_test,y_test)*100.0

print("Linear Regression took {0:.2e} seconds, with an accuracy of {1:.2f}%".format(time() - start, accuracy))
start = time()

''' LinearRegression Multiprocessing'''
start = time()

clf = LinearRegression(n_jobs=-1) # multiprocessing
clf.fit(X_train, y_train)

accuracy    = clf.score(X_test,y_test)*100.0

print("Linear Regression took {0:.2e} seconds, with an accuracy of {1:.2f}%".format(time() - start, accuracy))
start = time()

''' SVM Regression '''

clf = svm.SVR()
clf.fit(X_train, y_train)

accuracy    = clf.score(X_test,y_test)*100.0
# print("The accuracy was {0:.2e}".format(accuracy))

print("SVM Linear Kernel took {0:.2e} seconds, with an accuracy of {1:.2f}%".format(time() - start, accuracy))
start = time()

''' SVM Regression Polynomial'''

clf = svm.SVR(kernel='poly')
clf.fit(X_train, y_train)

accuracy    = clf.score(X_test,y_test)*100.0
# print("The accuracy was {0:.2e}".format(accuracy))

print("SVM Poly Kernel took {0:.2e} seconds, with an accuracy of {1:.2f}%".format(time() - start, accuracy))
start = time()

''' SVM Regression RBF (Gaussian)'''

clf = svm.SVR(kernel='rbf')
clf.fit(X_train, y_train)

accuracy    = clf.score(X_test,y_test)*100.0
# print("The accuracy was {0:.2e}".format(accuracy))

print("SVM RBF Kernel took {0:.2e} seconds, with an accuracy of {1:.2f}%".format(time() - start, accuracy))

ext: py

reason: import numpy


text:
import secrets
import string
import numpy as np
from functools import partial

def produce_amount_keys(amount_of_keys, _randint=np.random.randint):
    keys = set()
    pickchar = partial(secrets.choice, string.ascii_lowercase)
    while len(keys) < amount_of_keys:
        keys |= {''.join([pickchar() for _ in range(_randint(1, 10))]) for _ in range(amount_of_keys - len(keys))}
    return keys

def produce_amount_values(amount_of_values, _randint=np.random.randint):
    values = set()
    pickchar = partial(secrets.choice, string.ascii_lowercase)
    while len(values) < amount_of_values:
        values |= {''.join([pickchar() for _ in range(_randint(1, 20))]) for _ in range(amount_of_values - len(values))}
    return values

n = 1000
keys = list(produce_amount_keys(n))
# values = list(produce_amount_values(n))

for i in range(len(keys)):
    # print(keys[i] ," ", values[i])
    print(keys[i])


text:
import time
from pathlib import Path
import os

import numpy as np

from py_diff_pd.env.env_base import EnvBase
from py_diff_pd.common.common import create_folder, ndarray
from py_diff_pd.common.hex_mesh import generate_hex_mesh, get_contact_vertex
from py_diff_pd.common.display import render_hex_mesh, export_gif
from py_diff_pd.core.py_diff_pd_core import HexMesh3d, HexDeformable, StdRealVector
from py_diff_pd.common.project_path import root_path

class CowEnv3d(EnvBase):
    def __init__(self, seed, folder, options):
        EnvBase.__init__(self, folder)

        np.random.seed(seed)
        create_folder(folder, exist_ok=True)

        youngs_modulus = options['youngs_modulus'] if 'youngs_modulus' in options else 1e6
        poissons_ratio = options['poissons_ratio'] if 'poissons_ratio' in options else 0.49
        gait = options['gait'] if 'gait' in options else 'pronk'
        if 'actuator_parameters' in options:
            actuator_parameters = options['actuator_parameters']
        elif gait == 'gallop':
            actuator_parameters = ndarray([np.log10(5) + 5, np.log10(5) + 5])
        else:
            actuator_parameters = ndarray([np.log10(5) + 5,])
        state_force_parameters = options['state_force_parameters'] if 'state_force_parameters' in options else ndarray([0.0, 0.0, -9.81])

        # Mesh parameters.
        la = youngs_modulus * poissons_ratio / ((1 + poissons_ratio) * (1 - 2 * poissons_ratio))
        mu = youngs_modulus / (2 * (1 + poissons_ratio))
        density = 7e2

        bin_file_name = Path(root_path) / 'asset' / 'mesh' / 'spot.bin'
        mesh = HexMesh3d()
        mesh.Initialize(str(bin_file_name))
        cow_size = 0.1
        # Rescale the mesh.
        mesh.Scale(cow_size)
        tmp_bin_file_name = '.tmp.bin'
        mesh.SaveToFile(tmp_bin_file_name)

        dx =  1/15 * cow_size

        deformable = HexDeformable()
        deformable.Initialize(tmp_bin_file_name, density, 'none', youngs_modulus, poissons_ratio)
        os.remove(tmp_bin_file_name)
        # Elasticity.
        deformable.AddPdEnergy('corotated', [2 * mu,], [])
        deformable.AddPdEnergy('volume', [la,], [])
        # State-based forces.
        deformable.AddStateForce('gravity', state_force_parameters)
        # Collisions.
        vertex_num = mesh.NumOfVertices()
        friction_node_idx = []
        for i in range(vertex_num):
            vx, vy, vz = mesh.py_vertex(i)
            if vz < dx / 2:
                friction_node_idx.append(i)
        # Uncomment the code below if you would like to display the contact set for a sanity check:

        # import matplotlib.pyplot as plt
        # from mpl_toolkits.mplot3d import Axes3D
        # fig = plt.figure()
        # ax = fig.add_subplot(111, projection='3d')
        # v = ndarray([ndarray(mesh.py_vertex(idx)) for idx in friction_node_idx])
        # ax.scatter(v[:, 0], v[:, 1], v[:, 2])
        # plt.show()

        # Friction_node_idx = all vertices on the edge.
        deformable.SetFrictionalBoundary('planar', [0.0, 0.0, 1.0, 0.0], friction_node_idx)

        # Actuation: we have 4 legs and each leg has 4 muscles.
        # Convention: F (Front): positive x; R (Rear): negative x;
        #             L (Left): positive y; R (Right): negative y.
        leg_indices = {}
        act_indices = []
        count = 0
        leg_z_length = 4
        body_x_length = 9
        body_y_length = 6
        body_z_length = 4
        foot_x = 2
        foot_y = 2
        x_offset = 1
        y_offset = 1
        element_num = mesh.NumOfElements()
        for i in range(element_num):
            v_idx = ndarray(mesh.py_element(i))
            # Obtain the center of this cell.
            com = 0
            for vi in v_idx:
                com += ndarray(mesh.py_vertex(int(vi)))
            com /= len(v_idx)
            x_idx, y_idx, z_idx = com / dx

            if z_idx >= leg_z_length: continue
            if x_idx >= body_x_length + x_offset or \
                (x_idx >= foot_x + x_offset and x_idx <= body_x_length  + x_offset - foot_x): continue
            if y_idx >= foot_y + y_offset and y_idx <= body_y_length + y_offset - foot_y: continue
            # First, determine which leg the voxel is in.
            #print("i: {}, x: {}, y: {}, z: {}".format(i, x_idx, y_idx, z_idx))
            leg_key = ('F' if x_idx >= body_x_length * 0.5 + x_offset else 'R') \
                + ('L' if y_idx >= body_y_length * 0.5 + y_offset else 'R')
            # Second, determine which muscle this voxel should be in front (F) or back (B).
            if leg_key[0] == 'F':
                x_idx -= body_x_length - foot_x
            muscle_key = ('F' if x_idx >= foot_x/2 + x_offset else 'B') \

            key = leg_key + muscle_key
            if key not in leg_indices:
                leg_indices[key] = [count,]
                count += 1
                act_indices.append(i)
            else:
                leg_indices[key].append(count)
                act_indices.append(i)
                count += 1
        actuator_stiffness = self._actuator_parameter_to_stiffness(actuator_parameters)
        deformable.AddActuation(actuator_stiffness[0], [0.0, 0.0, 1.0], act_indices)

        spine_indices = {}
        if gait == 'gallop':
            act_indices =[]
            for i in range(element_num):
                v_idx = ndarray(mesh.py_element(i))
                # Obtain the center of this cell.
                com = 0
                for vi in v_idx:
                    com += ndarray(mesh.py_vertex(int(vi)))
                com /= len(v_idx)
                x_idx, y_idx, z_idx = com / dx

                if z_idx < leg_z_length: continue
                if z_idx > leg_z_length + body_z_length: continue
                if x_idx < foot_x + x_offset or \
                    x_idx >= body_x_length  + x_offset - foot_x: continue
                # First, determine whether muscle is on Dorsal side (D) or ventral side (V)
                key = 'D' if z_idx > leg_z_length + body_z_length / 2 else 'V'
                if key not in spine_indices:
                    spine_indices[key] = [count, ]
                    count += 1
                    act_indices.append(i)
                else:
                    spine_indices[key].append(count)
                    act_indices.append(i)
                    count += 1

            deformable.AddActuation(actuator_stiffness[1], [1.0, 0.0, 0.0], act_indices)
        # Initial states.
        dofs = deformable.dofs()
        print('Cow element: {:d}, DoFs: {:d}.'.format(element_num, dofs))
        act_dofs = deformable.act_dofs()
        q0 = ndarray(mesh.py_vertices())
        v0 = np.zeros(dofs)
        f_ext = np.zeros(dofs)

        # Data members.
        self._deformable = deformable
        self._q0 = q0
        self._v0 = v0
        self._f_ext = f_ext
        self._youngs_modulus = youngs_modulus
        self._poissons_ratio = poissons_ratio
        self._actuator_parameters = actuator_parameters
        self._state_force_parameters = state_force_parameters
        self._stepwise_loss = False
        self._leg_indices = leg_indices
        self._act_indices = act_indices
        self._spine_indices = spine_indices
        self.__element_num = mesh.NumOfElements()

        scale = 3
        self._spp = options['spp'] if 'spp' in options else 8
        self._camera_pos = (0.5, -1, 0.3)
        self._camera_lookat = (0.5, 0, 0.15)
        self._color = (0.9, 0.9, 0.3)
        self._scale = scale
        self._resolution = (1600, 900)

    def element_num(self):
        return self.__element_num

    def leg_indices(self):
        return self._leg_indices

    def act_indices(self):
        return self._act_indices

    def material_stiffness_differential(self, youngs_modulus, poissons_ratio):
        jac = self._material_jacobian(youngs_modulus, poissons_ratio)
        jac_total = np.zeros((2, 2))
        jac_total[0] = 2 * jac[1]
        jac_total[1] = jac[0]
        return jac_total

    def is_dirichlet_dof(self, dof):
        return False

    def _loss_and_grad(self, q, v):
        # Compute the center of mass.
        com = np.mean(q.reshape((-1, 3)), axis=0)
        # Compute loss.
        z_weight = 0.4
        loss = -com[0] - z_weight * com[2]
        # Compute grad.
        grad_q = np.zeros(q.size)
        vertex_num = int(q.size // 3)
        grad_q[::3] = -1 / vertex_num
        grad_q[2::3] = -1 * z_weight / vertex_num
        grad_v = np.zeros(v.size)
        return loss, grad_q, grad_v


text:
# triads.py - functions for analyzing triads of a graph
#
# Copyright 2015 NetworkX developers.
# Copyright 2011 Reya Group <http://www.reyagroup.com>
# Copyright 2011 Alex Levenson <alex@isnotinvain.com>
# Copyright 2011 Diederik van Liere <diederik.vanliere@rotman.utoronto.ca>
#
# This file is part of NetworkX.
#
# NetworkX is distributed under a BSD license; see LICENSE.txt for more
# information.
"""Functions for analyzing triads of a graph."""
import itertools
from networkx.utils import not_implemented_for
import networkx as nx

__author__ = '\n'.join(['Alex Levenson (alex@isnontinvain.com)',
                        'Diederik van Liere (diederik.vanliere@rotman.utoronto.ca)'])

__all__ = ['triadic_census']

#: The integer codes representing each type of triad.
#:
#: Triads that are the same up to symmetry have the same code.
TRICODES = (1, 2, 2, 3, 2, 4, 6, 8, 2, 6, 5, 7, 3, 8, 7, 11, 2, 6, 4, 8, 5, 9,
            9, 13, 6, 10, 9, 14, 7, 14, 12, 15, 2, 5, 6, 7, 6, 9, 10, 14, 4, 9,
            9, 12, 8, 13, 14, 15, 3, 7, 8, 11, 7, 12, 14, 15, 8, 14, 13, 15,
            11, 15, 15, 16)

#: The names of each type of triad. The order of the elements is
#: important: it corresponds to the tricodes given in :data:`TRICODES`.
TRIAD_NAMES = ('003', '012', '102', '021D', '021U', '021C', '111D', '111U',
               '030T', '030C', '201', '120D', '120U', '120C', '210', '300')

#: A dictionary mapping triad code to triad name.
TRICODE_TO_NAME = {i: TRIAD_NAMES[code - 1] for i, code in enumerate(TRICODES)}

def _tricode(G, v, u, w):
    """Returns the integer code of the given triad.

    This is some fancy magic that comes from Batagelj and Mrvar's paper. It
    treats each edge joining a pair of `v`, `u`, and `w` as a bit in
    the binary representation of an integer.

    """
    combos = ((v, u, 1), (u, v, 2), (v, w, 4), (w, v, 8), (u, w, 16),
              (w, u, 32))
    return sum(x for u, v, x in combos if v in G[u])

@not_implemented_for('undirected')
def triadic_enumeration(G, triads: set([])):
    """Determines the triadic enumeration of a directed graph.

    Parameters
    ----------
    G : digraph
       A NetworkX DiGraph
    triads : set
       A set of list with triads as strings

    Returns
    -------
    enumeration : dict
       Dictionary with triad names as keys and occurrences as tuples.

    Notes
    -----
    This algorithm has complexity $O(m)$ where $m$ is the number of edges in
    the graph.

    See also
    --------
    triad_graph

    References
    ----------
    .. [1] Vladimir Batagelj and Andrej Mrvar, A subquadratic triad census
        algorithm for large sparse networks with small maximum degree,
        University of Ljubljana,
        http://vlado.fmf.uni-lj.si/pub/networks/doc/triads/triads.pdf

    """
    if not isinstance(triads, set):
        return None
    for triad in triads:
        if triad not in TRIAD_NAMES:
            return None
    #: A dictionary mapping triad code to triad name.
    TRICODE_TO_NAME = {i: TRIAD_NAMES[code - 1]
                       for i, code in enumerate(TRICODES)}

    triad_nodes = {name: set([]) for name in triads}
    m = {v: i for i, v in enumerate(G)}
    for v in G:
        vnbrs = set(G.pred[v]) | set(G.succ[v])
        for u in vnbrs:
            if m[u] > m[v]:
                unbrs = set(G.pred[u]) | set(G.succ[u])
                neighbors = (vnbrs | unbrs) - {u, v}
                not_neighbors = set(G.nodes()) - neighbors - {u, v}
                # Find dyadic triads
                if bool(set(['102', '012']).intersection(triads)):
                    for w in not_neighbors:
                        if v in G[u] and u in G[v] and '102' in triads:
                            triad_nodes['102'].add(tuple(sorted([u, v, w])))
                        elif '012' in triads:
                            triad_nodes['012'].add(tuple(sorted([u, v, w])))
                for w in neighbors:
                    code = _tricode(G, v, u, w)
                    current_triad = TRICODE_TO_NAME[code]
                    if current_triad in triads and (
                        m[u] < m[w] or (m[v] < m[w] < m[u] and
                                        v not in G.pred[w] and
                                        v not in G.succ[w])
                    ):
                        triad_nodes[current_triad].add(
                            tuple(sorted([u, v, w]))
                        )
    if '003' in triads:
        # find null triads
        all_tuples = set()
        for s in triad_nodes.values():
            all_tuples = all_tuples.union(s)
        triad_nodes['003'] = set(
            itertools.combinations(G.nodes(), 3)
        ).difference(all_tuples)
    return triad_nodes

def get_missing_edges_for_bt(subgraph: nx.DiGraph):
    """Get missing edge for balance theory."""
    edges = []
    for node in subgraph.nodes:
        for node_neighbor in subgraph.neighbors(node):
            for neighbors_neighbor in subgraph.neighbors(node_neighbor):
                if neighbors_neighbor != node and \
                        not subgraph.has_edge(node, neighbors_neighbor):
                    edges.append((node, neighbors_neighbor))
    return edges

if __name__ == "__main__":
    existing_graph = nx.DiGraph()
    existing_edges = [
        ('A', 'B'), ('B', 'C'),
        ('C', 'A'),
        ('A', 'C'),
        ('C', 'B'),
        ('B', 'A'),
    ]
    existing_graph.add_edges_from(existing_edges, predicted=False)
    get_missing_edges_for_bt(existing_graph)

max_stars_repo_name: ESM-analytics/Tie-Prediction-Tool

max_issues_repo_name: ESM-analytics/Tie-Prediction-Tool

reason: import networkx,from networkx


text:
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.utils.data.sampler import SubsetRandomSampler
import numpy as np
import time
import pdb
import sys

class SuperPointNet(torch.nn.Module):
    """ Pytorch definition of SuperPoint Network. """
    def __init__(self):
        super(SuperPointNet, self).__init__()
        self.relu = torch.nn.ReLU(inplace=True)
        self.pool = torch.nn.MaxPool2d(kernel_size=2, stride=2)
        c1, c2, c3, c4, c5, d1 = 64, 64, 128, 128, 256, 256
        # Shared Encoder.
        self.conv1a = torch.nn.Conv2d(1, c1, kernel_size=3, stride=1, padding=1)
        self.conv1b = torch.nn.Conv2d(c1, c1, kernel_size=3, stride=1, padding=1)
        self.conv2a = torch.nn.Conv2d(c1, c2, kernel_size=3, stride=1, padding=1)
        self.conv2b = torch.nn.Conv2d(c2, c2, kernel_size=3, stride=1, padding=1)
        self.conv3a = torch.nn.Conv2d(c2, c3, kernel_size=3, stride=1, padding=1)
        self.conv3b = torch.nn.Conv2d(c3, c3, kernel_size=3, stride=1, padding=1)
        self.conv4a = torch.nn.Conv2d(c3, c4, kernel_size=3, stride=1, padding=1)
        self.conv4b = torch.nn.Conv2d(c4, c4, kernel_size=3, stride=1, padding=1)
        # Detector Head.
        self.convPa = torch.nn.Conv2d(c4, c5, kernel_size=3, stride=1, padding=1)
        self.convPb = torch.nn.Conv2d(c5, 65, kernel_size=1, stride=1, padding=0)
        # Descriptor Head.
        self.convDa = torch.nn.Conv2d(c4, c5, kernel_size=3, stride=1, padding=1)
        self.convDb = torch.nn.Conv2d(c5, d1, kernel_size=1, stride=1, padding=0)

    def forward(self, x):
        """ Forward pass that jointly computes unprocessed point and descriptor
        tensors.
        Input
          x: Image pytorch tensor shaped N x 1 x H x W.
        Output
          semi: Output point pytorch tensor shaped N x 65 x H/8 x W/8.
          desc: Output descriptor pytorch tensor shaped N x 256 x H/8 x W/8.
        """
        # Shared Encoder.
        x = self.relu(self.conv1a(x))
        x = self.relu(self.conv1b(x))
        x = self.pool(x)
        x = self.relu(self.conv2a(x))
        x = self.relu(self.conv2b(x))
        x = self.pool(x)
        x = self.relu(self.conv3a(x))
        x = self.relu(self.conv3b(x))
        x = self.pool(x)
        x = self.relu(self.conv4a(x))
        x = self.relu(self.conv4b(x))
        # Detector Head.
        cPa = self.relu(self.convPa(x))
        semi = self.convPb(cPa)
        # Descriptor Head.
        cDa = self.relu(self.convDa(x))
        desc = self.convDb(cDa)
        dn = torch.norm(desc, p=2, dim=1) # Compute the norm.
        desc = desc.div(torch.unsqueeze(dn, 1)) # Divide by norm to normalize.
        return semi, desc

'''

class VGGBlock(nn.Module):

    def __init__(self,inch,outch):
        super(VGGBlock,self).__init__()

        self.conv = nn.Conv2d(inch, outch, 3, stride=1, padding=0, bias=True)
        self.bn = nn.BatchNorm2d(outch)
        self.relu = nn.ReLU()

    def forward(self,inp):
        inp = self.conv(inp)
        inp = self.bn(inp)
        inp = self.relu(inp)

        return inp

class VGGBackBone(nn.Module):

    def __init(self,in_size):
        super(BaseModel,self).__init__()

        self.vgg1 = VGGBlock(in_size,64)
        self.pool1 = nn.MaxPool2d(2, stride=None, padding=0, dilation=1, return_indices=False, ceil_mode=False)

        self.vgg2 = VGGBlock(64,64)
        self.pool2 = nn.MaxPool2d(2)

        self.vgg3 = VGGBlock(64,128)
        self.pool3 = nn.MaxPool2d(2)

        self.vgg4 = VGGBlock(128,128)
        self.vgg5 = VGGBlock(128,128)

    def forward(self,inp):

        inp = self.vgg1(inp)
        inp = self.pool1(inp)

        inp = self.vgg2(inp)
        inp = self.pool2(inp)

        inp = self.vgg3(inp)
        inp = self.pool3(inp)

        inp = self.vgg4(inp)
        inp = self.vgg5(inp)

        return inp

class BaseModel(nn.Module):

    def __init__(self):

'''

max_stars_repo_name: srini-cmu/specialized-superpoint

max_issues_repo_name: srini-cmu/specialized-superpoint

max_forks_repo_name: srini-cmu/specialized-superpoint


text:
# Proof-of-concept
import cv2
import base64
import sys
from .constants import *
from .emotion_recognition import EmotionRecognition
import numpy as np
from collections import OrderedDict

cascade_classifier = cv2.CascadeClassifier(CASC_PATH)

def brighten(data,b):
     datab = data * b
     return datab

def format_image(image):
  if len(image.shape) > 2 and image.shape[2] == 3:
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
  else:
    image = cv2.imdecode(image, cv2.CV_LOAD_IMAGE_GRAYSCALE)

  faces = cascade_classifier.detectMultiScale(
      image,
      scaleFactor = 1.3,
      minNeighbors = 5
  )

  print(faces)
  # None is we don't found an image
  if not len(faces) > 0:
    return None
  max_area_face = faces[0]
  for face in faces:
    if face[2] * face[3] > max_area_face[2] * max_area_face[3]:
      max_area_face = face
  # Chop image to face
  face = max_area_face
  image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])]
  # Resize image to network size
  try:
    image = cv2.resize(image, (SIZE_FACE, SIZE_FACE), interpolation = cv2.INTER_CUBIC) / 255.
  except Exception:
    print("[+] Problem during resize")
    return None
  # cv2.imshow("Lol", image)
  # cv2.waitKey(0)
  return image

def data_uri_to_cv2_img(uri):
  encoded_data = uri.split(',')[1]
  nparr = np.fromstring(base64.b64decode(encoded_data), np.uint8)
  img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
  return img

# Load Model
network = EmotionRecognition()
network.build_network()

def predict_face(b):
  img = data_uri_to_cv2_img(b)
  d = OrderedDict()
  res = network.predict(format_image(img)).tolist()[0]

  for i in range(len(res)):
    d[EMOTIONS[i]] = res[i]
  return d


text:
#!/usr/bin/env python3

#####  separate functions for simulated vs real data
#####  added additional argument to indicate whether dataset is a real or simulated dataset
#####  can specify if only want to run subset of rows in the design file

# Load packages
import argparse, csv, os, sys
import numpy as np
import pandas as pd
import re

def getOptions():
    parser = argparse.ArgumentParser(description='Sum ase counts across tech reps(by sampleID) using the provided in design file')
    parser.add_argument('-design','--design',dest='design', action='store', required=True, help='Alignment Design File containing fastq fqNames and sampleIDs [Required]')
    parser.add_argument('-sim','--sim',dest='sim', action='store', required=True, help='Assign to True if this is a simulation dataset, assign to False otherwise')
    parser.add_argument('-bed','--bed',dest='bed', action='store', required=True, help='File name (give the full path) of the bed file [Required]')
    parser.add_argument('-collection_identifiers','--collection_identifiers',dest='collection_identifiers', action='store', required=True, help='Input original names [Required]')
    parser.add_argument('-collection_filenames','--collection_filenames',dest='collection_filenames', action='store', required=True, help='Input galaxy names [Required]\
')
    parser.add_argument('-begin','--begin',dest='begin', action='store', required=False, help='Start point in design file [Optional]')
    parser.add_argument('-end','--end', dest='end', action='store', required=False, help='End point in design file [Optional]')
    parser.add_argument('-designdir','--designdir', dest='designdir', action='store', required=True, help='Output design file name with full path in galaxy [Required]' )
    parser.add_argument('-out','--out', dest='out', action='store', required=True, help='Output directory for summed count table files [Required]')
    args = parser.parse_args()
    return(args)

##check that column names in count tables match expected
def headers_check(single_file, file_identifier):
    df = pd.read_table(single_file)
    columns = ['FEATURE_ID','BOTH_EXACT', 'BOTH_INEXACT_EQUAL', 'SAM_A_ONLY_EXACT', 'SAM_B_ONLY_EXACT', 'SAM_A_EXACT_SAM_B_INEXACT', 'SAM_B_EXACT_SAM_A_INEXACT', 'SAM_A_ONLY_SINGLE_INEXACT', 'SAM_B_ONLY_SINGLE_INEXACT', 'SAM_A_INEXACT_BETTER', 'SAM_B_INEXACT_BETTER']
    headers = list(df)
 #   print(headers)
    if headers != columns:
        print("ERROR: column headers in file " + file_identifier + " are incorrectly named")
        sys.exit()

def sum_counts_sim(input_dict,df_design,out,designdir):
    ##groupby sampleID; this allows us to select the correct count table files to sum together
    df_grouped = df_design.groupby('sampleID').agg(lambda x: x.tolist())

    ## count number of unique sampleIDs
    sampleID_count = len(df_grouped.index)
#    print("Number of Sample IDs:")
#    print(sampleID_count)

    ## Create list of filenames, iterate, put into a df
    for sampleID, filelist in df_grouped['fqName'].iteritems(): ##iterate grouped df using sampleID as index
        ddict = {}
        dfsum = pd.DataFrame(index=range(0),columns=['FEATURE_ID','BOTH_EXACT', 'BOTH_INEXACT_EQUAL', 'SAM_A_ONLY_EXACT', 'SAM_B_ONLY_EXACT', 'SAM_A_EXACT_SAM_B_INEXACT', 'SAM_B_EXACT_SAM_A_INEXACT', 'SAM_A_ONLY_SINGLE_INEXACT', 'SAM_B_ONLY_SINGLE_INEXACT', 'SAM_A_INEXACT_BETTER', 'SAM_B_INEXACT_BETTER'])
        dfsum = dfsum.set_index('FEATURE_ID')

        ## Iterate through the list of filenames in the filenames column
        if_found = 0
        for fqName in filelist:
            filename = fqName
            for i in input_dict.keys():
                if filename in i:
                    filename = i
                    if_found = 1
            if(if_found ==0):
                report = "Error: Cannot find file" + i  + "in the input collection!"
                raise ValueError(report)
            headers_check(input_dict[filename], filename)
            ddict[filename] = pd.read_table(input_dict[filename]) ##we create a dictionary to store each count table file as a dataframe with their filename as the key
            ddict[filename] = ddict[filename].set_index('FEATURE_ID')
            dfsum = dfsum.add(ddict[filename], fill_value =  0)

        outfile = out +'/' + sampleID  ##create outfile for summed matrix
        dfsum.to_csv(outfile, sep='\t')

    df_design['sample'] = df_design['sampleID'].str.rsplit('_', 1).str[0]
   # print(df_design)
    df_drop = df_design.drop(['fqName','readLength', 'fqExtension', 'techRep'], axis=1)
    df_unique = df_drop.drop_duplicates()
    df_unique.to_csv(os.path.join(designdir), index=False, sep='\t')

def sum_counts_data(input_dict,df_design,feature_lengths,out,designdir):

    ##groupby sampleID; this allows us to select the correct count table files to sum together
    df_grouped = df_design.groupby('sampleID').agg(lambda x: x.tolist())

    ## count number of unique sampleIDs
    sampleID_count = len(df_grouped.index)
  #  print("Number of Sample IDs:")
  #  print(sampleID_count)

    row = -1 ##this variable keeps track of which row in the design file (or subset design file) the filename would have been located in; we use this variable to extract the corresponding
                  ##read length from the design file
    ## Create list of filenames, iterate, put into a df
    for sampleID, filelist in df_grouped['fqName'].iteritems(): ##iterate grouped df using sampleID as index
        ddict = {}
        dfsum = pd.DataFrame(index=range(0),columns=['FEATURE_ID','BOTH_EXACT', 'BOTH_INEXACT_EQUAL', 'SAM_A_ONLY_EXACT', 'SAM_B_ONLY_EXACT', 'SAM_A_EXACT_SAM_B_INEXACT', 'SAM_B_EXACT_SAM_A_INEXACT', 'SAM_A_ONLY_SINGLE_INEXACT', 'SAM_B_ONLY_SINGLE_INEXACT', 'SAM_A_INEXACT_BETTER', 'SAM_B_INEXACT_BETTER'])
        dfsum = dfsum.set_index('FEATURE_ID')

        ## Iterate through the list of filenames in the filenames column
        for fqName in filelist:
            row += 1
            filename = fqName
            filename = fqName
            for i in input_dict.keys():
                if filename in i:
                    filename = i
                    if_found = 1
            if(if_found ==0):
                report = "Error: Cannot find file" + i  + "in the input collection!"
                raise ValueError(report)
            headers_check(input_dict[filename], filename)
            ddict[filename] = pd.read_table(input_dict[filename]) ##we create a dictionary to store each count table file as a dataframe with their filename as the key

            ##create columns for APN calculations
            ddict[filename]['total_reads_counted'] = ddict[filename]['BOTH_EXACT'] + ddict[filename]['BOTH_INEXACT_EQUAL'] + ddict[filename]['SAM_A_ONLY_EXACT'] + ddict[filename]['SAM_A_ONLY_SINGLE_INEXACT'] + ddict[filename]['SAM_A_EXACT_SAM_B_INEXACT'] + ddict[filename]['SAM_A_INEXACT_BETTER'] + ddict[filename]['SAM_B_ONLY_EXACT'] + ddict[filename]['SAM_B_ONLY_SINGLE_INEXACT'] + ddict[filename]['SAM_B_EXACT_SAM_A_INEXACT'] + ddict[filename]['SAM_B_INEXACT_BETTER']
            ddict[filename]['both_total'] = ddict[filename]['BOTH_EXACT'] + ddict[filename]['BOTH_INEXACT_EQUAL']
            readLength = df_design.iloc[row].loc['readLength'] ##get read length from design
            ddict[filename]['both x RL'] = ddict[filename]['both_total']*readLength

            ddict[filename]['total x RL'] = ddict[filename]['total_reads_counted']*int(readLength)
            ddict[filename] = ddict[filename].set_index('FEATURE_ID')
            dfsum = dfsum.add(ddict[filename], fill_value =  0)

            dfsum['APN_both'] = dfsum['both x RL']/feature_lengths
            dfsum['APN_total_reads'] = dfsum['total x RL']/feature_lengths
            dfout = dfsum.drop(['both x RL', 'total x RL', 'both_total', 'total_reads_counted'], axis=1) ##drop extra columns we no longer need

        outfile = out +'/' + sampleID  ##create outfile for summed matrix
        dfout.to_csv(outfile, sep = '\t')

    ##create new design file with g1, g2, sampleID columns and add new empty columns for comparison 1 and comparison 2; new design file is output to the directory given in the -out argument
    ###if a subset of the design is given, the new design file will be created from that subset of the original design
    df_design['comparate'] = df_design['sampleID'].str.rsplit('_', 1).str[0]
#    print(df_design)
    df_drop = df_design.drop(['fqName','readLength', 'fqExtension', 'techRep'], axis=1)
    df_unique = df_drop.drop_duplicates()
    df_unique.to_csv(os.path.join(designdir), index=False, sep='\t')

def main():
    args = getOptions()
    pattern = re.compile(r'(?<=\').*(?=\')')
    identifiers = [pattern.search(i).group() for i in args.collection_identifiers.split(",")]
    filenames = [i.strip() for i in args.collection_filenames.split(",")]
    input_dict = dict(zip(identifiers, filenames))
    ## Read in design file as dataframe
    df_design = pd.read_table(args.design)
 #   print(df_design)
    ##check for optional begin/end argument; subset dataframe accordingly if arguments given
    if args.begin is not "" and args.end is not "":
        ## reindex so pulling 'correct' rows
        begin = int(args.begin) -2
        end = int(args.end) -1
        df_design = df_design[begin:end]
        ##print what subset of the dataframe is being used
        begin = int(args.begin)
  #      print("begining on row " + str(begin))
        end = int(args.end)
  #      print("ending on row " + str(end))
    if args.sim == 'True':
        sum_counts_sim(input_dict,df_design,args.out,args.designdir)
    elif args.sim== 'False':
        ##read in bed file and get feature lengths for APN calculations
        df_bed = pd.read_table(args.bed, names = ['col', 'start', 'end', 'FEATURE_ID'])
        df_bed =df_bed.set_index(['FEATURE_ID'])
        lengths = df_bed.end - df_bed.start
        feature_lengths = lengths.tolist()
    #    print(len(feature_lengths))
        sum_counts_data(input_dict, df_design, feature_lengths,args.out,args.designdir)

if __name__=='__main__':
    main()


text:
# This is open-source software licensed under a BSD license.
# Please see the file LICENSE.txt for details.
#
import numpy as np

## from PIL import Image, ImageFilter
from scipy.ndimage.filters import median_filter
import cv2

from ginga import trcalc
from ginga.gw import Widgets

from .base import Stage

class Sharpen(Stage):
    """
    The Sharpen stage will perform an unsharp mask on a 2D image.

    """

    _stagename = 'sharpen'

    def __init__(self):
        super(Sharpen, self).__init__()

        self.radius = 2
        self.strength = 1.0

    def build_gui(self, container):
        fr = Widgets.Frame("Sharpen")

        captions = (('Radius:', 'label', 'radius', 'entryset'),
                    ('Strength:', 'label', 'strength', 'entryset'),
                    )
        w, b = Widgets.build_info(captions, orientation='vertical')

        b.radius.set_tooltip("Set the radius of the sharpening in pixels")
        b.radius.set_text(str(self.radius))
        b.radius.add_callback('activated', self.set_radius_cb)

        b.strength.set_tooltip("Set the strength of the sharpening")
        b.strength.set_text(str(self.strength))
        b.strength.add_callback('activated', self.set_strength_cb)

        self.w.update(b)
        fr.set_widget(w)

        container.set_widget(fr)

    def set_radius_cb(self, widget):
        self.radius = int(widget.get_text().strip())
        self.pipeline.run_from(self)

    def set_strength_cb(self, widget):
        self.strength = float(widget.get_text().strip())
        self.pipeline.run_from(self)

    def run(self, prev_stage):
        data = self.pipeline.get_data(prev_stage)
        self.verify_2d(data)

        if self._bypass:
            self.pipeline.send(res_np=data)
            return

        ## img = Image.fromarray(data)

        ## pct = int(round(self.strength * 100))
        ## sharp_img = img.filter(ImageFilter.UnsharpMask(radius=self.radius,
        ##                                                percent=pct))

        ## res_np = np.array(sharp_img, dtype=data.dtype)

        res_np = unsharpen(data, self.radius, self.strength)

        self.pipeline.send(res_np=res_np)

def unsharp(imarr, sigma, strength, minmax=None):
    """
    Credit: Unsharp masking with Python and OpenCV
    https://www.idtools.com.au/unsharp-masking-python-opencv/
    """
    if minmax is None:
        minmax = trcalc.get_minmax_dtype(imarr.dtype)

    # Median filtering
    image_mf = median_filter(imarr, sigma)

    # Calculate the Laplacian
    lap = cv2.Laplacian(image_mf, cv2.CV_64F)

    # Calculate the sharpened image
    sharp = (imarr - strength * lap).clip(minmax[0], minmax[1])
    return sharp

def unsharpen(imarr, sigma, strength):
    res = np.zeros_like(imarr)
    if len(imarr.shape) < 3:
        # monochrome image
        return unsharp(imarr, sigma, strength)

    # RGB image
    for i in range(imarr.shape[2]):
        res[:, :, i] = unsharp(imarr[:, :, i], sigma, strength)

    return res

max_stars_repo_name: kyraikeda/ginga

max_issues_repo_name: kyraikeda/ginga

max_forks_repo_name: kyraikeda/ginga

reason: import numpy,from scipy


text:
import numpy as np

def distances_to_point(lat_point, lon_point, lats, lons):
    """Method to calculate distances between a project and an array of lats and lons

    :Parameters:

    lat_project: float
        Project latitude

    lon_project: float
        Project longitude

    lats: np.array
        Latitudes from which to calculate distances

    lons: np.array
        Longitudes from which to calculate distances

    :Returns:

    out: np.array of distances
    """

    lat_point = np.deg2rad(lat_point)
    lon_point = np.deg2rad(lon_point)
    avg_earth_radius = 6373  # in km
    lats = np.deg2rad(lats)
    lons = np.deg2rad(lons)
    lat = lat_point - lats
    lon = lon_point - lons
    d = np.sin(lat * 0.5) ** 2 + np.cos(lat_point) * np.cos(lats) * np.sin(lon * 0.5) ** 2
    dist = 2 * avg_earth_radius * np.arcsin(np.sqrt(d))
    return dist

max_stars_repo_name: coryjog/anemoi

max_issues_repo_name: aosargin/anemoi

max_forks_repo_name: aosargin/anemoi


text:
import json
import logging
from copy import deepcopy
from collections import Counter
from os import getenv

import numpy as np
import sentry_sdk
from nltk.tokenize import sent_tokenize

from common.greeting import greeting_spec
from common.link import skills_phrases_map
from common.constants import CAN_CONTINUE_PROMPT, CAN_CONTINUE_SCENARIO, MUST_CONTINUE, CAN_NOT_CONTINUE
from common.sensitive import is_sensitive_situation
from common.universal_templates import (
    if_chat_about_particular_topic,
    is_switch_topic,
    is_any_question_sentence_in_utterance,
    if_not_want_to_chat_about_particular_topic,
    if_choose_topic,
)
from common.utils import (
    get_intent_name,
    get_intents,
    get_topics,
    get_entities,
    get_common_tokens_in_lists_of_strings,
    is_no,
)
from utils import (
    calculate_single_convers_evaluator_score,
    CONV_EVAL_STRENGTH,
    CONFIDENCE_STRENGTH,
    how_are_you_spec,
    what_i_can_do_spec,
    misheard_with_spec1,
    psycho_help_spec,
    misheard_with_spec2,
    alexa_abilities_spec,
    join_used_links_in_attributes,
    get_updated_disliked_skills,
)
from common.response_selection import (
    ACTIVE_SKILLS,
    ALMOST_ACTIVE_SKILLS,
    CAN_NOT_BE_DISLIKED_SKILLS,
    NOT_ADD_PROMPT_SKILLS,
)

sentry_sdk.init(getenv("SENTRY_DSN"))

logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.DEBUG)
logger = logging.getLogger(__name__)

force_intents_fname = "force_intents_intent_catcher.json"
FORCE_INTENTS_IC = json.load(open(force_intents_fname))

lets_chat_about_triggers_fname = "lets_chat_about_triggers.json"
LETS_CHAT_ABOUT_PARTICULAR_TOPICS = json.load(open(lets_chat_about_triggers_fname))

require_action_intents_fname = "require_action_intents.json"
REQUIRE_ACTION_INTENTS = json.load(open(require_action_intents_fname))

PROMPT_PROBA = 0.3
ACKNOWLEDGEMENT_PROBA = 0.5

LINK_TO_PHRASES = sum([list(list_el) for list_el in skills_phrases_map.values()], [])

# this is a list of skills which are not one-lines
GENERAL_TOPICS = ["Phatic", "Other"]

def categorize_candidate(
    cand_id,
    skill_name,
    categorized_hyps,
    categorized_prompts,
    _is_just_prompt,
    _is_active_skill,
    _can_continue,
    _same_topic_entity,
    _is_dialog_abandon,
    _is_required_da=False,
):
    """Hypotheses could be:
        - active or not
        - can continue tag (not bool) out of CAN_CONTINUE_PROMPT, CAN_CONTINUE_SCENARIO,
                                             MUST_CONTINUE, CAN_NOT_CONTINUE
        - if has at least one of the same topic/named entity/noun phrase
        - is dialog breakdown or not
        - if contains required dialog act (required by user)

    Categories in priority order:
        = in terms of appropriateness when required dialog act:
            - reqda_same_topic_entity_no_db
            - reqda_same_topic_entity_db
            - reqda_othr_topic_entity_no_db
            - reqda_othr_topic_entity_db
            - else: TODO: here should be grounding skill with sorry about can not answer your request!
                - same_topic_entity_no_db
                - same_topic_entity_db
                - othr_topic_entity_no_db
                - othr_topic_entity_db
        = in terms of continuation of script:
            - active: active or not
            - continued: must_continue, can_continue_scenario, can_continue_scenario_done, can_not_continue
            - finished: one liners and finished scripts (can_not_continue)
        = in terms of appropriateness (user initiated switching topic and general case):
            - same_topic_entity_no_db
            - same_topic_entity_db
            - othr_topic_entity_no_db
            - othr_topic_entity_db
    """
    if (_can_continue == MUST_CONTINUE) or _is_active_skill:
        # so, scripted skills with CAN_CONTINUE_PROMPT status are not considered as active!
        # this is a chance for other skills to be turned on
        actsuffix = "active"
    elif _can_continue in [CAN_CONTINUE_SCENARIO, CAN_CONTINUE_PROMPT]:
        actsuffix = "continued"
    else:
        actsuffix = "finished"

    if _same_topic_entity and not _is_dialog_abandon:
        # have at least one the same entity/topic/nounphrase, and NO dialog breakdown
        suffix = "same_topic_entity_no_db"
    elif _same_topic_entity and _is_dialog_abandon:
        # have at least one the same entity/topic/nounphrase, and dialog breakdown
        suffix = "same_topic_entity_db"
    elif not _is_dialog_abandon:
        # no same entity/topic/nounphrase, and NO dialog breakdown
        suffix = "othr_topic_entity_no_db"
    else:
        # no same entity/topic/nounphrase, and dialog breakdown
        suffix = "othr_topic_entity_db"

    if _is_required_da:
        dasuffix = "reqda"
    else:
        dasuffix = ""

    categorized_hyps[f"{actsuffix}_{suffix}_{dasuffix}"] += [cand_id]
    if _is_just_prompt:
        categorized_prompts[f"{actsuffix}_{suffix}_{dasuffix}"] += [cand_id]
    return categorized_hyps, categorized_prompts

def choose_best_with_scores(curr_cands_ids, curr_single_scores, candidates, bot_utterances):
    for i, cand_id in enumerate(curr_cands_ids):
        if candidates[cand_id]["skill_name"] in [
            "dummy_skill",
            "convert_reddit",
            "alice",
            "eliza",
            "tdidf_retrieval",
            "program_y",
        ]:
            if "question" in candidates[cand_id].get("type", "") or "?" in candidates[cand_id]["text"]:
                penalty_start_utt = 1
                if candidates[cand_id]["skill_name"] == "program_y":
                    penalty_start_utt = 4
                n_questions = 0
                if len(bot_utterances) >= penalty_start_utt and "?" in bot_utterances[-1]:
                    curr_single_scores[i] /= 1.5
                    n_questions += 1
                if len(bot_utterances) >= penalty_start_utt + 1 and "?" in bot_utterances[-2]:
                    curr_single_scores[i] /= 1.1
                    n_questions += 1
                if n_questions == 2:
                    # two subsequent questions (1 / (1.5 * 1.1 * 1.2) = ~0.5)
                    curr_single_scores[i] /= 1.2

    curr_scores = [curr_single_scores[i] for i in curr_cands_ids]
    best_id = np.argmax(curr_scores)
    return curr_cands_ids[best_id]

def get_main_info_annotations(annotated_utterance):
    intents = get_intents(annotated_utterance, which="all")
    topics = get_topics(annotated_utterance, which="all")
    named_entities = [ent[0]["text"] for ent in annotated_utterance.get("annotations", {}).get("ner", []) if ent]
    nounphrases = [
        nounph for nounph in annotated_utterance.get("annotations", {}).get("spacy_nounphrases", []) if nounph
    ]
    return intents, topics, named_entities, nounphrases

def pickup_best_id(categorized, candidates, curr_single_scores, bot_utterances):
    """Choose best hypotheses or prompt using priorities:
    - containing required dialog act, not containing required dialog act [second case also if
        user does not require particular dialog act];
    - active, continued, finished;
    - containing same topic/entity without dialog breakdown, containing same topic/entity with dialog breakdown,
        containing other topic/entity without dialog breakdown, containing other topic/entity with dialog breakdown.
    """
    best_cand_id = 0
    for dasuffix in ["reqda", ""]:
        # firstly, consider ACTIVE SKILL
        for actsuffix in ["active"]:
            for suffix in [
                "same_topic_entity_no_db",
                "same_topic_entity_db",
                "othr_topic_entity_no_db",
                "othr_topic_entity_db",
            ]:
                if len(categorized[f"{actsuffix}_{suffix}_{dasuffix}"]) > 0:
                    best_cand_id = choose_best_with_scores(
                        categorized[f"{actsuffix}_{suffix}_{dasuffix}"], curr_single_scores, candidates, bot_utterances
                    )
                    logger.info(f"==========Found {actsuffix}_{suffix}_{dasuffix} hyp: {candidates[best_cand_id]}")
                    return best_cand_id
        # secondly, consider all skills with the same topic/entities, priority those who can continue
        for actsuffix in ["continued", "finished"]:
            for suffix in ["same_topic_entity_no_db", "same_topic_entity_db"]:
                if len(categorized[f"{actsuffix}_{suffix}_{dasuffix}"]) > 0:
                    best_cand_id = choose_best_with_scores(
                        categorized[f"{actsuffix}_{suffix}_{dasuffix}"], curr_single_scores, candidates, bot_utterances
                    )
                    logger.info(f"==========Found {actsuffix}_{suffix}_{dasuffix} hyp: {candidates[best_cand_id]}")
                    return best_cand_id
        # thirdly, consider all skills with other topic/entities, priority those who can continue
        for actsuffix in ["continued", "finished"]:
            for suffix in ["othr_topic_entity_no_db", "othr_topic_entity_db"]:
                if len(categorized[f"{actsuffix}_{suffix}_{dasuffix}"]) > 0:
                    best_cand_id = choose_best_with_scores(
                        categorized[f"{actsuffix}_{suffix}_{dasuffix}"], curr_single_scores, candidates, bot_utterances
                    )
                    logger.info(f"==========Found {actsuffix}_{suffix}_{dasuffix} hyp: {candidates[best_cand_id]}")
                    return best_cand_id

    return best_cand_id

def prompt_decision():
    if np.random.uniform() < PROMPT_PROBA:
        return True
    return False

def acknowledgement_decision(all_user_intents):
    _is_user_opinion = "opinion" in all_user_intents
    if (_is_user_opinion and np.random.uniform() < ACKNOWLEDGEMENT_PROBA) or (
        not _is_user_opinion and np.random.uniform() < ACKNOWLEDGEMENT_PROBA / 5
    ):
        return True
    return False

def compute_curr_single_scores(candidates, scores, confidences):
    curr_single_scores = []
    if all(["hypothesis_scorer" in cand["annotations"] for cand in candidates]):
        for i in range(len(candidates)):
            curr_single_scores.append(candidates[i]["annotations"]["hypothesis_scorer"])
    else:
        for i in range(len(scores)):
            cand_scores = scores[i]
            confidence = confidences[i]
            skill_name = candidates[i]["skill_name"]
            score_conv_eval = calculate_single_convers_evaluator_score(cand_scores)
            score = CONV_EVAL_STRENGTH * score_conv_eval + CONFIDENCE_STRENGTH * confidence
            toxicity = max(candidates[i].get("annotations", {}).get("toxic_classification", {"toxic": 0.0}).values())

            logger.info(
                f"Skill {skill_name} has final score: {score}. Confidence: {confidence}. "
                f"Toxicity: {toxicity}. Cand scores: {cand_scores}"
            )
            curr_single_scores.append(score)

    return curr_single_scores

def add_to_top1_category(cand_id, categorized, _is_require_action_intent):
    if _is_require_action_intent:
        categorized["active_same_topic_entity_no_db_reqda"].append(cand_id)
    else:
        categorized["active_same_topic_entity_no_db_"].append(cand_id)
    return categorized

def does_not_require_prompt(candidates, best_cand_id):
    _is_already_prompt = "prompt" in candidates[best_cand_id].get("response_parts", [])
    _is_question = "?" in candidates[best_cand_id]["text"]
    _is_very_long = len(candidates[best_cand_id]["text"]) > 200

    _best_cand_intents = get_intents(candidates[best_cand_id], which="all")
    _is_request = any([intent in _best_cand_intents for intent in REQUIRE_ACTION_INTENTS.keys()])
    _is_not_add_prompt_skill = candidates[best_cand_id]["skill_name"] in NOT_ADD_PROMPT_SKILLS

    _is_any_question = is_any_question_sentence_in_utterance(candidates[best_cand_id])
    _can_continue = candidates[best_cand_id].get("can_continue", CAN_NOT_CONTINUE) != CAN_NOT_CONTINUE
    if (
        _is_already_prompt
        or _is_question
        or _is_very_long
        or _is_request
        or _is_not_add_prompt_skill
        or _is_any_question
        or _can_continue
    ):
        return True
    return False

def if_acknowledgement_in_previous_bot_utterance(dialog):
    if len(dialog["bot_utterances"]) > 0 and len(dialog["human_utterances"]) > 1:
        prev_bot_uttr_text = dialog["bot_utterances"][-1]["text"].lower()
        prev_human_uttr = dialog["human_utterances"][-2]
        acknowledgments = []
        for hyp in prev_human_uttr["hypotheses"]:
            if hyp.get("response_parts", []) == ["acknowledgment"]:
                acknowledgments += [hyp["text"].lower()]
        for ackn in acknowledgments:
            if ackn in prev_bot_uttr_text:
                return True
    return False

def tag_based_response_selection(dialog, candidates, scores, confidences, bot_utterances, all_prev_active_skills=None):
    all_prev_active_skills = all_prev_active_skills if all_prev_active_skills is not None else []
    all_prev_active_skills = Counter(all_prev_active_skills)
    annotated_uttr = dialog["human_utterances"][-1]
    all_user_intents, all_user_topics, all_user_named_entities, all_user_nounphrases = get_main_info_annotations(
        annotated_uttr
    )

    _is_switch_topic_request = is_switch_topic(annotated_uttr)
    _is_force_intent = any([_intent in all_user_intents for _intent in FORCE_INTENTS_IC.keys()])
    # if user utterance contains any question (REGEXP & punctuation check!)
    _is_require_action_intent = is_any_question_sentence_in_utterance(
        {"text": annotated_uttr.get("annotations", {}).get("sentseg", {}).get("punct_sent", annotated_uttr["text"])}
    )
    # if user utterance contains any question AND requires some intent by socialbot
    _is_require_action_intent = _is_require_action_intent and any(
        [_intent in all_user_intents for _intent in REQUIRE_ACTION_INTENTS.keys()]
    )
    _force_intents_detected = [_intent for _intent in FORCE_INTENTS_IC.keys() if _intent in all_user_intents]
    # list of user intents which require some action by socialbot
    _require_action_intents_detected = [
        _intent for _intent in REQUIRE_ACTION_INTENTS.keys() if _intent in all_user_intents
    ]
    _force_intents_skills = sum([FORCE_INTENTS_IC.get(_intent, []) for _intent in _force_intents_detected], [])
    # list of intents required by the socialbot
    _required_actions = sum(
        [REQUIRE_ACTION_INTENTS.get(_intent, []) for _intent in _require_action_intents_detected], []
    )
    _contains_entities = len(get_entities(annotated_uttr, only_named=False, with_labels=False)) > 0
    _is_active_skill_can_not_continue = False

    _prev_bot_uttr = dialog["bot_utterances"][-1] if len(dialog["bot_utterances"]) > 0 else {}
    _prev_active_skill = dialog["bot_utterances"][-1]["active_skill"] if len(dialog["bot_utterances"]) > 0 else ""
    _prev_prev_active_skill = dialog["bot_utterances"][-2]["active_skill"] if len(dialog["bot_utterances"]) > 1 else ""
    _no_script_two_times_in_a_row = False
    if _prev_active_skill and _prev_prev_active_skill:
        if all(
            [
                skill not in ACTIVE_SKILLS + ALMOST_ACTIVE_SKILLS
                for skill in [_prev_active_skill, _prev_prev_active_skill]
            ]
        ):
            _no_script_two_times_in_a_row = True
    disliked_skills = get_updated_disliked_skills(dialog, can_not_be_disliked_skills=CAN_NOT_BE_DISLIKED_SKILLS)

    _is_dummy_linkto_available = any(
        [
            cand_uttr["skill_name"] == "dummy_skill" and cand_uttr.get("type", "") == "link_to_for_response_selector"
            for cand_uttr in candidates
        ]
    )

    categorized_hyps = {}
    categorized_prompts = {}
    for dasuffix in ["reqda", ""]:
        for actsuffix in ["active", "continued", "finished"]:
            for suffix in [
                "same_topic_entity_no_db",
                "same_topic_entity_db",
                "othr_topic_entity_no_db",
                "othr_topic_entity_db",
            ]:
                categorized_hyps[f"{actsuffix}_{suffix}_{dasuffix}"] = []
                categorized_prompts[f"{actsuffix}_{suffix}_{dasuffix}"] = []

    CASE = ""
    acknowledgement_hypothesis = {}

    for cand_id, cand_uttr in enumerate(candidates):
        if confidences[cand_id] == 0.0 and cand_uttr["skill_name"] not in ACTIVE_SKILLS:
            logger.info(f"Dropping cand_id: {cand_id} due to toxicity/badlists")
            continue

        all_cand_intents, all_cand_topics, all_cand_named_entities, all_cand_nounphrases = get_main_info_annotations(
            cand_uttr
        )
        skill_name = cand_uttr["skill_name"]
        _is_dialog_abandon = False  # "abandon" in all_cand_intents
        _is_just_prompt = (
            cand_uttr["skill_name"] == "dummy_skill"
            and any(
                [
                    question_type in cand_uttr.get("type", "")
                    for question_type in ["normal_question", "link_to_for_response_selector"]
                ]
            )
        ) or cand_uttr.get("response_parts", []) == ["prompt"]
        if cand_uttr["confidence"] == 1.0:
            cand_uttr["can_continue"] = MUST_CONTINUE

        _user_wants_to_chat_about_topic = (
            if_chat_about_particular_topic(annotated_uttr) and "about it" not in annotated_uttr["text"].lower()
        )
        _user_does_not_want_to_chat_about_topic = if_not_want_to_chat_about_particular_topic(annotated_uttr)
        _user_wants_bot_to_choose_topic = if_choose_topic(annotated_uttr, _prev_bot_uttr)

        if any([phrase.lower() in cand_uttr["text"].lower() for phrase in LINK_TO_PHRASES]):
            # add `prompt` to response_parts if any linkto phrase in hypothesis
            cand_uttr["response_parts"] = cand_uttr.get("response_parts", []) + ["prompt"]

        _same_named_entities = (
            len(get_common_tokens_in_lists_of_strings(all_cand_named_entities, all_user_named_entities)) > 0
        )
        _same_nounphrases = len(get_common_tokens_in_lists_of_strings(all_cand_nounphrases, all_user_nounphrases)) > 0
        _same_topic_entity = False

        # if cand_uttr["skill_name"] == 'program_y' and cand_uttr['confidence'] == 0.98 and \
        #         len(cand_uttr["text"].split()) > 6:
        #     cand_uttr["can_continue"] = CAN_CONTINUE_SCENARIO
        _can_continue = cand_uttr.get("can_continue", CAN_NOT_CONTINUE)

        _is_active_skill = (
            _prev_active_skill == cand_uttr["skill_name"] or cand_uttr.get("can_continue", "") == MUST_CONTINUE
        )
        _is_active_skill = _is_active_skill and skill_name in ACTIVE_SKILLS
        _is_active_skill = _is_active_skill and (
            _can_continue in [MUST_CONTINUE, CAN_CONTINUE_SCENARIO, CAN_NOT_CONTINUE]
            or (_can_continue == CAN_CONTINUE_PROMPT and all_prev_active_skills.get(skill_name, []) < 10)
        )
        if _is_active_skill:
            # we will forcibly add prompt if current scripted skill finishes scenario,
            # and has no opportunity to continue at all.
            _is_active_skill_can_not_continue = _is_active_skill and _can_continue in [CAN_NOT_CONTINUE]

        if _is_force_intent:
            # =====force intents, choose as best_on_topic hypotheses from skills responding this request=====

            CASE = "Force intent."
            if cand_uttr["skill_name"] in _force_intents_skills:
                categorized_hyps, categorized_prompts = categorize_candidate(
                    cand_id,
                    skill_name,
                    categorized_hyps,
                    categorized_prompts,
                    _is_just_prompt,
                    _is_active_skill,
                    _can_continue,
                    _same_topic_entity,
                    _is_dialog_abandon,
                    _is_required_da=False,
                )

        elif _is_switch_topic_request or _user_does_not_want_to_chat_about_topic or _user_wants_bot_to_choose_topic:
            # =====direct request by user to switch the topic of current conversation=====
            # give priority to dummy linkto hypothesis if available, else other prompts if available.
            _is_active_skill = (
                cand_uttr.get("type", "") == "link_to_for_response_selector"
                if _is_dummy_linkto_available
                else _is_just_prompt
            )
            # no priority to must_continue to skip incorrect continuation of script
            _can_continue = CAN_CONTINUE_SCENARIO if _can_continue == MUST_CONTINUE else _can_continue

            CASE = "Switch topic intent."
            if len(all_user_named_entities) > 0 or len(all_user_nounphrases) > 0:
                # -----user defines new topic/entity-----

                _same_named_entities = (
                    len(get_common_tokens_in_lists_of_strings(all_cand_named_entities, all_user_named_entities)) > 0
                )
                _same_nounphrases = (
                    len(get_common_tokens_in_lists_of_strings(all_cand_nounphrases, all_user_nounphrases)) > 0
                )
                _same_topic_entity = _same_named_entities or _same_nounphrases

                categorized_hyps, categorized_prompts = categorize_candidate(
                    cand_id,
                    skill_name,
                    categorized_hyps,
                    categorized_prompts,
                    _is_just_prompt,
                    _is_active_skill,
                    _can_continue,
                    _same_topic_entity,
                    _is_dialog_abandon,
                    _is_required_da=False,
                )
            else:
                # -----user want socialbot to define new topic/entity-----

                categorized_hyps, categorized_prompts = categorize_candidate(
                    cand_id,
                    skill_name,
                    categorized_hyps,
                    categorized_prompts,
                    _is_just_prompt,
                    _is_active_skill,
                    _can_continue,
                    _same_topic_entity,
                    _is_dialog_abandon,
                    _is_required_da=False,
                )

        elif _user_wants_to_chat_about_topic:
            # user wants to chat about particular topic

            CASE = "User wants to talk about topic."
            # in this case we do not give priority to previously active skill (but give to must continue skill!)
            # because now user wants to talk about something particular
            _is_active_skill = cand_uttr.get("can_continue", "") == MUST_CONTINUE
            _same_named_entities = (
                len(get_common_tokens_in_lists_of_strings(all_cand_named_entities, all_user_named_entities)) > 0
            )
            _same_nounphrases = (
                len(get_common_tokens_in_lists_of_strings(all_cand_nounphrases, all_user_nounphrases)) > 0
            )
            _same_topic_entity = _same_named_entities or _same_nounphrases

            categorized_hyps, categorized_prompts = categorize_candidate(
                cand_id,
                skill_name,
                categorized_hyps,
                categorized_prompts,
                _is_just_prompt,
                _is_active_skill,
                _can_continue,
                _same_topic_entity,
                _is_dialog_abandon,
                _is_required_da=False,
            )

        elif _is_require_action_intent:
            # =====user intent requires particular action=====

            CASE = "User intent requires action. USER UTTERANCE CONTAINS QUESTION."
            _is_grounding_reqda = (
                skill_name == "dff_grounding_skill" and cand_uttr.get("type", "") == "universal_response"
            )
            _is_active_skill = cand_uttr.get("can_continue", "") == MUST_CONTINUE  # no priority to prev active skill
            _can_continue = CAN_NOT_CONTINUE  # no priority to scripted skills

            if set(all_cand_intents).intersection(set(_required_actions)) or _is_grounding_reqda or _is_active_skill:
                # -----one of the can intent is in intents required by user-----
                categorized_hyps, categorized_prompts = categorize_candidate(
                    cand_id,
                    skill_name,
                    categorized_hyps,
                    categorized_prompts,
                    _is_just_prompt,
                    _is_active_skill,
                    _can_continue,
                    _same_topic_entity,
                    _is_dialog_abandon,
                    _is_required_da=True,
                )
            else:
                # -----NO required dialog acts-----
                categorized_hyps, categorized_prompts = categorize_candidate(
                    cand_id,
                    skill_name,
                    categorized_hyps,
                    categorized_prompts,
                    _is_just_prompt,
                    _is_active_skill,
                    _can_continue,
                    _same_topic_entity,
                    _is_dialog_abandon,
                    _is_required_da=False,
                )

        else:
            # =====user intent does NOT require particular action=====

            CASE = "General case."
            categorized_hyps, categorized_prompts = categorize_candidate(
                cand_id,
                skill_name,
                categorized_hyps,
                categorized_prompts,
                _is_just_prompt,
                _is_active_skill,
                _can_continue,
                _same_topic_entity,
                _is_dialog_abandon,
                _is_required_da=False,
            )

        # a bit of rule based help
        is_misheard = misheard_with_spec1 in cand_uttr["text"] or misheard_with_spec2 in cand_uttr["text"]
        intent_name = get_intent_name(cand_uttr["text"])
        is_intent_candidate = intent_name and cand_uttr["skill_name"] in ["intent_responder", "program_y"]
        if (
            len(dialog["human_utterances"]) == 1
            and cand_uttr["skill_name"] == "dff_friendship_skill"
            and greeting_spec in cand_uttr["text"]
        ):
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        elif (
            cand_uttr["skill_name"] == "dff_friendship_skill"
            and (how_are_you_spec in cand_uttr["text"] or what_i_can_do_spec in cand_uttr["text"])
            and len(dialog["utterances"]) < 16
        ):
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        # elif cand_uttr["skill_name"] == 'program_y_dangerous' and cand_uttr['confidence'] == 0.98:
        #     categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        elif cand_uttr["skill_name"] == "small_talk_skill" and is_sensitive_situation(dialog["human_utterances"][-1]):
            # let small talk to talk about sex ^_^
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        elif cand_uttr["skill_name"] == "misheard_asr" and is_misheard:
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        elif is_intent_candidate:
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        elif cand_uttr["skill_name"] == "dff_friendship_skill" and alexa_abilities_spec in cand_uttr["text"]:
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        elif "program_y" in cand_uttr["skill_name"] and psycho_help_spec in cand_uttr["text"]:
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)
        elif cand_uttr["confidence"] >= 1.0:
            # -------------------- SUPER CONFIDENCE CASE HERE! --------------------
            categorized_hyps = add_to_top1_category(cand_id, categorized_hyps, _is_require_action_intent)

        if cand_uttr["skill_name"] == "dff_grounding_skill" and "acknowledgement" in cand_uttr.get(
            "response_parts", []
        ):
            acknowledgement_hypothesis = deepcopy(cand_uttr)

    logger.info(f"Current CASE: {CASE}")
    # now compute current scores as one float value
    curr_single_scores = compute_curr_single_scores(candidates, scores, confidences)

    # remove disliked skills from hypotheses
    for category in categorized_hyps:
        new_ids = []
        for cand_id in categorized_hyps[category]:
            if (
                candidates[cand_id]["skill_name"] in disliked_skills
                and candidates[cand_id].get("can_continue", CAN_NOT_CONTINUE) == MUST_CONTINUE
            ):
                disliked_skills.remove(candidates[cand_id]["skill_name"])
            if candidates[cand_id]["skill_name"] not in disliked_skills:
                new_ids.append(cand_id)
        categorized_hyps[category] = deepcopy(new_ids)
    for category in categorized_prompts:
        new_ids = []
        for cand_id in categorized_prompts[category]:
            if (
                candidates[cand_id]["skill_name"] in disliked_skills
                and candidates[cand_id].get("can_continue", CAN_NOT_CONTINUE) == MUST_CONTINUE
            ):
                disliked_skills.remove(candidates[cand_id]["skill_name"])
            if candidates[cand_id]["skill_name"] not in disliked_skills:
                new_ids.append(cand_id)
        categorized_prompts[category] = deepcopy(new_ids)

    best_cand_id = pickup_best_id(categorized_hyps, candidates, curr_single_scores, bot_utterances)
    best_candidate = candidates[best_cand_id]
    best_candidate["human_attributes"] = best_candidate.get("human_attributes", {})
    # save updated disliked skills to human attributes of the best candidate
    best_candidate["human_attributes"]["disliked_skills"] = disliked_skills
    logger.info(f"Best candidate: {best_candidate}")
    n_sents_without_prompt = len(sent_tokenize(best_candidate["text"]))
    _is_best_not_script = best_candidate["skill_name"] not in ACTIVE_SKILLS + ALMOST_ACTIVE_SKILLS
    no_question_by_user = "?" not in dialog["human_utterances"][-1]["annotations"].get("sentseg", {}).get(
        "punct_sent", dialog["human_utterances"][-1]["text"]
    )

    # if `no` to 1st in a row linkto question, and chosen response is not from scripted skill
    _no_to_first_linkto = is_no(dialog["human_utterances"][-1]) and any(
        [phrase.lower() in _prev_bot_uttr.get("text", "").lower() for phrase in LINK_TO_PHRASES]
    )
    # if chosen short response or question by not-scripted skill
    _is_short_or_question_by_not_script = _is_best_not_script and (
        "?" in best_candidate["text"] or len(best_candidate["text"].split()) < 4
    )

    if (_no_script_two_times_in_a_row and _is_short_or_question_by_not_script and no_question_by_user) or (
        _no_to_first_linkto and _is_best_not_script
    ):
        # if no scripted skills 2 time sin a row before, current chosen best cand is not scripted, contains `?`,
        # and user utterance does not contain "?", replace utterance with dummy!
        best_prompt_id = pickup_best_id(categorized_prompts, candidates, curr_single_scores, bot_utterances)
        best_candidate = deepcopy(candidates[best_prompt_id])
        best_cand_id = best_prompt_id

    if does_not_require_prompt(candidates, best_cand_id):
        # the candidate already contains a prompt or a question or of a length more than 200 symbols
        logger.info("Best candidate contains prompt, question, request or length of > 200 symbols. Do NOT add prompt.")
        pass
    elif sum(categorized_prompts.values(), []):
        # best cand is 3d times in a row not scripted skill, let's append linkto

        # need to add some prompt, and have a prompt
        _add_prompt_forcibly = best_candidate["skill_name"] == _prev_active_skill and _is_active_skill_can_not_continue
        _add_prompt_forcibly = _add_prompt_forcibly and not _contains_entities

        if _add_prompt_forcibly or prompt_decision() or (_no_script_two_times_in_a_row and _is_best_not_script):
            logger.info("Decided to add a prompt to the best candidate.")
            best_prompt_id = pickup_best_id(categorized_prompts, candidates, curr_single_scores, bot_utterances)
            # as we have only one active skill, let's consider active skill as that one providing prompt
            # but we also need to reassign all the attributes
            best_prompt = candidates[best_prompt_id]
            best_candidate["text"] = f'{best_candidate["text"]} {best_prompt["text"]}'
            # TODO: how to correctly reassign human-/bot-/hyp- attributes??? how to deal with skill name?
            best_candidate["attributes"] = best_candidate.get("attributes", {})
            best_candidate["attributes"]["prompt_skill"] = best_prompt

            # anyway we must combine used links
            best_candidate["human_attributes"] = best_candidate.get("human_attributes", {})
            best_candidate["human_attributes"] = join_used_links_in_attributes(
                best_candidate["human_attributes"], best_prompt.get("human_attributes", {})
            )
            if len(best_candidate["human_attributes"]["used_links"]) == 0:
                best_candidate["human_attributes"].pop("used_links")

    was_ackn = if_acknowledgement_in_previous_bot_utterance(dialog)
    best_resp_cont_ackn = "acknowledgement" in best_candidate.get("response_parts", [])

    if (
        acknowledgement_hypothesis
        and acknowledgement_decision(all_user_intents)
        and n_sents_without_prompt == 1
        and not was_ackn
        and not best_resp_cont_ackn
    ):
        logger.info(
            "Acknowledgement is given, Final hypothesis contains only 1 sentence, no ackn in prev bot uttr,"
            "and we decided to add an acknowledgement to the best candidate."
        )
        best_candidate["text"] = f'{acknowledgement_hypothesis["text"]} {best_candidate["text"]}'
        best_candidate["response_parts"] = ["acknowledgement"] + best_candidate.get("response_parts", [])

    return best_candidate, best_cand_id, curr_single_scores

max_stars_repo_name: oserikov/dream

max_issues_repo_name: oserikov/dream

max_forks_repo_name: oserikov/dream


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Author    : allqoow
# Contact   : allqoow@gmail.com
# Started on: 20161121(yyyymmdd)
# Project   : exerciseML(Exercise for Machine Learning)

# H5.1.A
import matplotlib.pyplot
import math
import numpy.random
import numpy.linalg

def getNumberRows(mat):
	return len(mat)

def getNumberCols(mat):
	return len(mat[0])

def makeTranspose(mat):
	return zip(*mat)

def makeInnerProduct(vec0, vec1):
	seqTerm = map(lambda x: x[0]*x[1], zip(vec0, vec1))
	return sum(seqTerm)

def doMatrixMultiplication(mat0, mat1):
	ret = None
	if getNumberCols(mat0) == getNumberRows(mat1):
		nRows = getNumberRows(mat0)
		nCols = getNumberCols(mat1)
		mat1Transpose = makeTranspose(mat1)
		ret = []
		for i in range(nRows):
			tmpRow = []
			for j in range(nCols):
				tmpRow.append(makeInnerProduct(mat0[i],mat1Transpose[j]))
			ret.append(tmpRow)
	else:
		pass
	return ret
def doScalarMultiplication(mat, scalar):
	nRows = getNumberRows(mat)
	nCols = getNumberCols(mat)
	ret = []
	for i in range(nRows):
		tmp = []
		for j in range(nCols):
			tmp.append(mat[i][j]*scalar)
		ret.append(tmp)
	return ret

def readCSV(data, flag):
	seqX1 = []
	seqX2 = []
	seqObs = []
	with open(data, flag) as f:
		readline = f.readline()
		while readline:
			try:
				x1 = float(readline.strip().split(",")[0])
				x2 = float(readline.strip().split(",")[1])
				obs = float(readline.strip().split(",")[2])
				seqX1.append(x1)
				seqX2.append(x2)
				seqObs.append(obs)
				# print readline.strip()
			except ValueError:
				# print "ValueError: This row is the header"
				pass
			finally:
				readline = f.readline()
	return zip(seqX1,seqX2,seqObs)

def doCentering(data, flag):
	seqX1 = []
	seqX2 = []
	seqObs = []
	# with open("TrainingRidge.csv","r") as f:
	with open(data, flag) as f:
		readline = f.readline()
		while readline:
			try:
				x1 = float(readline.strip().split(",")[0])
				x2 = float(readline.strip().split(",")[1])
				obs = float(readline.strip().split(",")[2])
				seqX1.append(x1)
				seqX2.append(x2)
				seqObs.append(obs)
				# print readline.strip()
			except ValueError:
				# print "ValueError: This row is the header"
				pass
			finally:
				readline = f.readline()

	# Transpose
	seqXX = makeTranspose(zip(seqX1, seqX2))

	# Sample mean of seqX1
	smX1 = reduce(lambda tmp, x1: tmp+x1, seqX1)/len(seqX1)
	smX2 = reduce(lambda tmp, x2: tmp+x2, seqX2)/len(seqX2)
	seqX1Centered = list(map(lambda x1: round(x1-smX1,2), seqX1))
	seqX2Centered = list(map(lambda x2: round(x2-smX2,1), seqX2))

	seqXXCentered = makeTranspose(zip(seqX1Centered,seqX2Centered))
	return seqXXCentered

def doSphering(data, flag):
	seqX1 = []
	seqX2 = []
	seqObs = []
	# with open("TrainingRidge.csv","r") as f:
	with open(data, flag) as f:
		readline = f.readline()
		while readline:
			try:
				x1 = float(readline.strip().split(",")[0])
				x2 = float(readline.strip().split(",")[1])
				obs = float(readline.strip().split(",")[2])
				seqX1.append(x1)
				seqX2.append(x2)
				seqObs.append(obs)
				# print readline.strip()
			except ValueError:
				# print "ValueError: This row is the header"
				pass
			finally:
				readline = f.readline()

	# Transpose
	seqXX = makeTranspose(zip(seqX1, seqX2))

	# Sample mean of seqX1
	smX1 = reduce(lambda tmp, x1: tmp+x1, seqX1)/len(seqX1)
	smX2 = reduce(lambda tmp, x2: tmp+x2, seqX2)/len(seqX2)
	seqX1Centered = list(map(lambda x1: round(x1-smX1,2), seqX1))
	seqX2Centered = list(map(lambda x2: round(x2-smX2,1), seqX2))

	seqXXCentered = makeTranspose(zip(seqX1Centered,seqX2Centered))

	cVec=(0,0,1,1) # blue
	# matplotlib.pyplot.scatter(seqXX[0], seqXX[1], c=cVec)
	# matplotlib.pyplot.show()
	# matplotlib.pyplot.clf()

	# matplotlib.pyplot.scatter(seqXXCentered[0], seqXXCentered[1], c=cVec)
	# matplotlib.pyplot.show()
	# matplotlib.pyplot.clf()

	matC = doMatrixMultiplication(seqXXCentered, makeTranspose(seqXXCentered))
	matC = doScalarMultiplication(matC, 0.005)
	print matC

	eigvals = numpy.linalg.eigvals(matC)

	print numpy.linalg.eig(matC)
	matV = numpy.linalg.eig(matC)[1]
	print matV
	matVTranspose = makeTranspose(matV)
	matLambda = [[eigvals[0], 0], [0, eigvals[1]]]
	print matLambda

	print "\n"
	print matC
	result = doMatrixMultiplication(matV,matLambda)
	result = doMatrixMultiplication(result,matVTranspose)
	print result

	# eigVec1 = matVTranspose[0]
	# eigVec2 = matVTranspose[1]
	eigVec1 = matV[0]
	eigVec2 = matV[1]
	tmp1 = []
	tmp2 = []
	for x in makeTranspose(seqXXCentered):
		tmp1.append(makeInnerProduct(eigVec1, x))
		tmp2.append(makeInnerProduct(eigVec2, x))
	seqXXDecorr = [tmp1, tmp2]
	print getNumberRows(seqXXDecorr)
	print getNumberCols(seqXXDecorr)

	# matplotlib.pyplot.scatter(seqXXDecorr[0], seqXXDecorr[1], c=(1,0,0,1))
	# matplotlib.pyplot.show()
	# matplotlib.pyplot.clf()

	tmp1 = []
	tmp2 = []
	someVec1 = [matLambda[0][0]**(-0.5), 0]
	someVec2 = [0, matLambda[1][1]**(-0.5)]
	for x in makeTranspose(seqXXDecorr):
		tmp1.append(makeInnerProduct(someVec1, x))
		tmp2.append(makeInnerProduct(someVec2, x))
	seqXXWhiten = [tmp1, tmp2]

	matLambda05 = [someVec1, someVec2]
	matTransform = doMatrixMultiplication(matLambda05, matVTranspose)
	return matTransform
	# matplotlib.pyplot.scatter(seqXXWhiten[0], seqXXWhiten[1], c=cVec)
	# matplotlib.pyplot.show()
	# matplotlib.pyplot.clf()

# Start of Validation
matTransform = doSphering("TrainingRidge.csv","r")
print matTransform
seqXXCenteredValid = doCentering("ValidationRidge-Y.csv","r")

seqXXWhitenValid = doMatrixMultiplication(matTransform, seqXXCenteredValid)
print seqXXWhitenValid

cVec=(0,0,1,1) # blue
# H5.2.B.

# matplotlib.pyplot.scatter(seqXXWhitenValid[0], seqXXWhitenValid[1], c=cVec)
# matplotlib.pyplot.show()
# matplotlib.pyplot.clf()

vv = readCSV("ValidationRidge-Y.csv","r")

for k in range(4):
	for l in range(k+1):
		alpha = l
		beta = k - l
		seqMonom =[]
		seqEstm = []
		for i in range(getNumberCols(seqXXWhitenValid)):
			monom = (seqXXWhitenValid[0][i]**alpha)*(seqXXWhitenValid[1][i]**beta)
			estm = vv[i][2]
			seqMonom.append(monom)
			seqEstm.append(estm)
		print str(alpha) + " " + str(beta)
		matplotlib.pyplot.scatter(seqMonom, seqEstm, c=cVec)
		matplotlib.pyplot.show()
		matplotlib.pyplot.clf()

# def getEigenvalues(mat):
# 	dim = getNumberRows(mat)
# 	mat1 = []
# 	for i in range(dim):
# 		tmpRow = []
# 		for j in range(dim):
# 			if i==j:
# 				elem = mat[i][j] - eigVal
# 			else:
# 				elem = mat[i][j]
# 			tmpRow.append(elem)
# 		mat1.append(mat1)

# def getDeterminant22(mat):
# 	pass

max_stars_repo_name: allqoow/exerciseML

max_issues_repo_name: allqoow/exerciseML

max_forks_repo_name: allqoow/exerciseML