import getopt
import sys
import time
import datetime
import socket
import select
#import threading

host = '127.0.0.1'
port = 1518

line_fields = '{0:>8.8s} {1:>5.5s} {2:>5.5s} {3:>7.7s} {4:>5.5s} {5:>5.5s} {6:>5.5s} {7:>5.5s} {8:>5.5s} {9:>5.5s} {10:>5.5s} {11:>5.5s} {12:>5.5s} {13:>5.5s} {14:>5.5s} {15:>5.5s} {16:>5.5s} {17:>4.4s} {18:>5.5s} {19:>5.5s} {20:>5.5s} {21:>1.1s}'

response = ''

def usage ():
    print('python feedmonitor.py -h host -p port')

def filter_chars(chars):
    filtered_string = ''

    for i in range(len(chars)):
        char = chars[i]
        if ((char == '|') or (ord(char) == 0x0d) or (ord(char) == 0x0a)):
            filtered_string = filtered_string + chr(0x0d)
            continue
        if ((ord(char) >= 0x20) and (ord(char) <= 0x7e)):
            filtered_string = filtered_string + char

    return (filtered_string)

def get_line(sckt):
    global response

    timeout_count = 0
    while (timeout_count < 9):
        lines = response.split(chr(0x0d), 1)
        if (len(lines) == 2):
            response = lines[1]
            return (lines[0])
        readable, writable, exceptional = select.select([sckt], [], [], 0.1)
        if (readable):
            response = response + filter_chars(sckt.recv(999))
            timeout_count = 0
        else:
            timeout_count = timeout_count + 1

    return (None)  # timeout

def purge_response(sckt):
    global response

    response = ''

    timeout_count = 0
    while (timeout_count < 2):
        readable, writable, exceptional = select.select([sckt], [], [], 0.1)
        if (readable):
            sckt.recv(999)
            timeout_count = 0
        else:
            timeout_count = timeout_count + 1

def get_and_print_status(sckt):
    dt = datetime.datetime.now()

    status_line = line_fields.format(
        dt.strftime('%H:%M:%S'),
        session(sckt, 'TC', 1, 1, cnvrt_temp),
        session(sckt, 'gd', 0, 1, cnvrt_temp),
        session(sckt, 'gv', 0, 1, cnvrt_vacuum),
        session(sckt, 'p398', 0, 1, cnvrt_turbo_rpm),
        session(sckt, 'p310', 0, 1, cnvrt_turbo_amps),
        session(sckt, 'p316', 0, 1, cnvrt_turbo_watts),
        session(sckt, 'p326', 0, 1, cnvrt_turbo_temp),
        session(sckt, 'p330', 0, 1, cnvrt_turbo_temp),
        session(sckt, 'p342', 0, 1, cnvrt_turbo_temp),
        session(sckt, 'p346', 0, 1, cnvrt_turbo_temp),
#        session(sckt, 'gt a0', 0, 1, cnvrt_temp),
        session(sckt, 'gettemp a0', 0, 1, cnvrt_temp),
#        session(sckt, 'gt a1', 0, 1, cnvrt_temp),
        session(sckt, 'gettemp a1', 0, 1, cnvrt_temp),
        session(sckt, 'gt a2', 0, 1, cnvrt_temp),
        session(sckt, 'gt a3', 0, 1, cnvrt_temp),
        session(sckt, 'gt a5', 0, 1, cnvrt_temp),
        session(sckt, 'gt a6', 0, 1, cnvrt_temp),
        session(sckt, 'getfanrpm', 0, 1, cnvrt_fan_rpm),
        session(sckt, 'E', 1, 3, cnvrt_E_pwr),
        session(sckt, 'E', 1, 3, cnvrt_E_max),
        session(sckt, 'E', 1, 3, cnvrt_E_min),
        session(sckt, 'getcryoattemp', 0, 1, cnvrt_at_temp)
    )

    print(status_line)

def session(sckt, command, num_echo_lns, num_rspns_lns, cnvrt_func):
    for r in range(2):
        try:
            purge_response(sckt)
            lines = []
            sckt.send(command + '\r')
            for l in range(num_echo_lns):
                get_line(sckt)  # throw away echoed command
            # then get response line(s)
            for l in range(num_rspns_lns):
                lines.append('')
                ln = get_line(sckt)
                if (ln is None):
                    raise Exception('')
                lines[l] = ln
                if (not(len(lines[l]) > 0)):
                    raise Exception('')
            return (cnvrt_func(lines))
        except Exception:
            time.sleep(0.7)
            pass

    return ('----------')  # will fill because wider than any particular field

def cnvrt_temp(lines):
    return ('{0:5.1f}'.format(float(lines[0])))

def cnvrt_vacuum(lines):
    return ('{0:.1e}'.format(float(lines[0].split()[0])))

def cnvrt_turbo_rpm(lines):
    return ('{0:5d}'.format(int(lines[0], 10)))

def cnvrt_turbo_amps(lines):
    return ('{0:5.1f}'.format(int(lines[0], 10) / 100.0))

def cnvrt_turbo_watts(lines):
    return ('{0:5d}'.format(int(lines[0], 10)))

def cnvrt_turbo_temp(lines):
    return ('{0:5d}'.format(int(lines[0], 10)))

def cnvrt_fan_rpm(lines):
    return ('{0:4d}'.format(int(lines[0], 10)))

def cnvrt_E_pwr(lines):
    return ('{0:5.1f}'.format(float(lines[2])))

def cnvrt_E_min(lines):
    return ('{0:5.1f}'.format(float(lines[1])))

def cnvrt_E_max(lines):
    return ('{0:5.1f}'.format(float(lines[0])))

def cnvrt_at_temp(lines):
    if (lines[0].strip().startswith('y')):
        return ('Y')
    if (lines[0].strip().startswith('n')):
        return ('N')
    return (' ')

def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'h:p:')
        for o,a in opts:
            if o in ('-h'):
                host = a
            if o in ('-p'):
                port = a
    except getopt.GetoptError:
        usage()
        sys.exit()

    if ((host == None) or (port == None)):
        usage()
        sys.exit()

    sckt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
        sckt.settimeout(3.0)
        sckt.connect((host, int(port)))
    except:
        print('exception while attempting connection to host ' + host + ' on port ' + port)
        sys.exit()

    time.sleep(0.3)

    sckt.settimeout(None)
    sckt.setblocking(1)

    purge_response(sckt)
    sckt.send(chr(0x0d))
    get_line(sckt);

    purge_response(sckt)
    sckt.send(chr(0x0d))
    get_line(sckt);

#   try three times
    for i in range(3):
        purge_response(sckt)
        sckt.send('stty')
        sckt.send(chr(0x0d))
        ln = get_line(sckt);
        if (not(ln is None) and (ln == 'OK')):
            get_and_print_status(sckt)
            break
        time.sleep(0.3)

    sckt.close()

main()


import sys
import itertools

def processTest():
    LINE = [int(x) for x in f.readline()[:-1].split(' ')]
    R = LINE[0]
    C = LINE[1]
    N = LINE[2]

    minnoise = 100000000000
    for el in itertools.combinations(range(R*C),N):
        arr = [[0 for j in range(C)] for i in range(R)]
        for i in range(R):
            for j in range(C):
                if i+j*R in el:
                    arr[i][j] = 1
        #print(arr)
        def countNoise(arr):
            noiselvl = 0
            for i in range(R):
                for j in range(C):
                    if arr[i][j]>0:
                        if i<R-1 and arr[i+1][j]>0:
                            noiselvl +=1
                        if j<C-1 and arr[i][j+1]>0:
                            noiselvl +=1
            return noiselvl
        noise = countNoise(arr)
        if noise<minnoise:
            minnoise = noise
    return minnoise

f = open(sys.argv[1],'r')
maxtests = int(f.readline())
testnum = 1
while testnum <= maxtests:
    y = processTest()
    print('Case #'+str(testnum)+': '+ str(y))
    testnum+=1

f.close()


import numpy as np
import imutils
import cv2
import cv2 as cv
import os
from skimage import morphology, measure
import argparse
import skimage.io as io
import matplotlib.pyplot as plt
import time

from skimage import color
from functools import partial
from mpl_toolkits.mplot3d import Axes3D

diameters = []

def process_one_image(val,path,factor,scale):
    img = cv2.imread(path)
    x,y,_ = img.shape
    img = cv2.resize(img,(y//scale,x//scale))

    kernel = np.ones((5,5),np.uint8)
    blurr_val = cv2.getTrackbarPos('blur','blur')
    canny = cv2.getTrackbarPos('canny','canny')
    size = cv2.getTrackbarPos('size','Low size')
    #size = size/10
    dst_val = cv2.getTrackbarPos('dst_va','dst_va')
    dst_val = dst_val/100
    temp_image = img.copy()
    img = cv2.bilateralFilter(img,blurr_val,75,75)
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    #gray = cv2.Canny(gray,canny_val,2*canny_val)
    gray = cv.Canny(gray, threshold1 = canny , threshold2 = 2*canny)

    gray = cv2.dilate(gray,kernel,iterations = 1)
    ret, thresh = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)

    # noise removal
    opening = cv2.morphologyEx(thresh,cv2.MORPH_OPEN,kernel, iterations = 1)
    closing = cv2.morphologyEx(opening,cv2.MORPH_CLOSE,kernel, iterations = 1)

    # sure background area
    sure_bg = cv2.dilate(thresh,kernel,iterations=1)

    # Finding sure foreground area
    dist_transform = cv2.distanceTransform(sure_bg,cv2.DIST_L2,3)

    # Threshold
    ret, sure_fg = cv2.threshold(dist_transform,dst_val*dist_transform.max(),255,0)

    # Finding unknown region
    sure_fg = np.uint8(sure_fg)
    unknown = cv2.subtract(sure_bg,sure_fg)

    # Marker labelling
    ret, markers = cv2.connectedComponents(sure_fg)

    # Add one to all labels so that sure background is not 0, but 1
    markers = markers+1

    # Now, mark the region of unknown with zero
    markers[unknown==255] = 0

    markers = cv2.watershed(img,markers)

    img[markers == -1] = [255,0,0]

  #------------------------------------
    import imutils
    import random as rng
    contours = []

    # loop over the unique labels, and append contours to all_cnts
    labels = markers
    for label in np.unique(labels):
        if label == 0:
            continue

        # draw label on the mask
        mask = np.zeros(thresh.shape, dtype="uint8")
        mask[labels == label] = 255

        # detect contours in the mask and grab the largest one
        cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        #im2, cnts, hierarchy = cv2.findContours(mask.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)
        c = max(cnts, key=cv2.contourArea)
        for c in cnts:
            contours.append(c)
    contours = sorted(contours, key=lambda x: cv2.contourArea(x),reverse = True)[2:]

    global diameters
    for c in contours:
        #Ignore small contours
        x,y,_ = img.shape
        if cv2.contourArea(c) > 1 and cv2.contourArea(c) < (x*y)/4:
            color = (rng.randint(0,256), rng.randint(0,256), rng.randint(0,256))
            ((x, y), r) = cv2.minEnclosingCircle(c) #finding diameters of contours
            l = round(2*r*factor,2)
            if l > size:
                diameters.append(l)
                cv2.drawContours(temp_image,[c], 0, color,2)
                #cv2.circle(temp_image, (int(x), int(y)), int(r),(0,255,0), 1)
                #cv2.putText(temp_image, "{}cm".format(2*round(r*factor,2)), (int(x) - 10, int(y)),cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0), 2)

    cv2.imshow('image', temp_image)
    return #[temp_image,contours]

def fine_tune(path,factor,scale):

	cv2.namedWindow('blur', cv2.WINDOW_NORMAL)
	cv2.createTrackbar('blur','blur',25,500,partial(process_one_image,path=path,factor = factor,scale = scale))

	cv2.namedWindow('canny', cv2.WINDOW_NORMAL)
	cv2.createTrackbar('canny','canny',100,255,partial(process_one_image,path=path,factor = factor,scale = scale))
	cv2.namedWindow('Low size', cv2.WINDOW_NORMAL)
	cv2.createTrackbar('size','Low size',1,500,partial(process_one_image,path=path,factor = factor,scale = scale))
	cv2.namedWindow('dst_va', cv2.WINDOW_NORMAL)
	cv2.createTrackbar('dst_va','dst_va',10,500,partial(process_one_image,path=path,factor = factor,scale = scale))

	process_one_image(0,path,factor,scale)

	cv2.waitKey()
	blurr_val = cv2.getTrackbarPos('blur','blur')
	canny = cv2.getTrackbarPos('canny','canny')
	size = cv2.getTrackbarPos('size','Low size')
	#size = size/10
	dst_val = cv2.getTrackbarPos('dst_va','dst_va')
	dst_val = dst_val/100
	cv2.destroyAllWindows()
	return (blurr_val,canny,size,dst_val)


from __future__ import absolute_import, division, print_function
import time
import logging
from collections import OrderedDict
from yokolibs import Transport
from yokolibs.Exceptions import Error, ErrorBadArgument, ErrorBadResponse

# This makes sure all classes are the new-style classes by default.
__metaclass__ = type # pylint: disable=invalid-name

_LOG = logging.getLogger("_yokobase")

# Special return value from a command function idicating that raw command should be executed upon
# function exit.
_CMD_CONTINUE = object

# The data items supported by all power meters.
_DATA_ITEMS = OrderedDict([
    ("V", "voltage"),
    ("I", "current"),
    ("P", "active power"),
    ("S", "apparent power"),
    ("Q", "reactive power"),
    ("Lambda", "power factor (λ)"),
    ("Phi", "phase difference (Φ)"),
    ("Fv", "voltage frequency"),
    ("Fi", "current frequency"),
    ("Wh", "watt-hours"),
    ("Whp", "positive watt-hours"),
    ("Whm", "negative watt-hours"),
    ("Ah", "ampere-hours"),
    ("Ahp", "positive ampere hours"),
    ("Ahm", "negative ampere hours"),
    ("Vmax", "maximum voltage"),
    ("Imax", "maximum current"),
    ("Time", "integration time"),
    ("Math", "value computed during integration"),
])

# Virtual data items are are generated on-the-fly in software.
_VDATA_ITEMS = OrderedDict([
    ("T", "time stamp at the end of the measurement interval"),
    ("J", "Joules (calculated as Power * Interval)"),
])

# Power meter math function names and description.
_MATH_NAMES = OrderedDict([
    ("cfv", "voltage crest factor"),
    ("cfi", "current crest factor"),
    ("add", "A+B"),
    ("sub", "A-B"),
    ("mul", "A*B"),
    ("div", "A/B"),
    ("diva", "A/B^2"),
    ("divb", "A^2/B"),
    ("avw", "Average active power"),
])

# The power meter commands common for all power meters.
# * property - whether this command just reads or changes a power meter configuration option or a
#              property. Can be 'None'.
# * property-descr - if the command is about reading/changing a property, this is a short human
#                    readable string describing the the property the command deals with. Can be
#                    'None'.
# * descr - human-readable description for the command.
#
# The following keys are added dynamically when the power meter object is initialized:
# * choices - list of all the possible values accepted or return by the command. Can be 'None'.
# * choices-set - same as choices, but of the 'set' type. Can be 'None'.
# * value-descr - human-readable text description of the possible values the command returns or
#                 accepts. Can be 'None'.
COMMANDS = OrderedDict([
    ("get-id", {
        "property-descr" : "device ID",
        "descr" : "get the device identification string",
    }),
    ("get-current-auto-range", {
        "property-descr" : "current auto range status",
        "descr" : "check whether the automatic current range feature is enabled",
    }),
    ("set-current-auto-range", {
        "property-descr" : "current auto range status",
        "descr" : "enable or disable the automatic current range feature",
    }),
    ("get-current-range", {
        "property-descr" : "current range (amperes)",
        "descr" : "get current range in amperes",
    }),
    ("set-current-range", {
        "property-descr" : "current range (amperes)",
        "descr" : "set current range in amperse",
    }),
    ("get-voltage-auto-range", {
        "property-descr" : "voltage auto range status",
        "descr" : "check whether the automatic voltage range feature is enabled",
    }),
    ("set-voltage-auto-range", {
        "property-descr" : "voltage auto range status",
        "descr" : "enable or disable the automatic voltage range feature",
    }),
    ("get-voltage-range", {
        "property-descr" : "voltage range (volts)",
        "descr" : "get voltage range in volts",
    }),
    ("set-voltage-range", {
        "property-descr" : "voltage range (volts)",
        "descr" : "set current range in volts",
    }),
    ("get-interval", {
        "property-descr" : "data update interval (seconds)",
        "descr" : "get the data update interval in seconds",
    }),
    ("set-interval", {
        "property-descr" : "data update interval (seconds)",
        "descr" : "set the data update interval in seconds",
    }),
    ("configure-data-items", {
        "property-descr" : None,
        "descr" : "set data items to read",
    }),
    ("wait-data-update", {
        "property-descr" : None,
        "descr" : "wait for data update",
    }),
    ("read-data", {
        "property-descr" : None,
        "descr" : "read power meter data",
    }),
    ("get-crest-factor", {
        "property-descr" : "crest factor",
        "descr" : "get crest factor",
    }),
    ("set-crest-factor", {
        "property-descr" : "crest factor",
        "descr" : "set crest factor",
    }),
    ("get-smoothing-status", {
        "property-descr" : "smoothing feature status",
        "descr" : "check whether the smoothing feature is enabled or disabled",
    }),
    ("set-smoothing-status", {
        "property-descr" : "smoothing feature status",
        "descr" : "enable or disable the smoothing feature",
    }),
    ("get-smoothing-type", {
        "property-descr" : "smoothing type",
        "descr" : "get smoothing type",
    }),
    ("set-smoothing-type", {
        "property-descr" : "smoothing type",
        "descr" : "set smoothing type",
    }),
    ("get-smoothing-factor", {
        "property-descr" : "smoothing factor",
        "descr" : "get the configured smoothing factor",
    }),
    ("set-smoothing-factor", {
        "property-descr" : "smoothing factor",
        "descr" : "set smoothing factor",
    }),
    ("get-integration-mode", {
        "property-descr" : "integration mode",
        "descr" : "get integration mode",
    }),
    ("set-integration-mode", {
        "property-descr" : "integration mode",
        "descr" : "set integration mode",
    }),
    ("get-integration-state", {
        "property-descr" : "integration state",
        "descr" : "get integration state",
    }),
    ("get-integration-timer", {
        "property-descr" : "integration timer value",
        "descr" : "get the integration timer value value",
    }),
    ("set-integration-timer", {
        "property-descr" : "integration timer value",
        "descr" : "get the integration timer value",
    }),
    ("start-integration", {
        "property-descr" : None,
        "descr" : "start integration",
    }),
    ("stop-integration", {
        "property-descr" : None,
        "descr" : "stop integration",
    }),
    ("reset-integration", {
        "property-descr" : None,
        "descr" : "reset integration",
    }),
    ("get-math", {
        "property-descr" : "computation function",
        "descr" : "get the currently configured computation function",
    }),
    ("set-math", {
        "property-descr" : "computation function",
        "descr" : "set the computation function",
    }),
    ("get-remote-mode", {
        "property-descr" : "remote mode status",
        "descr" : "check whether the remote mode is enabled or disabled",
    }),
    ("set-remote-mode", {
        "property-descr" : "remote mode status",
        "descr" : "enable or disable the remote mode",
    }),
    ("get-local-mode", {
        "property-descr" : "local mode status",
        "descr" : "check whether the local mode is enabled or disabled",
    }),
    ("set-local-mode", {
        "property-descr" : "local mode status",
        "descr" : "enable or disable the local mode",
    }),
    ("get-wiring-system", {
        "property-descr" : "wiring system type",
        "descr" : "get the wiring system type",
    }),
    ("factory-reset", {
        "property-descr" : None,
        "descr" : "reset to the factory default settings",
    }),
    ("calibrate", {
        "property-descr" : None,
        "descr" : "execute zero-level compensation",
    }),
    ("clear", {
        "property-descr" : None,
        "descr" : "clear the device output queue"
    }),
    ("get-installed-opts", {
        "property-descr" : "installed device options",
        "descr" : "get information about the installed device options",
    }),
    ("get-measurement-mode", {
        "property-descr" : "measurement mode",
        "descr" : "get the measurement mode",
    }),
    ("set-measurement-mode", {
        "property-descr" : "measurement mode",
        "descr" : "set the measurement mode",
    }),
    ("get-sync-source", {
        "property-descr"  : "synchronization source",
        "descr"  : "the information about the synchronization source",
    }),
    ("set-sync-source", {
        "property-descr"  : "synchronization source",
        "descr"  : "set the synchronization source",
    }),
    ("get-hold", {
        "property-descr" : "the 'hold' feature status",
        "descr"  : "check whether the 'hold' feture is on or off",
    }),
    ("set-hold", {
        "property-descr" : "the 'hold' feature status",
        "descr" : "switch the 'hold' feture is on or off",
    }),
    ("get-max-hold", {
        "property-descr" : "the 'max hold' feature status",
        "descr" : "check whether the 'max hold' feture is on or off",
    }),
    # WT210 says no help text for max hold on ./yokotool wt210 set max-hold
    ("set-max-hold", {
        "property-descr" : "the 'max hold' feature status",
        "descr" : "switch the 'max hold' feture is on or off",
    }),
    ("get-line-filter", {
        "property-descr" : "line filter status",
        "descr" : "check if the line filter is enabled or disabled",
    }),
    ("set-line-filter", {
        "property-descr" : "line filter status",
        "descr" : "enable or disable the line filter",
    }),
    ("get-freq-filter", {
        "property-descr" : "frequency filter status",
        "descr" : "check if the frequency filter is enabled or disabled",
    }),
    ("set-freq-filter", {
        "property-descr" : "frequency filter status",
        "descr" : "enable or disable the frequency filter",
    }),
])

_RAW_COMMANDS = (
    ("get-id", "*IDN?"),
    ("get-installed-opts", "*OPT?"),
    ("get-interval", ":SAMP:RATE?"),
    ("set-interval", ":SAMP:RATE"),
    ("get-hold", ":SAMP:HOLD?"),
    ("set-hold", ":SAMP:HOLD"),
    ("get-integration-mode", ":INTEG:MODE?"),
    ("set-integration-mode", ":INTEG:MODE"),
    ("get-integration-timer", ":INTEG:TIM?"),
    ("set-integration-timer", ":INTEG:TIM"),
    ("start-integration", ":INTEG:STAR"),
    ("stop-integration", ":INTEG:STOP"),
    ("reset-integration", ":INTEG:RES"),
    ("get-math", ":MATH?"),
    ("get-remote-mode", ":COMM:REM?"),
    ("set-remote-mode", ":COMM:REM"),
    ("get-local-mode", ":COMM:LOCK?"),
    ("set-local-mode", ":COMM:LOCK"),
    ("factory-reset", "*RST"),
    ("calibrate", "*CAL?"),
    ("get-error-status", ":STAT:ERR?"),
    ("clear", "\n*CLS"),
    ("set-verbose-errors", ":STAT:QMES"),
    ("set-headers", ":COMM:HEAD"),
    ("set-verbose-mode", ":COMM:VERB"),
    ("get-eesr", ":STAT:EESR?"),
)

# Valid arguments for the "enable/disable" type of commands.
ON_OFF_CHOICE = ("on", "off")

# The valid arguments for various commands.
_CHOICES = (
    {
        "commands" : ("get-current-range", "set-current-range"),
        # Note: the allowed values also depend on the crest-factor.
        "choices" : ("auto", "0.0025", "0.005", "0.01", "0.02", "0.05", "0.1", "0.2", "0.5", "1",
                     "2", "5", "10", "20"),
    },
    {
        "commands" : ("get-voltage-range", "set-voltage-range"),
        # Note: the allowed values also depend on the crest-factor.
        "choices" : ("7.5", "15", "30", "60", "75", "150", "300", "600"),
    },
    {
        "commands" : ("get-current-auto-range", "set-current-auto-range"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-voltage-auto-range", "set-voltage-auto-range"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-crest-factor", "set-crest-factor"),
        "choices" : ("3", "6"),
    },
    {
        "commands" : ("get-interval", "set-interval"),
        "choices" : ("0.1", "0.25", "0.5", "1", "2", "5"),
    },
    {
        "commands" : ("get-line-filter", "set-line-filter"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-freq-filter", "set-freq-filter"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-smoothing-status", "set-smoothing-status"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-smoothing-type", "set-smoothing-type"),
        "choices" : ("linear", "exponent"),
    },
    {
        "commands" : ("get-smoothing-factor", "set-smoothing-factor"),
        "choices" : ("8", "16", "32", "64"),
    },
    {
        "commands" : ("get-integration-mode", "set-integration-mode"),
        "choices" : ("normal", "continuous"),
    },
    {
        "commands" : ("get-measurement-mode", "set-measurement-mode"),
        "choices" : ("rms", "vmean", "dc"),
    },
    {
        "commands" : ("get-display-digits", "set-display-digits"),
        "choices" : ("4", "5"),
    },
    {
        "commands" : ("get-remote-mode", "set-remote-mode"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-local-mode", "set-local-mode"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-verbose-errors", "set-verbose-errors"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-verbose-mode", "set-verbose-mode"),
        "choices" : ON_OFF_CHOICE,
    },
    {
        "commands" : ("get-headers", "set-headers"),
        "choices" : ON_OFF_CHOICE,
    },
)

# The EESR register bits. It is important to keep this list ordered correctly.
_EESR_BITS = {
    "upd":  0,  # Measurement data is being updated.
    "itg":  1,  # Integration started.
    "itm":  2,  # Integration timer started.
    "ovrs": 3,  # Computation result overflow.
    "fov":  4,  # Frequency is out of range.
    "str":  5,  # 'storage' is ongoing.
    "ovr1": 6,  # Voltage or current on the 1st element is out of range.
    "pov1": 7,  # Peak voltage on the 1st element is out of range.
    "poa1": 8,  # Peak current on the 1st element is out of range.
    "ovr2": 9,  # Voltage or current on the 2nd element is out of range.
    "pov2": 10, # Peak voltage on the 2nd element is out of range.
    "poa2": 11, # Peak current on the 2nd element is out of range.
    "ovr3": 12, # Voltage or current on the 3rd element is out of range.
    "pov3": 13, # Peak voltage on the 3rd element is out of range.
    "poa3": 14, # Peak current on the 3rd element is out of range.
}

#
# Power meter's input and output values are not always very human-friendly. Below set of functions,
# referred to as 'tweaks', transform these values into a human-friendly format, and vice-versa, they
# also transform human-friendly notations into power meter's format.
#

def on_off_tweak(_, value):
    """Translate '0' and non-zero strings to 'off' and 'on' strings."""

    if value == "0":
        return "off"
    return "on"

def to_lower_tweak(_, value):
    """Translate 'value' to lowercase."""
    return value.lower()

def _to_lower_capitalize_tweak(_, value):
    """Translate 'value' to lowercase and capitalize it."""
    return value.lower().capitalize()

def _float_to_str_tweak(_, value):
    """Translate a float to string dropping superfluous zeros."""
    return "%g" % float(value)

def _success_failure_tweak(_, value):
    """Translate '0' and non-zero strings to 'success' and 'failure' strings."""

    if value == "0":
        return "success"
    return "failure"

def _csv_to_seconds_tweak(_, value):
    """Convert time from 'h,m,s' CSV format to seconds."""

    seconds = 0
    for item in value.split(','):
        seconds = seconds * 60 + int(item)
    return str(seconds)

def _seconds_to_csv_tweak(_, value):
    """Convert time from seconds to 'h,m,s' CSV format."""

    minutes, seconds = divmod(int(value), 60)
    hours, minutes = divmod(minutes, 60)
    return "%d,%d,%d" % (hours, minutes, seconds)

def _first_data_element_tweak(_, value):
    """Remove the ',1' ending from a data item."""

    if value.endswith(",1"):
        return value[:-2]
    return value

# The tweaks.
_TWEAKS = {
    "get-voltage-auto-range" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-current-auto-range" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-hold" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-max-hold" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-line-filter" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-freq-filter" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-smoothing-status" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-sync-source" : {
        "response-tweaks" : (to_lower_tweak,),
    },
    "get-measurement-mode" : {
        "response-tweaks" : (to_lower_tweak,),
    },
    "get-remote-mode" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-local-mode" : {
        "response-tweaks" : (on_off_tweak,),
    },
    "get-voltage-range" : {
        "response-tweaks" : (_float_to_str_tweak,),
    },
    "get-current-range" : {
        "response-tweaks" : (_float_to_str_tweak,),
    },
    "get-interval" : {
        "response-tweaks" : (_float_to_str_tweak,),
    },
    "set-interval" : {
        "input-tweaks" : (_float_to_str_tweak,),
    },
    "calibrate" : {
        "response-tweaks" : (_success_failure_tweak,),
    },
    "get-integration-mode" : {
        "response-tweaks" : (to_lower_tweak,),
    },
    "get-integration-timer" : {
        "response-tweaks" : (_csv_to_seconds_tweak,),
    },
    "set-integration-timer" : {
        "input-tweaks" : (_seconds_to_csv_tweak,),
    },
}

# Map a power meter error code into a human-readable message. We do not cover all codes here so far.
_ERROR_CODES_MAP = {
    813 : {"msg" : "operation is not allowed during integration, please reset integration first"},
}

def is_in_range(value, start=0, stop=0):
    """Verify whether or not a string 'value' contains an integer in the [min, max] range."""

    try:
        value = int(value)
    except (ValueError, TypeError):
        return False
    if value < start or value > stop:
        return False
    return True

def _verify_integration_time(value):
    """
    Verify whether or not the input value for integration time is an integer representing seconds,
    and if it is lower or equal to 10000 hours.
    """
    return is_in_range(value, 0, 10000 * 60 * 60)

def _cmd_to_str(cmd, arg):
    """Convert command 'cmd' with argument 'arg' to a string for printing."""

    if arg is not None:
        return "%s %s" % (cmd, arg)
    return cmd

class YokoBase():
    """
    This is the base class for all power meters. Each specific type of power meter should be derived
    from this class.
    """

    def _htop_tweak(self, _, value):
        """Translate data item in human notation to the protocol notation."""
        return self._ditt["htop"].get(value, value)

    def _ptoh_tweak(self, _, value):
        """Translate data item in protocol notation to the human notation."""
        return self._ditt["ptoh"].get(value, value)

    def _add_choices_from_dict(self, cmd, choices_dict):
        """
        Add "choices" and "value-descr" for command 'cmd' from the the 'choices_dict' dictionary.
        """

        self.commands[cmd]["choices"] = tuple(choices_dict)

        lines = []
        for name, descr in choices_dict.items():
            lines.append("%s - %s" % (name, descr))
        self.commands[cmd]["value-descr"] = "\n".join(lines)

    def _iter_data_item_commands(self):
        """
        Yield the (cmd_part, get_cmd, set_cmd) tuples for each possible data item command. Must be
        implemented by the child class.
        """

    def _populate_choices(self, choices):
        """Populate the valid values for various WT310 commands to 'self.commands'."""

        cmds = self.commands
        for info in _CHOICES:
            for cmd in info["commands"]:
                if cmd in cmds:
                    cmds[cmd]["choices"] = info["choices"]

        for info in choices:
            for cmd in info["commands"]:
                if cmd in cmds:
                    cmds[cmd]["choices"] = info["choices"]

        self._add_choices_from_dict("read-data", self._data_items)
        self._add_choices_from_dict("configure-data-items", self._data_items)
        self._add_choices_from_dict("get-math", _MATH_NAMES)
        self._add_choices_from_dict("set-math", _MATH_NAMES)

        descr = "integer amount of seconds (0-10000 hours)"
        cmds["get-integration-timer"]["value-descr"] = descr
        cmds["set-integration-timer"]["value-descr"] = descr

        for info in cmds.values():
            if "choices" not in info:
                info["choices"] = None
                info["choices-set"] = None
            if "choices-set" not in info:
                info["choices-set"] = set(info["choices"])
            if "value-descr" not in info:
                info["value-descr"] = None

    def _populate_data_items(self, items, pairs):
        """
        Initialize data items dictionary and build the data items translation tables which is needed
        because often human name for the data item is different to the protocol name and we
        translate the name. This helper creates and returns the data item translation table, which
        is just a dictionary that can be used to quickly get human name by the protocol name and
        vice versa.  The 'pairs' input argument is an iterable containing the (human name, protocol
        name) pairs.
        """

        self._data_items = _DATA_ITEMS
        for name, descr in items.items():
            self._data_items[name] = descr
        for name, descr in _VDATA_ITEMS.items():
            self._data_items[name] = descr

        self._ditt = {}
        self._ditt["htop"] = {}
        self._ditt["ptoh"] = {}
        for hname, pname in pairs:
            self._ditt["htop"][hname] = pname
            self._ditt["ptoh"][pname] = hname

    def _populate_raw_commands(self, raw_commands):
        """Populate the raw (wire) power meter commands to 'self._commands'."""

        for cmd, raw_cmd in _RAW_COMMANDS:
            self._commands[cmd] = {}
            self._commands[cmd]["raw-cmd"] = raw_cmd

        for cmd, raw_cmd in raw_commands:
            self._commands[cmd] = {}
            self._commands[cmd]["raw-cmd"] = raw_cmd

    def _populate_tweaks(self, tweaks):
        """
        Pupulate the tweaks to 'self._commands'. The tweaks are functions applied to command
        arguments before running the command or to the results of a command before returning the
        results to the end user.
        """

        for cmd, info in _TWEAKS.items():
            for key, value in info.items():
                self._commands[cmd][key] = value

        for cmd, info in tweaks.items():
            for key, value in info.items():
                self._commands[cmd][key] = value

        self._commands["read-data"]["response-tweaks"] = (self._get_data_tweak,)

        # Cover the data item get/set commands as well.
        response_tweaks = (_first_data_element_tweak, _to_lower_capitalize_tweak, self._ptoh_tweak)
        input_tweaks = (_to_lower_capitalize_tweak, self._htop_tweak)
        for _, get_cmd, set_cmd in self._iter_data_item_commands():
            self._commands[get_cmd]["response-tweaks"] = response_tweaks
            self._commands[set_cmd]["input-tweaks"] = input_tweaks

    def _populate_arg_verify_funcs(self):
        """Populate arugment verification functions. They are executed before the command is run."""

        self._commands["set-integration-timer"]["verify-arg"] = _verify_integration_time

    def _add_command_func(self, cmd, func):
        """
        Some commands require a handler function, this helpers adds one to the internal
        'self._commands' dictionary.
        """

        if cmd not in self._commands:
            self._commands[cmd] = {}
        if "raw-cmd" not in self._commands[cmd]:
            self._commands[cmd]["raw-cmd"] = None
        self._commands[cmd]["func"] = func

    def _populate_raw_commands_post(self):
        """Add post process and add common raw commands."""

        # Cover the EESR-related commands (one command per a EESR bit).
        for name, bit in self._eesr_bits.items():
            cmd = "set-eesr-filter-%s" % name
            self._commands[cmd] = {}
            self._commands[cmd]["raw-cmd"] = ":STAT:FILT%d" % (bit + 1)
            cmd = "eesr-wait-%s" % name
            self._commands[cmd] = {}
            self._commands[cmd]["raw-cmd"] = ":COMM:WAIT %d" % (bit + 1)

        self._add_command_func("wait-data-update", self._wait_data_update_cmd)
        self._add_command_func("get-current-range", self._get_range_cmd)
        self._add_command_func("set-current-range", self._set_range_cmd)
        self._add_command_func("get-voltage-range", self._get_range_cmd)
        self._add_command_func("set-voltage-range", self._set_range_cmd)
        self._add_command_func("start-integration", self._start_integration_cmd)

        if hasattr(self._transport, "set_timeout"):
            self._add_command_func("set-interval", self._set_interval_timeout_cmd)

        for cmd, info in self._commands.items():
            if cmd.startswith("get-") or (info["raw-cmd"] and info["raw-cmd"].endswith("?")):
                info["has-response"] = True
                info["has-argument"] = False
            else:
                info["has-response"] = False
                info["has-argument"] = True

    def _populate_errors_map_map(self):
        """
        Error codes map mapes power meter error code number either to a human-readable message or to
        an error handler that may either generate a human-readable message or handle the error
        condition.
        """

        self._errors_map = _ERROR_CODES_MAP
        self._errors_map[842] = self._errors_map[844] = self._errors_map[845] = \
                {"func" : self._integration_error_handler}

    @staticmethod
    def _integration_error_handler(cmd, arg, code, rawmsg): # pylint: disable=unused-argument
        """This function implements error handler for the integration commands."""

        if code == 845:
            # start -> reset
            return "current integration state is 'start' and it cannot be changed to 'reset', " \
                   "please stop it first"
        if code == 842:
            # start -> start
            return "integration is already in the 'start' state"
        if code == 844:
            # This is either 'reset->stop' or 'stop->stop'.
            return "cannot stop integration because it is not in the 'start' state, please " \
                   "start it first"

        return ""

    def _prepare_data_items_to_read(self, items):
        """
        The input 'items' list is a mix of pyisical and vitual data items. The former are provided
        by the power meter and the latter are computed in software. This function constructs and
        returns a list of physical data items from 'items'. We also avoid reading the same data item
        more than once with help of this function.
        """

        result = []
        self._item_indexes = {}

        idx = 0
        for item in items:
            # If users request to read the 'Joules' virtual data item, we read the and will use
            # this value to later compute the Joules.
            if item == "J":
                item = "P"
            if item not in _VDATA_ITEMS and item not in self._item_indexes:
                result.append(item)
                self._item_indexes[item] = idx
                idx += 1

        return result

    def _get_data_tweak(self, _, response):
        """Inject the virtual data items into power meter read response."""

        timestamp = time.time()
        items = []

        for item in self._items_to_read:
            if item not in _VDATA_ITEMS:
                items.append(response[self._item_indexes[item]])
            elif item == "T":
                items.append(str(timestamp))
            elif item == "J":
                items.append(str(float(response[self._item_indexes["P"]]) * self._interval))

        return items

    def _configure_data_items_cmd(self, _, items):
        """Set the data items that will be read by next "get-data" command."""

        self._items_to_read = items

        if len(items) > self.max_data_items:
            raise Error("too many data items, please, specify at most %s" % self.max_data_items)

        for item in items:
            if item not in self._data_items:
                msg = "bad data item '%s'" % item
                raise ErrorBadArgument(None, None, msg=msg)

        # The interval is needed to compute the Joules virtual data item.
        self._interval = float(self._command("get-interval"))
        items = self._prepare_data_items_to_read(items)
        _LOG.debug("data items to read from power meter: %s", ",".join(items))

        # Configure a trigger for the "UPD" bit changing from 1 to 0, which happens when data update
        # finishes.
        self._command("set-eesr-filter-upd", "fall")
        return items

    def _wait_data_update_cmd(self, _, __):
        """Wait until WT210 updates the data."""

        # Note, we use 'check_status=False' in this function because the status check commands may
        # take take long time in case of a low baud rate serial connection and we may be late when
        # the update interval is short.

        # Clear EESR by reading it.
        self._command("get-eesr", check_status=False)
        # Wait for the event.
        self._command("eesr-wait-upd", check_status=False)

    def _set_interval_timeout_cmd(self, _, arg):
        """Make sure transport device's timeout is larger than the interval."""

        # Set the timeout to 2x interval length as a margin of safty.
        timeout = int(float(arg) * 1000) * 2
        self._transport.set_timeout(timeout)

        return _CMD_CONTINUE

    def _get_range_cmd(self, cmd, _):
        """Implements the 'get-current-range' and 'get-voltage-range' commands."""

        result = self._command(cmd, func=False)
        auto = self._command(cmd.replace("-range", "-auto-range"))

        if auto == "on":
            result += " (auto)"
        return result

    def _set_range_cmd(self, cmd, arg):
        """Implements the 'set-current-range' and 'set-voltage-range' commands."""

        auto_range_cmd = cmd.replace("-range", "-auto-range")
        if arg == "auto":
            self._command(auto_range_cmd, "on")
            return

        self._command(auto_range_cmd, "off")
        choices = self.commands[cmd]["choices"]
        if arg in (choices[1], choices[-1]):
            # The first and the last current/voltage range availability depends on the crest factor.
            what = cmd.split("-")[1]
            crest = self._command("get-crest-factor")
            crest_needed = None
            if crest == "3" and arg == choices[1]:
                crest_needed = "6"
            elif crest == "6" and arg == choices[-1]:
                crest_needed = "3"
            if crest_needed:
                raise Error("%s range %s is only available when crest factor is %s, but currently "
                            "it is %s" % (what, arg, crest_needed, crest))

        self._command(cmd, arg, func=False)

    def _start_integration_cmd(self, cmd, _):
        """
        Adds a delay after the 'start' integration command. The delay is necessary because some of
        the subsequent commands may fail if executed right after integration had been started. There
        is some sort of the internal integrator busy period which we let pass by introducing a short
        delay.
        """

        self._command(cmd, func=False)
        time.sleep(0.2)

    def _apply_response_tweaks(self, cmd, response):
        """
        Tweak 'response' as defined within a specific type of power meter, then return the tweaked
        argument.
        """

        if "response-tweaks" in self._commands[cmd]:
            for tweak_func in self._commands[cmd]["response-tweaks"]:
                response = tweak_func(cmd, response)
        return response

    def _apply_input_tweaks(self, cmd, arg):
        """
        Tweak 'input' as defined within a specific type of power meter, then return the tweaked
        argument.
        """

        if "input-tweaks" in self._commands[cmd]:
            for tweak_func in self._commands[cmd]["input-tweaks"]:
                arg = tweak_func(cmd, arg)
        return arg

    def _verify_argument(self, cmd, arg):
        """Verify whether or not 'arg' argument is valid for 'cmd' command."""

        choices = self.commands[cmd]["choices-set"]
        if choices:
            # 'arg' may be a list, in which case we check every element of the list.
            if isinstance(arg, (list, tuple)):
                for elt in arg:
                    tweaked_elt = self._apply_input_tweaks(cmd, elt)
                    if tweaked_elt not in choices:
                        raise ErrorBadArgument(cmd, elt)
            else:
                tweaked_arg = self._apply_input_tweaks(cmd, arg)
                if tweaked_arg not in choices:
                    raise ErrorBadArgument(cmd, arg)

        if "verify-arg" in self._commands[cmd]:
            func = self._commands[cmd]["verify-arg"]
            if not func(arg):
                raise ErrorBadArgument(cmd, arg)

        return True

    def _check_error_status(self, cmd, arg):
        """
        Check whether the power meter error status and possibly apply the error handlers. Returns
        'None' if there were no errors (or the error was handled) and the error message otherwise.
        """

        status_cmd = self._commands["get-error-status"]["raw-cmd"]
        try:
            response = self._transport.queryline(status_cmd)
        except Transport.Error as err:
            raise type(err)("failed to check error status of command '%s':\n%s\nRaw command was "
                            "'%s'" % (_cmd_to_str(cmd, arg), err, status_cmd))

        try:
            code, rawmsg = response.split(',', 1)
            code = int(code)
        except ValueError as err:
            raise ErrorBadResponse(raw_cmd=status_cmd, response=response)

        if code == 0:
            return None

        if code in self._errors_map:
            if "func" in self._errors_map[code]:
                msg = self._errors_map[code]["func"](cmd, arg, code, rawmsg)
                if not msg:
                    return None
                if msg == "":
                    # Empty string means that the error handler could generate a customized error
                    # message and the default one should be used.
                    msg = response
            else:
                msg = self._errors_map[code]["msg"]
        else:
            msg = response

        return "command '%s' failed:\n%s" % (_cmd_to_str(cmd, arg), msg)

    def _command(self, cmd, arg=None, check_status=True, func=True):
        """
        Actually execute the command, the tweaks, and the status checks, unless 'check_status' is
        'False'. Some commands have both the "raw-cmd" and "func" keys in their 'self._command'
        description, and the 'func' argument whether the function should be executed or the raw
        command should be executed.
        """

        _LOG.debug(_cmd_to_str(cmd, arg))

        if func and "func" in self._commands[cmd]:
            retval = self._commands[cmd]["func"](cmd, arg)
            if retval is not _CMD_CONTINUE:
                return retval

        if arg is not None:
            arg = self._apply_input_tweaks(cmd, arg)

        raw_cmd = self._commands[cmd]["raw-cmd"]
        if arg is not None:
            raw_cmd += " %s" % arg

        try:
            self._transport.writeline(raw_cmd)
        except Transport.Error as err:
            raise type(err)("failed to write command '%s' to the power meter:\n%s\nRaw command "
                            "was '%s'" % (cmd, err, raw_cmd))
        response = None
        if self._commands[cmd]["has-response"]:
            try:
                response = self._transport.readline()
            except Transport.Error as err:
                raise type(err)("failed to read power meter response to '%s':\n%s\nRaw command was "
                                "'%s'" % (_cmd_to_str(cmd, arg), err, raw_cmd))
            response = self._apply_response_tweaks(cmd, response)

        if check_status:
            msg = self._check_error_status(cmd, arg)
            if msg:
                raise Error(msg)

        return response

    def command(self, cmd, arg=None):
        """
        Execute the power meter command 'cmd' with argument 'arg' if it is not null. Return the
        command response or 'None' if the command has no response. 'cmd' should be a string, 'arg'
        can be of any type since 'command()' handles the typecast to string.
        """

        if not isinstance(cmd, str) or cmd not in self.commands:
            raise Error("bad command '%s'" % cmd)

        # We allow 'arg' to be of different types and convert it into a string
        if arg is not None:
            if isinstance(arg, list):
                for idx, item in enumerate(arg):
                    if not isinstance(item, str):
                        arg[idx] = str(item)
            elif not isinstance(arg, str):
                arg = str(arg)

        # Sanity checks.
        if self._commands[cmd]["has-argument"]:
            self._verify_argument(cmd, arg)
        elif arg is not None:
            raise Error("command '%s' accepts no arguments, but '%s' was provided" % (cmd, arg))

        return self._command(cmd, arg)

    def reset(self, configure=True):
        """
        Reset the power meter and if 'configure' is 'True', also configure with reasonable defaults.
        """

        # Reset to factory settings.
        self.command("clear")
        self.command("factory-reset")
        if not configure:
            return

        # Set crest factor to 3 for better measurement precision.
        self.command("set-crest-factor", "3")
        # We are measuring AC, so need the RMS mode.
        self.command("set-measurement-mode", "rms")
        # Autmatically find the suitable voltage and current ranges.
        self.command("set-voltage-auto-range", "on")
        self.command("set-current-auto-range", "on")
        # Set the measurement interval to 1 second.
        self.command("set-interval", 1)
        # Disable line filer to inclue all the frequencies into the measurements.
        self.command("set-line-filter", "off")
        # Enable the frequency filter. It only affects frequency measurements and generally
        # recommended to be enabled.
        self.command("set-freq-filter", "on")

    def _init_pmeter(self):
        """Initialize the power meter."""

        # Clear the output and error queues of the power meter. The first command may fail with the
        # "interrupted" error if the power meter is currently expencting the results of the previous
        # command to be read, so clear 2 times.
        try:
            self._command("clear")
        except Error:
            self._command("clear")

        # Make sure that in case of error the device sends verbose error strings, not just the
        # status code.
        self._command("set-verbose-errors", "on")
        # Disable headers in responses.
        self._command("set-headers", "off")

        # Clear all the EESR trigger conditions.
        for name in self._eesr_bits:
            self._command("set-eesr-filter-%s" % name, "never")

    def __init__(self, transport):
        """The class constructor. The 'transport' argument is the power meter transport object."""

        self._transport = transport
        self._eesr_bits = _EESR_BITS

        # The data items translation table.
        self._ditt = None
        # Create the user-visible commands dictionary.
        self.commands = COMMANDS
        # The private commands dictionary.
        self._commands = {}
        # List of items configured to be read by 'configure-data-items'.
        self._items_to_read = []
        # Indexes of the data items that we are reading.
        self._item_indexes = None
        # All the supported data items, including the virtual ones.
        self._data_items = None
        # Virtual data items.
        self._vdata_items = _VDATA_ITEMS
        # Maximum count of data items that can be read at the same time.
        self.max_data_items = None
        # Saved value of the power meter data update interval.
        self._interval = None
        # Messages and actions in case power meter reports and error.
        self._errors_map = None

    def __del__(self):
        """The class destructor."""

        try:
            self.close()
        except Exception:
            pass

    def close(self):
        """Close the communication interface with the power meter."""

        if getattr(self, "_transport", None):
            self._transport.close()
            self._transport = None


import torch
from diffusers import DiffusionPipeline

class DiffusionInferencePipeline(DiffusionPipeline):
    def __init__(self, network, scheduler, num_inference_timesteps=1000):
        super().__init__()

        self.register_modules(network=network, scheduler=scheduler)
        self.num_inference_timesteps = num_inference_timesteps

    @torch.inference_mode()
    def __call__(
        self,
        initial_noise: torch.Tensor,
        conditioner: torch.Tensor = None,
    ):
        r"""
        Args:
            initial_noise: The initial noise to be denoised.
            conditioner:The conditioner.
            n_inference_steps: The number of denoising steps. More denoising steps
                usually lead to a higher quality at the expense of slower inference.
        """

        mel = initial_noise
        batch_size = mel.size(0)
        self.scheduler.set_timesteps(self.num_inference_timesteps)

        for t in self.progress_bar(self.scheduler.timesteps):
            timestep = torch.full((batch_size,), t, device=mel.device, dtype=torch.long)

            # 1. predict noise model_output
            model_output = self.network(mel, timestep, conditioner)

            # 2. denoise, compute previous step: x_t -> x_t-1
            mel = self.scheduler.step(model_output, t, mel).prev_sample

            # 3. clamp
            mel = mel.clamp(-1.0, 1.0)

        return mel


import datetime
import numpy as np

from src.algorithms import utilities

def general(all_contexts, all_rewards, max_num_observations, s_o_max_general):
    time_horizon = all_rewards.shape[0]
    number_of_actions = all_rewards.shape[1]
    context_dimensionality = all_contexts.shape[1]

    all_perms = utilities.perm_construct(context_dimensionality, max_num_observations)
    number_of_perms_general = all_perms.shape[0]

    selected_context_general = np.zeros((time_horizon, context_dimensionality))
    selected_action_general = np.zeros(time_horizon)
    all_gain_general = np.zeros(time_horizon + 1)

    number_of_unique_state_s_when_applying_perm_i = np.zeros((number_of_perms_general, s_o_max_general))

    # d_os = np.zeros((number_of_perms_general, s_o_max_general)) #Definition: N_os / N_o
    true_prob_so = np.zeros((number_of_perms_general,
                             s_o_max_general))  # Definition: number_of_unique_state_s_when_applying_perm_i / time_horizon

    S_Size = np.zeros(number_of_perms_general)

    true_average_reward = np.zeros((number_of_perms_general, s_o_max_general, number_of_actions))

    number_of_visits_for_average_reward = np.zeros((number_of_perms_general, s_o_max_general, number_of_actions))

    sum_of_rewards = np.zeros((number_of_perms_general, s_o_max_general, number_of_actions))

    for i in range(number_of_perms_general):

        one_perm = all_perms[i]

        O_temp = np.tile(one_perm, [time_horizon, 1])

        all_contexts_temp = all_contexts * O_temp  # a*b element.wise multiplication, np.multiply(a,b) is also correct

        # S_i is the matrix whose rows are the unique(between already existing contexts) state realizations of all of the contexts with observation action one_perm[i]
        S_i = np.unique(all_contexts_temp, axis=0)

        # number of unique state realizations for each domain
        # it has a subtle difference with s_o[i] = number of different states(realizations) with the same observation action all_perms[i]
        S_Size[i] = S_i.shape[0]

        for j in range(int(S_Size[i])):

            all_unique_state_s_when_applying_perm_i = (all_contexts_temp == S_i[j, :]).all(1)
            number_of_unique_state_s_when_applying_perm_i[i, j] = number_of_unique_state_s_when_applying_perm_i[
                                                                      i, j] + np.sum(
                all_unique_state_s_when_applying_perm_i)

            true_prob_so[i, j] = number_of_unique_state_s_when_applying_perm_i[i, j] / time_horizon

            all_rewards_temp = all_rewards[all_unique_state_s_when_applying_perm_i, :]

            for k in range(number_of_actions):
                number_of_visits_for_average_reward[i, j, k] = all_rewards_temp[:, k].shape[0]
                sum_of_rewards[i, j, k] = np.sum(all_rewards_temp[:, k])

                true_average_reward[i, j, k] = sum_of_rewards[i, j, k] / number_of_visits_for_average_reward[i, j, k]

    return [true_prob_so, true_average_reward, S_Size]

class SimOOS_Oracle:
    """Fixed-I oracle policy, as defined in SimOOS paper
    "Data-Driven Online Recommender Systems with Costly Information Acquisition"
    Atan et al. 2021

    Different from SimOOS algorithm in that it has access to true probabilites of observing partial state vectors,
    has true expected rewards for a given partial vector and arm.
    Also for this oracle best observation is chosen and fixed for the whole time horizon.
    """
    def __init__(self,
                 all_contexts: np.array,
                 all_rewards: np.array,
                 cost_vector: np.array,
                 number_of_actions: int,
                 max_num_observations: int,
                 beta: float,
                 ):

        self.name = f"SimOOS-Oracle (beta={beta})"

        self.time_horizon = all_contexts.shape[0]
        self.context_dimensionality = all_contexts.shape[1]
        self.max_num_observations = max_num_observations
        self.number_of_actions = number_of_actions
        self.beta = beta

        # All possible subsets of features (I in paper)
        self.all_perms = utilities.perm_construct(self.context_dimensionality, self.max_num_observations)
        self.perm_to_index = {}
        for i, perm in enumerate(self.all_perms):
            self.perm_to_index[tuple(perm)] = i

        self.number_of_perms = self.all_perms.shape[0]
        self.s_o = np.zeros(self.number_of_perms)

        self.selected_context = np.zeros((self.time_horizon, self.context_dimensionality))
        self.selected_action = np.zeros(self.time_horizon)
        self.all_gain = np.zeros(self.time_horizon + 1)

        self.collected_gains = np.zeros(self.time_horizon)
        self.collected_rewards = np.zeros(self.time_horizon)
        self.collected_costs = np.zeros(self.time_horizon)

        # debug variables
        self.states = np.zeros(self.time_horizon)
        self.true_average_rewards = np.zeros((self.time_horizon, self.number_of_actions))

        ########################################################################################
        self.feature_values, self.all_feature_counts = utilities.save_feature_values(all_contexts)

        for i in range(self.number_of_perms):
            # psi[i] = number of different partial vectors(realizations) with given observation action self.all_perms[i]
            # How many partial vectors with support given by self.all_perms[i].
            # Equal to cardinality of Psi(I) in the paper. Used to determine Psi_total, for confidence bounds.

            # s_o[i] - size of state array for a given observation action. It is bigger than psi[i] because
            # it also includes states which have None for observed features (although they are unreachable).
            self.s_o[i] = utilities.state_construct(self.all_feature_counts, all_contexts, self.all_perms[i])

        # s_o_max - the largest state vector for all observations, needed to create arrays.
        self.s_o_max = int(np.amax(self.s_o))
        self.Psi_total = int(np.sum(self.s_o))

        # Oracle variables (not present in SimOOS)
        self.true_prob_so, self.true_average_reward, self.S_Size = general(
            all_contexts, all_rewards, self.max_num_observations, self.s_o_max
        )

        self.r_star = np.zeros((self.number_of_perms, self.s_o_max))
        self.action_star = np.zeros((self.number_of_perms, self.s_o_max))

        for i in range(self.number_of_perms):

            for j in range(int(self.S_Size[i])):
                self.r_star[i, j] = np.max(self.true_average_reward[i, j, :])

                self.action_star[i, j] = np.argmax(self.true_average_reward[i, j, :])

        self.value_o = np.zeros(self.number_of_perms)

        for i in range(self.number_of_perms):
            self.value_o[i] = self.beta * np.dot(self.true_prob_so[i, :], self.r_star[i, :]) - np.dot(self.all_perms[i],
                                                                                cost_vector)

        self.index_of_observation_action_star = np.argmax(self.value_o)

        self.selected_observation_action_at_t = self.all_perms[self.index_of_observation_action_star]

    def choose_features_to_observe(self, t, feature_indices, cost_vector):
        self.observation_action_at_t = self.selected_observation_action_at_t
        return [ind for ind, value in enumerate(self.observation_action_at_t) if value]

    def choose_arm(self, t: int, context_at_t: np.array, pool_indices: list):
        """Return best arm's index relative to the pool of available arms.

        Args:
            t: number of trial
            context_at_t: user context at time t. One for each trial, arms don't have contexts.
            pool_indices: indices of arms available at time t.
        """
        s_t = utilities.state_extract(self.feature_values, self.all_feature_counts, context_at_t, self.observation_action_at_t)
        self.states[t] = s_t
        self.true_average_rewards[t, :] = self.true_average_reward[self.index_of_observation_action_star, s_t]

        self.action_at_t = int(self.action_star[self.index_of_observation_action_star, s_t])
        return pool_indices.index(self.action_at_t)

    def update(self, t, action_index_at_t, reward_at_t, cost_vector_at_t, context_at_t, pool_indices):
        if t % 500 == 0:
            print(f"Trial {t}, time {datetime.datetime.now()}")

        cost_at_t = np.dot(cost_vector_at_t, self.observation_action_at_t)

        action_at_t = pool_indices[action_index_at_t]

        self.selected_context[t, :] = self.selected_observation_action_at_t

        self.all_gain[t + 1] = self.all_gain[t] + reward_at_t - cost_at_t

        self.selected_action[t] = action_at_t

        self.collected_gains[t] = reward_at_t - cost_at_t
        self.collected_rewards[t] = reward_at_t
        self.collected_costs[t] = cost_at_t


import numpy as np
import numpy.ma as ma
from netCDF4 import Dataset
import os
import os.path
import scipy.interpolate as interp
import scipy.stats as stats
from config import *

## PART 1: Import data and combine

# Import model files for RCP 2.6 and 8.5, replacing masked data with nans
def importData(dataFiles):
	data = []
	for dataFileName in dataFiles:
		dataFile = dataDir + "/" + dataFileName
		# Because latitudes and logitudes are of different sizes, we must interpolate.
		dataRaw = Dataset(dataFile, "r").variables[varOfInterest][:]
		# Loop through time, interpolating latitude and longitude
		dataInterp = np.zeros((numMonths, latitudeRes, longitudeRes))
		for i in range(numMonths):
			dataInterpGen = interp.interp2d(np.arange(dataRaw.shape[2]), np.arange(dataRaw.shape[1]), dataRaw[i])
			dataInterp[i] = dataInterpGen(np.linspace(0, dataRaw.shape[2], longitudeRes), np.linspace(0, dataRaw.shape[1], latitudeRes))
		data += [dataInterp]
	# Stack, average, and return
	return np.stack(data,axis=0).mean(axis=0)

# Create cache, try to import from cache, and if it isn't possible, import from data instead
if not os.path.exists(cacheDir):
	os.mkdir(cacheDir)
if not os.path.exists(cacheDir + '/rcpData.npz'):
	np.savez_compressed(cacheDir + '/rcpData.npz', rcp26=importData(dataFilesRCP26), rcp85=importData(dataFilesRCP85))
allData = np.load(cacheDir + '/rcpData.npz')
dataRCP26 = allData['rcp26']
dataRCP85 = allData['rcp85']

## PART 2: Process data

# Process data if it wasn't cached
if not os.path.exists(cacheDir + '/processedData.npz'):
	# Remove irrelavent realms using ocean data
	landSea = Dataset(dataDir + "/" + landSeaFile,'r')
	landSeaData = landSea.variables['LSMASK'][:]
	landSeaInterpGen = interp.interp2d(np.arange(landSeaData.shape[1]), np.arange(landSeaData.shape[0]), landSeaData)
	landSeaDataInterp = landSeaInterpGen(np.linspace(0, landSeaData.shape[1], longitudeRes), np.linspace(0, landSeaData.shape[0], latitudeRes))
	# Set mask according to realm specified in config
	if realm == 'land':
		landSeaMask = np.tile(landSeaDataInterp == 0, (numMonths, 1, 1))
	elif realm == 'sea':
		landSeaMask = np.tile(landSeaDataInterp != 0, (numMonths, 1, 1))
	elif realm == 'atmos':
		landSeaMask = False

	# Apply mask
	dataRCP26 = ma.masked_array(dataRCP26, mask=landSeaMask)
	dataRCP85 = ma.masked_array(dataRCP85, mask=landSeaMask)

	# Get difference between the two RCPs and between each RCP and its initial value and divide it to a difference (-1 for -100% and +1 for +100%) for our difference graph.
	def getPercentDifference(data1, data2):
		dataDiff = data1 - data2
		return dataDiff/data2
	# Difference from initial
	diffRCP26 = getPercentDifference(dataRCP26, dataRCP26[0,:])
	diffRCP85 = getPercentDifference(dataRCP85, dataRCP85[0,:])
	diffRCPs = getPercentDifference(dataRCP85, dataRCP26)

	# Change data to rankings, replacing oceans with -inf, then mask oceans (again; stats.rankdata doesn't like masks.) They'll come as the lowest value.
	percRCP26 = stats.rankdata(ma.filled(dataRCP26, -np.inf), method='dense').reshape(dataRCP26.shape)
	percRCP85 = stats.rankdata(ma.filled(dataRCP85, -np.inf), method='dense').reshape(dataRCP85.shape)
	percRCP26 = ma.masked_array(percRCP26, mask=(percRCP26 == 1))
	percRCP85 = ma.masked_array(percRCP85, mask=(percRCP85 == 1))

	# Normalize data to between 0 and 1 and reverse it to make them into precentiles
	minVal = min(percRCP26.min(), percRCP85.min())
	maxVal = max(percRCP26.max(), percRCP85.max())
	rangeVal = maxVal-minVal
	percRCP26 = (percRCP26 - minVal)/rangeVal
	percRCP85 = (percRCP85 - minVal)/rangeVal

	# Cache our processed data
	if realm == 'atmos':
		np.savez_compressed(cacheDir + '/processedData.npz',
			perc26=percRCP26, perc26_mask=percRCP26.mask,
			perc85=percRCP85, perc85_mask=percRCP85.mask,
			diff26=diffRCP26, diff26_mask=diffRCP26.mask,
			diff85=diffRCP85, diff85_mask=diffRCP85.mask,
			diffRCP=diffRCPs, diffRCP_mask=diffRCPs.mask,
			landSea=landSeaDataInterp)
	else: # Realm is land or sea
		np.savez_compressed(cacheDir + '/processedData.npz',
			perc26=percRCP26, perc26_mask=percRCP26.mask,
			perc85=percRCP85, perc85_mask=percRCP85.mask,
			diff26=diffRCP26, diff26_mask=diffRCP26.mask,
			diff85=diffRCP85, diff85_mask=diffRCP85.mask,
			diffRCP=diffRCPs, diffRCP_mask=diffRCPs.mask)

print("Done. Run plot.py to show.")


import pandas as pd
from statistics import mean, stdev

def getStats(df, survivaldict):
	col = 'subtypes'
	list1 = df.loc[df[col] == 0, 'samples']
	list2 = df.loc[df[col] == 1, 'samples']
	list3 = df.loc[df[col] == 2, 'samples']
	list4 = df.loc[df[col] == 3, 'samples']
	list5 = df.loc[df[col] == 4, 'samples']
	vals1 = [float(survivaldict[i]) for i in list1 if (float(survivaldict[i]) > 0)]
	vals2 = [float(survivaldict[i]) for i in list2 if (float(survivaldict[i]) > 0)]
	vals3 = [float(survivaldict[i]) for i in list3 if (float(survivaldict[i]) > 0)]
	vals4 = [float(survivaldict[i]) for i in list4 if (float(survivaldict[i]) > 0)]
	vals5 = [float(survivaldict[i]) for i in list5 if (float(survivaldict[i]) > 0)]
	if vals1:
		mean1 = mean(vals1)
		std1 = stdev(vals1)
	else:
		mean1 = 'NA'
		std1 = ''

	if vals2:
		mean2 = mean(vals2)
		std2 = stdev(vals2)
	else:
		mean2 = 'NA'
		std2 = ''

	if vals3:
		mean3 = mean(vals3)
		std3 = stdev(vals3)
	else:
		mean3 = 'NA'
		std3 = ''
	if vals4:
		mean4 = mean(vals4)
		std4 = stdev(vals4)
	else:
		mean4 = 'NA'
		std4 = ''
	if vals5:
		mean5 = mean(vals5)
		std5 = stdev(vals5)
	else:
		mean5 = 'NA'
		std5 = ''
	print('Survival (months): ')
	print('Yellow: ', mean1, "+/-", std1)
	print('Blue: ', mean2, "+/-", std2)
	print('Green: ', mean3, "+/-", std3)
	print('Cyan: ', mean4, "+/-", std4)
	print('Red: ', mean5, "+/-", std5)

def getsubtypes():
	df = pd.read_csv('novel_results.csv')
	del df['Unnamed: 0']
	return df

def getmeta():
	f = open('GSE7696_series_matrix.txt')
	lines = f.readlines()
	i=0
	for line in lines:
		if "!Sample_geo_accession" in line:
			samples = line.split()
			samples.remove('!Sample_geo_accession')
			samples = [str(i).replace('"', '') for i in samples]
		if '!Sample_characteristics_ch1' in line:
			if 'survival time in months' in line:
				pre = line.split('\t')
				pre.remove('!Sample_characteristics_ch1')
				surv = [str(i).replace('survival time in months: ', '') for i in pre]
				surv = [str(i).replace('NA', '-100') for i in surv]
				surv = [str(i).replace('"', '') for i in surv]
				surv = [str(i).replace('\n', '') for i in surv]
	f.close()

	survivaldict = {samples[i]: surv[i] for i in range(len(samples))}
	return survivaldict

def main():
	df = getsubtypes()
	survivaldict = getmeta()
	getStats(df, survivaldict)

if __name__ == "__main__":
	main()


#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import argparse as ap
import textwrap

import pickle as pkl
import gzip

import keras
from keras.models import load_model
import numpy as np

description = '''\
Ceci est un programme qui cherche à identifier la relation binaire
reliant deux noms dans le contexte d'une phrase.

--------------------------------------------------------------------------

Le résultat peut être l'une de ces relations binaires,
ainsi que leur symétries :
    - Cause-Effet(e1,e2) : relation de cause à effet entre deux mots
    - Outil-Agent(e1,e2) : agent utilisant un outil, un instrument
    - Produit-Fabricant(e1,e2) : produit conçu par un fabricant
    - Contenant-Contenu(e1,e2) : contenant contenant un contenu (oui)
    - Entité-Origine(e1,e2) : entité provenant d'une source, géographique
ou physique
    - Entité-Destination(e1,e2) : entité se dirigeant vers une destination
    - Composant-Catégorie(e1,e2) : composant d'une catégorie plus globale
    - Membre-Collection(e1,e2) : membre d'un groupe d'objet de même sorte
    - Message-Sujet(e1,e2) : message dont le thème est <e2>
    - Autre(e1,e2) (cette relation est symétrique)

--------------------------------------------------------------------------
'''

parser = ap.ArgumentParser(formatter_class=ap.RawDescriptionHelpFormatter,
description=textwrap.dedent(description),
epilog= "Matthieu RÉ, Yaohui WANG")
parser.add_argument("sentence",
                    help="La phrase dans laquelle on veut trouver une relation")
parser.add_argument("-e1","--first_term", type=int,
                    help="La position du premier terme")
parser.add_argument("-e2","--second_term", type=int,
                    help="La position de second terme")
parser.add_argument("-v","--verbose", action="store_true",
                    help="Affiche certaines informations supplémentaires")
args = vars(parser.parse_args())
pos1 = args["first_term"]
pos2 = args["second_term"]
sentence = args["sentence"]

################################################################################
######################### Preprocessing de la phrase ###########################
################################################################################

embeddingsPath = 'corpus/deps.words'

distanceMapping = {'PADDING': 0, 'LowerMin': 1, 'GreaterMax': 2}
minDistance = -30
maxDistance = 30
for dis in np.arange(minDistance,maxDistance+1):
    distanceMapping[dis] = len(distanceMapping)

labelsMapping = {'Other':0,
                 'Message-Topic(e1,e2)':1, 'Message-Topic(e2,e1)':2,
                 'Product-Producer(e1,e2)':3, 'Product-Producer(e2,e1)':4,
                 'Instrument-Agency(e1,e2)':5, 'Instrument-Agency(e2,e1)':6,
                 'Entity-Destination(e1,e2)':7, 'Entity-Destination(e2,e1)':8,
                 'Cause-Effect(e1,e2)':9, 'Cause-Effect(e2,e1)':10,
                 'Component-Whole(e1,e2)':11, 'Component-Whole(e2,e1)':12,
                 'Entity-Origin(e1,e2)':13, 'Entity-Origin(e2,e1)':14,
                 'Member-Collection(e1,e2)':15, 'Member-Collection(e2,e1)':16,
                 'Content-Container(e1,e2)':17, 'Content-Container(e2,e1)':18}
word2Idx = {}
words = {}
folder = 'files/'
files = [folder+'train.txt', folder+'test.txt']

# Create a dictionnary of words of the sentence
for fileIdx in np.arange(len(files)):
    file = files[fileIdx]
    for line in open(file):
        splits = line.strip().split('\t')

        sentence_ = splits[3]
        tokens = sentence_.split(" ")
        for token in tokens:
            words[token.lower()] = True

for token in sentence.split(" "):
    words[token.lower()] = True

# Create the word to index dictionary using 'deps.words'
for line in open(embeddingsPath):
    split = line.strip().split(" ")
    word = split[0]

    if len(word2Idx) == 0: #Add padding+unknown
        word2Idx["PADDING"] = len(word2Idx)
        vector = np.zeros(len(split)-1) #Zero vector vor 'PADDING' word

        word2Idx["UNKNOWN"] = len(word2Idx)
        vector = np.random.uniform(-0.25, 0.25, len(split)-1)

    if split[0].lower() in words:
        vector = np.array([float(num) for num in split[1:]])
        word2Idx[split[0]] = len(word2Idx)

def getWordIdx(token, word2Idx):
    """Returns from the word2Idex table the word index for a given token"""
    if token in word2Idx:
        return word2Idx[token]
    elif token.lower() in word2Idx:
        return word2Idx[token.lower()]

    return word2Idx["UNKNOWN"]

def createMatricesSentence(sentence, pos1, pos2, word2Idx, maxSentenceLen=100):
    """Creates matrices for the events and sentence for the given file"""
    positionMatrix1 = []
    positionMatrix2 = []
    tokenMatrix = []

    tokens = sentence.split(" ")

    tokenIds = np.zeros(maxSentenceLen)
    positionValues1 = np.zeros(maxSentenceLen)
    positionValues2 = np.zeros(maxSentenceLen)

    for idx in range(0, min(maxSentenceLen, len(tokens))):
        tokenIds[idx] = getWordIdx(tokens[idx], word2Idx)

        distance1 = idx - int(pos1)
        distance2 = idx - int(pos2)

        if distance1 in distanceMapping:
            positionValues1[idx] = distanceMapping[distance1]
        elif distance1 <= minDistance:
            positionValues1[idx] = distanceMapping['LowerMin']
        else:
            positionValues1[idx] = distanceMapping['GreaterMax']

        if distance2 in distanceMapping:
            positionValues2[idx] = distanceMapping[distance2]
        elif distance2 <= minDistance:
            positionValues2[idx] = distanceMapping['LowerMin']
        else:
            positionValues2[idx] = distanceMapping['GreaterMax']

    tokenMatrix.append(tokenIds)
    positionMatrix1.append(positionValues1)
    positionMatrix2.append(positionValues2)

    return [np.array(tokenMatrix, dtype='int32'),
           np.array(positionMatrix1, dtype='int32'),
           np.array(positionMatrix2, dtype='int32')]

################################################################################
########################## Prédiction de la relation ###########################
################################################################################

stcP, pos1P, pos2P = createMatricesSentence(sentence, pos1, pos2, word2Idx, 97)

if (args["verbose"]):
    print(stcP)
    print(pos1P)
    print(pos2P)

model = load_model('models/model1.h5')
pred = model.predict_classes([stcP,pos1P,pos2P],
                             verbose=args["verbose"])[0]

if (args["verbose"]):
    print("Valeur de la relation : "+ str(pred))

relation = [r for r in labelsMapping.keys() if (labelsMapping[r]==pred)][0]
relation = relation.split("(")[0]

if (pred%2==0):
    relation += "("+sentence.split(" ")[pos2]+", "+sentence.split(" ")[pos1]+")"
else:
    relation += "("+sentence.split(" ")[pos1]+", "+sentence.split(" ")[pos2]+")"

print("Relation trouvée : "+relation)

del model
import gc; gc.collect()


from cgitb import grey, text
from itertools import count
from tokenize import Number
import cv2
import imutils
import pytesseract

pytesseract.pytesseract.tesseract_cmd=r"C:\Program Files\Tesseract-OCR\tesseract.exe"

image = cv2.imread('car2.jpg')

image = imutils.resize(image, width=400)

cv2.imshow("original Image",image)
cv2.waitKey(0)

gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
cv2.imshow("Gray Scale Image",gray)
cv2.waitKey(0)

gray = cv2.bilateralFilter(gray,11,17,17)
cv2.imshow("Smoother Image",gray)
cv2.waitKey(0)

edged = cv2.Canny(gray,170,200)
cv2.imshow("Canny edge",edged)
cv2.waitKey(0)

cnts, new = cv2.findContours(edged.copy(), cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)

image1 = image.copy()
cv2.drawContours(image1 , cnts,-1,(0,255,0),3)
cv2.imshow("Canny after contouring", image1)
cv2.waitKey(0)

cnts = sorted(cnts, key=cv2.contourArea, reverse= True)[:30]
NumberPlateCount = None

image2 = image.copy()
cv2.drawContours(image2, cnts,-1,(0,255,0),3)
cv2.waitKey(0)

count = 0
name =1
for i in cnts:
    perimeter =  cv2.arcLength(i, True)
    approx = cv2.approxPolyDP(i,0.02*perimeter,True)

    if(len(approx)==4):
        NumberPlateCount= approx
        x,y,w,h = cv2.boundingRect(i)
        crp_img = image[y:y*h, x:x*w]

        cv2.imwrite(str(name)+ '.png',crp_img)
        name += 1
        break

cv2.drawContours(image,[NumberPlateCount],-1,(0,255,0),3)
cv2.imshow("Final Image",image)
cv2.waitKey(0)

crop_img_loc ='1.png'
cv2.imshow("Cropped Image",cv2.imread(crop_img_loc))

text = pytesseract.image_to_string(crop_img_loc,lang='eng')
print("Number is :",text)
cv2.waitKey(0)


from datetime import datetime, timedelta

from flask import Blueprint, abort, flash, redirect, render_template, request, url_for
from flask_login import current_user, login_required
from website import db
from website.budgets.forms import BudgetForm
from website.models import Budget

budgets = Blueprint("budgets", __name__)

@budgets.route("/budgets")
@login_required
def show_budgets():
    budgets = Budget.query.filter_by(user_id=current_user.id).all()
    return render_template("budgets.html", budgets=budgets)

@budgets.route("/budgets/new", methods=["GET", "POST"])
@login_required
def new_budget():
    form = BudgetForm()
    if form.validate_on_submit():
        budget = Budget(
            name=form.name.data,
            initial=form.initial.data,
            present=form.present.data,
            tenure=form.tenure.data,
            acc_holder=current_user,
            date=datetime.utcnow() + timedelta(hours=5, minutes=30),
        )

        db.session.add(budget)
        db.session.commit()
        flash("Your budget has been created!", "success")
        return redirect(url_for("budgets.show_budgets"))
    return render_template(
        "create_budget.html", title="New Budget", form=form, legend="New Budget"
    )

@budgets.route("/budgets/<int:budget_id>/update", methods=["GET", "POST"])
@login_required
def update_budget(budget_id):
    budget = Budget.query.filter_by(id=budget_id).first()
    if budget.user_id != current_user.id:
        abort(403)
    form = BudgetForm()
    if form.validate_on_submit():
        budget.name = form.name.data
        budget.initial = form.initial.data
        budget.present = form.present.data
        budget.tenure = form.tenure.data
        budget.date = datetime.utcnow() + timedelta(hours=5, minutes=30)
        db.session.commit()
        flash("Your budget has been updated!", "success")
        return redirect(url_for("budgets.show_budgets"))
    elif request.method == "GET":
        form.name.data = budget.name
        form.initial.data = budget.initial
        form.present.data = budget.present
        form.tenure.data = budget.tenure
        form.submit.label.text = "Update"

    return render_template(
        "create_budget.html",
        title="Update budget",
        form=form,
        legend="Update budget",
    )

@budgets.route("/budgets/<int:budget_id>/delete", methods=["POST"])
@login_required
def delete_budget(budget_id):
    budget = Budget.query.filter_by(id=budget_id).first()
    if budget.user_id != current_user.id:
        abort(403)
    db.session.delete(budget)
    db.session.commit()
    flash("Your budget has been deleted!", "success")
    return redirect(url_for("budgets.show_budgets"))


import multiprocessing

def f(l, n):

    with l:
        print("hello zhenji", n)

if __name__ == "__main__":
    l = multiprocessing.Lock()

    for num in range(10):

        multiprocessing.Process(target=f, args=(l, num)).start()


'''

i might need to just write this myself. chat is having a hard time
'''
import torch
import torchtext
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
from nltk.parse.corenlp import CoreNLPDependencyParser

def main():
        # Example list of sentences
    sentences = ['I prefer the morning flight through Denver', 'What is the airspeed of an unladen swallow ?']

    # Create the dataset
    dataset = DependencyParserDataset(sentences)

    # dataset[0]
    # print(dataset[0][1])
    # Create a DataLoader to iterate over the dataset
    dataloader = DataLoader(dataset, batch_size=2, shuffle=True, collate_fn=collate_fn)

    # # # Iterate over the data
    for (sentence, label) in dataloader:
    #     # Clean up the formatting of the label
    #     label = dataset.clean_label(label)
        print(sentence)
        print(label)

class DependencyParserDataset(Dataset):
    def __init__(self, sentences):
        self.sentences = sentences
        self.dep_parser = CoreNLPDependencyParser(url='http://localhost:9000')

    def __len__(self):
        return len(self.sentences)

    def __getitem__(self, index):
        sentence = self.sentences[index]

        # print("sentence\n", sentence)
        label = self.generate_label(sentence)
        # print("\nlabel\n",label)

        # Split the sentence into words
        words = sentence.split()

        # Create a vocabulary that maps words to integers
        vocabulary = {word: i for i, word in enumerate(set(words))}

        # Convert the sentence to a sequence of integers
        sentence_data = [vocabulary[word] for word in words]

        # Convert the sequence to a tensor
        sentence_tensor = torch.tensor(sentence_data, dtype=torch.long)

        return (sentence_tensor, label)

    def from_tuples(self, tuples):
        # Initialize an empty list of tokens
        tokens = []
        counter = 0

        # Iterate over the tuples
        for i, tup in enumerate(tuples):
            # Extract the token, part-of-speech, and dependency index from the tuple
            # token, pos, dep = tup[0][0], tup[0][1], i+1
            token, pos, dep = tup[0][0], tup[0][1], tup[2][0]
            # Create a dictionary for the token and add it to the list of tokens
            tokens.append({'token': token, 'pos': pos, 'dep': dep, 'key': counter})
            #we can ignore counter in our training and conversion back to human readable
            counter+=1

        return tokens

    def generate_label(self, sentence):
        parses = self.dep_parser.parse(sentence.split())
        prelim = [[(governor, dep, dependent) for governor, dep, dependent in parse.triples()] for parse in parses]
        # tokenized = nltk.word_tokenize(prelim[0])
        embeddings = torchtext.vocab.GloVe(name='6B', dim=100)
        tokens = prelim[0]
        #conversion function here
        tokens = self.from_tuples(tokens)
        print(tokens)
        embedding_vectors = [embeddings.vectors[embeddings.stoi[token['token']]] for token in
                             tokens]
        # print(embedding_vectors)

        '''
        [(('word', 'nsub'), 'connection', (('word','nn')), ...] is a terrible label bro. adjust
        your preprocessing. see the tmp file to continue. need to get dinner
        '''
        tokenized_label = embeddings.stoi(prelim)
        return embeddings.vector[tokenized_label]

    # def clean_label(self, label):
    #     return [[(governor[0], dep, dependent[0]) for governor, dep, dependent in parse] for parse in label]

def collate_fn(data):
    # Unpack the data into a list of sentences and a list of labels
    sentences, labels = zip(*data)
    # print(sentences, labels)

    # Convert the sentences to tensors
    tensor_sentences = [torch.tensor(s, dtype=torch.long) for s in sentences]
    # tensor_sentences = tensor_sentences[0]

    # Pad the sequences so they are all the same length
    padded_sentences = torch.nn.utils.rnn.pad_sequence(tensor_sentences, batch_first=True)
    print(labels)
    padded_labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True)

    return (padded_sentences, padded_labels)

if __name__ == '__main__':
    main()


import psutil
import time
import datetime
import os
import socket

monitoring_interval = 5
memory_threshold = 80

current_date = datetime.datetime.now().strftime("%Y-%m-%d")
log_filename = f"{current_date}-pub.log"

def log_system_info():
    cpu_usage = psutil.cpu_percent(interval=1)
    num_logical_cpus = psutil.cpu_count(logical=True)
    used_memory = psutil.virtual_memory().percent
    used_disk_space = psutil.disk_usage('/').percent
    current_host_ip = get_host_ip()

    log_line = f"{datetime.datetime.now()}, {cpu_usage}, {num_logical_cpus}, {used_memory}%, {used_disk_space}%, {current_host_ip}\n"

    with open(log_filename, 'a') as log_file:
        log_file.write(log_line)

def check_memory_threshold():
    used_memory = psutil.virtual_memory().percent
    if used_memory > memory_threshold:
        notification_filename = f"{current_date}-notification.log"
        with open(notification_filename, 'w') as notification_file:
            notification_file.write(
                "Low memory: System is running low on available memory.")

def get_host_ip():
    try:
        host_name = socket.gethostname()
        host_ip = socket.gethostbyname(host_name)
        return host_ip
    except Exception as e:
        return "N/A"

if __name__ == "__main__":
    if not os.path.exists(log_filename):
        with open(log_filename, 'w') as log_file:
            log_file.write(
                "Timestamp, CPU Usage (%), Logical CPUs, Memory Usage (%), Disk Space Usage (%), Host IP\n")

    while True:
        log_system_info()
        check_memory_threshold()
        time.sleep(monitoring_interval)


import os
import sys
import pandas as pd
import numpy as np
from src.logging.logging import logging
from src.exception.exception import CustomException
from src.utils.utils import save_object
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OrdinalEncoder, LabelEncoder
from dataclasses import dataclass

@dataclass
class DataTransformationConfig:
    preprocessor_pickle_path = os.path.join("artifacts", "preprocessor.pkl")

class DataTransformation:
    def __init__(self):
        self.data_transformation_config = DataTransformationConfig()

    def get_data_transformation_object(self):
        try:
            cat_cols = ['Gender', 'family_history_with_overweight', 'FAVC', 'CAEC', 'SMOKE', 'SCC',
                        'CALC', 'MTRANS']
            num_cols = ['Age', 'Height', 'Weight', 'FCVC', 'NCP', 'CH2O', 'FAF', 'TUE']
            target_col = ['NObeyesdad']

            target_categories = ['Obesity_Type_I', 'Obesity_Type_III', 'Obesity_Type_II',
                                 'Overweight_Level_I', 'Overweight_Level_II', 'Normal_Weight',
                                 'Insufficient_Weight']

            # create pipeline to impute missing data and encode the columns to numerical
            cat_pipeline = Pipeline(steps=[
                ("imputer", SimpleImputer(strategy="most_frequent")),
                ("encoder", OrdinalEncoder())
            ])

            num_pipeline = Pipeline(steps=[
                ("imputer", SimpleImputer(strategy="median")),
            ])

            target_pipeline = Pipeline(steps=[
                ("encode", OrdinalEncoder(categories=[target_categories]))
            ])

            preprocessor = ColumnTransformer([
                ("cat", cat_pipeline, cat_cols),
                ("num", num_pipeline, num_cols)
            ])

            target_preprocessor = ColumnTransformer([
                ("target", target_pipeline, target_col)
            ])

            logging.info("Preprocessor file has been successfully created")
            return preprocessor, target_preprocessor

        except Exception as err:
            raise CustomException(sys, err)

    def initialize_data_transformation(self, train_path, test_path):
        try:
            logging.info("Data Transformation has been initiated")

            # read train and test data
            train_df = pd.read_csv(train_path)
            test_df = pd.read_csv(test_path)

            logging.info(f"Train dataframe head: \n {train_df.head().to_string()}")
            logging.info(f"Test dataframe head: \n {test_df.head().to_string()}")

            target_feature_col = "NObeyesdad"

            # split the data into input and target feature
            input_feature_train_df = train_df.drop([target_feature_col], axis=1)
            target_feature_train = train_df[target_feature_col]
            input_feature_test_df = test_df.drop([target_feature_col], axis=1)
            target_feature_test = test_df[target_feature_col]
            logging.info("Input and Target feature has been successfully split")

            # transform the feature using preprocessor
            preprocessor_obj, target_preprocessor_obj = self.get_data_transformation_object()
            input_train_arr = preprocessor_obj.fit_transform(input_feature_train_df)
            input_test_arr = preprocessor_obj.transform(input_feature_test_df)
            target_feature_train = target_preprocessor_obj.fit_transform(pd.DataFrame(target_feature_train))
            target_feature_test = target_preprocessor_obj.transform(pd.DataFrame(target_feature_test))

            train_arr = np.c_[input_train_arr, np.array(target_feature_train)]
            test_arr = np.c_[input_test_arr, np.array(target_feature_test)]

            logging.info("Data Transformation has been successfully completed")

            save_object(self.data_transformation_config.preprocessor_pickle_path, preprocessor_obj)
            logging.info("Preprocessor picked was successfully saved")

            return (train_arr,
                    test_arr,
                    self.data_transformation_config.preprocessor_pickle_path)
        except Exception as err:
            raise CustomException(sys, err)


# Read jsonlines files predicted with wl-coref and evaluate them with coreference-eval
# coreference-eval presently requires python 3.9

import corefeval
import jsonlines
import os

gold_folder = "/home/egil/gits_wsl/narc-baseline/data/wl-formatted"
gold_filenames = {"dev": "narc_development.jsonl", "test": "narc_test.jsonl"}
predicted_folder = "/home/egil/gits_wsl/narc-baseline/experiments/outputs"

def read_clusters(source_path:str, key="clusters"):
    """ read the clusters from multiline jsonlines,
    return list of clusters """
    with jsonlines.open(source_path) as rf:
        clusters = [doc[key] for doc in rf]
    return clusters

to_score = os.listdir(predicted_folder)

for prediction in to_score:
    if "_dev" in prediction:
        gold_path = os.path.join(gold_folder, gold_filenames["dev"])
    else:
        gold_path = os.path.join(gold_folder, gold_filenames["test"])
    pred_path = os.path.join(predicted_folder, prediction)

    pred_split = prediction.split("_")
    experiment = pred_split[1]+"_"+str(pred_split[2])
    split = prediction.split("_")[-2]
    # print(gold_path, os.path.exists(gold_path))
    # print(pred_path, os.path.exists(predicted_folder))
    # print(experiment, split)

    gold_clusters = read_clusters(gold_path)
    pred_clusters = read_clusters(pred_path, key="span_clusters" )

    assert len(gold_clusters) == len(pred_clusters) # Should be one for each document

    scorer = corefeval.Scorer()
    for gold_doc, pred_doc in zip(gold_clusters, pred_clusters):
        # print(all([g==p for gold_c, pred_c in zip(gold_doc, pred_doc)
        #                     for g,p in zip(gold_c, pred_c)]))
        scorer.update(corefeval.Document(predicted = pred_doc, truth = gold_doc))

    conll_f1, metrics = scorer.detailed_score(experiment, split)
    # Feiler, har fire, ikke tre listenivåer: [[[[1, 3], [26, 27], [
    # corefeval.get_metrics(pred_clusters, gold_clusters)
    print(conll_f1)
    print(metrics)

# for experiment in ["POC2_000", "POC2_001"]:
#     for split in ["dev", "test"]:
#         gold_path = os.path.join(gold_folder, f"narc_{wl_naming[split]}.jsonl")
#         pred_path = os.path.join(predicted_folder, f"{experiment}_{split}_predicted.jsonlines")

#         for f in [gold_path, pred_path]:
#             assert os.path.exists(f)
#         gold_clusters = read_clusters(gold_path)
#         pred_clusters = read_clusters(pred_path )

#         assert len(gold_clusters) == len(pred_clusters) # Should be one for each document

#         scorer = corefeval.Scorer()
#         for gold_doc, pred_doc in zip(gold_clusters, pred_clusters):
#             print(all([g==p for gold_c, pred_c in zip(gold_doc, pred_doc)
#                                 for g,p in zip(gold_c, pred_c)]))
#             scorer.update(corefeval.Document(predicted = pred_doc, truth = gold_doc))

#         conll_f1, metrics = scorer.detailed_score(experiment, split)
#         # Feiler, har fire, ikke tre listenivåer: [[[[1, 3], [26, 27], [
#         # corefeval.get_metrics(pred_clusters, gold_clusters)
#         print(conll_f1)
#         print(metrics)


from rest_framework import serializers
from .models import Member

class MemberSerializer(serializers.ModelSerializer):

    class Meta:
        model = Member
        fields = ('id', 'first_name', 'middle_name', 'last_name',
                  'email', 'address', 'city', 'state','zip_code',
                  'is_active', 'balance')


from rdkit.ML.Descriptors import MoleculeDescriptors
from rdkit.Chem import AllChem
from rdkit.Chem import MACCSkeys
from rdkit.Chem import Descriptors
from rdkit import DataStructs
from rdkit import Chem
from rdkit.ML.Cluster import Butina
import pandas as pd
from rdkit.Chem import PandasTools
from typing import List
import math

def calc_descriptors_from_mol(mol):
    """
    Encode a molecule from a RDKit Mol into a set of descriptors.

    Parameters
    ----------
    mol : RDKit Mol
        The RDKit molecule.

    Returns
    -------
    list
        The set of chemical descriptors as a list.

    """
    calc = MoleculeDescriptors.MolecularDescriptorCalculator([desc[0] for desc in Descriptors.descList])
    return list(calc.CalcDescriptors(mol))

def make_descriptors(inchis: List[str]) -> pd.DataFrame:
    mols = [Chem.MolFromInchi(inchi) for inchi in inchis]
    desc_list = []

    for mol in mols:
        desc = calc_descriptors_from_mol(mol)
        desc_list.append(desc)

    chemical_descriptors = pd.DataFrame(desc_list,
                                        columns=[desc[0] for desc in Descriptors.descList],
                                        index=inchis)
    chemical_descriptors = chemical_descriptors.drop('Ipc', axis=1)
    chemical_descriptors = chemical_descriptors.loc[:, chemical_descriptors.notnull().all()]
    return chemical_descriptors.fillna(chemical_descriptors.mean())

def get_fps(mol_list, fp_type):
    fp_dict = {
        "morgan2": [AllChem.GetMorganFingerprintAsBitVect(x, 2) for x in mol_list],
        "rdkit": [Chem.RDKFingerprint(x) for x in mol_list],
        "maccs": [MACCSkeys.GenMACCSKeys(x) for x in mol_list],
    }

    return fp_dict[fp_type]

def cluster_mols(fps, sim_cutoff=0.8):
    dist_cutoff = 1 - sim_cutoff
    dists = []
    nfps = len(fps)
    for i in range(1, nfps):
        sims = DataStructs.BulkTanimotoSimilarity(fps[i], fps[:i])
        dists.extend([1 - x for x in sims])
    mol_clusters = Butina.ClusterData(dists, nfps, dist_cutoff, isDistData=True)
    cluster_id_list = [0] * nfps
    for idx, cluster in enumerate(mol_clusters, 1):
        for member in cluster:
            cluster_id_list[member] = idx
    return [x - 1 for x in cluster_id_list]

def z_score(x_bar_0, x_bar_1, n_0, n_1, var_0):
    """

    :param x_bar_0: mean of full set
    :param x_bar_1: mean of subset
    :param n_0: size of full set
    :param n_1: size of sub set
    :param var_0: variance of full set
    :return: z score

    tkae from finding discriminating structrual feature sby reassembling common building blocks.  Cross Kevin
    """
    z = (x_bar_0 - x_bar_1)*math.sqrt((n_0*n_1) / (var_0*(n_0-n_1)))
    return z