text:
import numpy as np
import cv2
import matplotlib.pyplot as plt
import os
import random
import math
import pytesseract
MAX_DIAG_SIZE_MULTIPLE_AWAY = 5.0
MAX_CHANGE_IN_AREA = 0.5
MAX_CHANGE_IN_WIDTH = 0.8
MAX_CHANGE_IN_HEIGHT = 0.2
MAX_ANGLE_BETWEEN_CHARS = 12.0
PLATE_WIDTH_PADDING_FACTOR = 1.3
PLATE_HEIGHT_PADDING_FACTOR = 1.5
MIN_NUMBER_OF_MATCHING_CHARS = 3
RESIZED_CHAR_IMAGE_WIDTH = 20
RESIZED_CHAR_IMAGE_HEIGHT = 30
MIN_CONTOUR_AREA = 100
kNearest = cv2.ml.KNearest_create()

showSteps=True
def loadKNNDataAndTrainKNN():
	allContoursWithData = []
	validContoursWithData = []
	npaClassifications = np.loadtxt("/home/lol/Desktop/project/classifications.txt", np.float32)
	npaFlattenedImages = np.loadtxt("/home/lol/Desktop/project/flattened_images.txt", np.float32)
	npaClassifications = npaClassifications.reshape((npaClassifications.size, 1))
	kNearest.setDefaultK(1)
	kNearest.train(npaFlattenedImages, cv2.ml.ROW_SAMPLE, npaClassifications)

	return True
# end function
class PossiblePlate:

	def __init__(self):
		self.imgPlate = None
		self.imgGrayscale = None
		self.imgThresh = None
		self.rrLocationOfPlateInScene = None
		self.strChars = ""
class PossibleChar:
    def __init__(self,contour):
        self.contour = contour
        self.boundingRect = cv2.boundingRect(self.contour)
        [intX, intY, intWidth, intHeight] = self.boundingRect
        self.intBoundingRectX = intX
        self.intBoundingRectY = intY
        self.intBoundingRectWidth = intWidth
        self.intBoundingRectHeight = intHeight
        self.intBoundingRectArea = self.intBoundingRectWidth * self.intBoundingRectHeight

        self.intCenterX = (self.intBoundingRectX + self.intBoundingRectX + self.intBoundingRectWidth) / 2
        self.intCenterY = (self.intBoundingRectY + self.intBoundingRectY + self.intBoundingRectHeight) / 2

        self.fltDiagonalSize = math.sqrt((self.intBoundingRectWidth ** 2)+(self.intBoundingRectHeight ** 2))

        self.fltAspectRatio = float(self.intBoundingRectWidth)/float(self.intBoundingRectHeight)

###################################################################################################

def checkIfPossibleChar(possibleChar):
    if (possibleChar.intBoundingRectArea > (80) and possibleChar.intBoundingRectWidth > (2) and possibleChar.intBoundingRectHeight > (8) and  (0.25)< possibleChar.fltAspectRatio and possibleChar.fltAspectRatio < (0.8)):
        return True
    else:
        return False

def extractValue(imgOriginal):
    height, width, numChannels = imgOriginal.shape

    imgHSV = np.zeros((height, width, 3), np.uint8)
    imgHSV = cv2.cvtColor(imgOriginal, cv2.COLOR_BGR2HSV)

    imgHue, imgSaturation, imgValue = cv2.split(imgHSV)

    return imgValue
def findPossibleCharsInPlate(imgGrayscale, imgThresh):
    listOfPossibleChars = []
    contours = []
    imgThreshCopy = imgThresh.copy()

            # find all contours in plate
    imgContours, contours, npaHierarchy = cv2.findContours(imgThreshCopy, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)

    for contour in contours:
        possibleChar = PossibleChar(contour)

        if checkIfPossibleChar(possibleChar):              # if contour is a possible char, note this does not compare to other chars (yet) . . .
            listOfPossibleChars.append(possibleChar)

    return listOfPossibleChars

def maximizeContrast(imgGrayscale):

    height, width = imgGrayscale.shape

    imgTopHat = np.zeros((height, width, 1), np.uint8) #top hat
    imgBlackHat = np.zeros((height, width, 1), np.uint8)#black hat

    structuringElement = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))

    imgTopHat = cv2.morphologyEx(imgGrayscale, cv2.MORPH_TOPHAT, structuringElement)
    imgBlackHat = cv2.morphologyEx(imgGrayscale, cv2.MORPH_BLACKHAT, structuringElement)

    imgGrayscalePlusTopHat = cv2.add(imgGrayscale, imgTopHat)# grayscale + top hat
    imgGrayscalePlusTopHatMinusBlackHat = cv2.subtract(imgGrayscalePlusTopHat, imgBlackHat)# grayscale+tophat-blackhat

    return imgGrayscalePlusTopHatMinusBlackHat

def preprocess(imgOriginal):
    imgGrayscale = extractValue(imgOriginal)

    imgMaxContrastGrayscale = maximizeContrast(imgGrayscale)

    height, width = imgGrayscale.shape

    imgBlurred = np.zeros((height, width, 1), np.uint8)

    imgBlurred = cv2.GaussianBlur(imgMaxContrastGrayscale, (5,5), 0)

    imgThresh = cv2.adaptiveThreshold(imgBlurred, 255.0, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 19, 9)

    return imgGrayscale, imgThresh

def findPossibleCharsInScene(imgThresh):
    listOfPossibleChars = []

    intCountOfPossibleChars = 0

    imgThreshCopy = imgThresh.copy()

    imgContours, contours, npaHierarchy = cv2.findContours(imgThreshCopy, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
    print("length of contours",len(contours))

    height, width = imgThresh.shape
    imgContours = np.zeros((height, width, 3), np.uint8)

    for i in range(0, len(contours)):

        if showSteps == True:
            cv2.drawContours(imgContours, contours, i, (255.0,255.0,255.0))

        possibleChar = PossibleChar(contours[i])

        if checkIfPossibleChar(possibleChar):

            intCountOfPossibleChars = intCountOfPossibleChars + 1
            print("possibleChar contour",(intCountOfPossibleChars))
            listOfPossibleChars.append(possibleChar)

    return listOfPossibleChars,imgContours

def findListOfListsOfMatchingChars(listOfPossibleChars):

    listOfListsOfMatchingChars = []

    for possibleChar in listOfPossibleChars:
        listOfMatchingChars = findListOfMatchingChars(possibleChar, listOfPossibleChars)

        listOfMatchingChars.append(possibleChar)

        if len(listOfMatchingChars) < MIN_NUMBER_OF_MATCHING_CHARS:
            continue

        listOfListsOfMatchingChars.append(listOfMatchingChars)

        listOfPossibleCharsWithCurrentMatchesRemoved = []

        listOfPossibleCharsWithCurrentMatchesRemoved = list(set(listOfPossibleChars) - set(listOfMatchingChars))

        recursiveListOfListsOfMatchingChars = findListOfListsOfMatchingChars(listOfPossibleCharsWithCurrentMatchesRemoved)

        for recursiveListOfMatchingChars in recursiveListOfListsOfMatchingChars:
            listOfListsOfMatchingChars.append(recursiveListOfMatchingChars)

        break

    return listOfListsOfMatchingChars

def findListOfMatchingChars(possibleChar, listOfChars):

    listOfMatchingChars = []

    for possibleMatchingChar in listOfChars:
        if possibleMatchingChar == possibleChar:
            continue

        fltDistanceBetweenChars = distanceBetweenChars(possibleChar, possibleMatchingChar)

        fltAngleBetweenChars = angleBetweenChars(possibleChar, possibleMatchingChar)

        fltChangeInArea = float(abs(possibleMatchingChar.intBoundingRectArea - possibleChar.intBoundingRectArea)) / float(possibleChar.intBoundingRectArea)

        fltChangeInWidth = float(abs(possibleMatchingChar.intBoundingRectWidth - possibleChar.intBoundingRectWidth)) / float(possibleChar.intBoundingRectWidth)
        fltChangeInHeight = float(abs(possibleMatchingChar.intBoundingRectHeight - possibleChar.intBoundingRectHeight)) / float(possibleChar.intBoundingRectHeight)

        if (fltDistanceBetweenChars < (possibleChar.fltDiagonalSize * MAX_DIAG_SIZE_MULTIPLE_AWAY) and fltAngleBetweenChars < MAX_ANGLE_BETWEEN_CHARS and
            fltChangeInArea < MAX_CHANGE_IN_AREA and fltChangeInWidth < MAX_CHANGE_IN_WIDTH and fltChangeInHeight < MAX_CHANGE_IN_HEIGHT):

            listOfMatchingChars.append(possibleMatchingChar)

    return listOfMatchingChars

def distanceBetweenChars(firstChar, secondChar):
    intx = abs(firstChar.intCenterX - secondChar.intCenterX)
    inty = abs(firstChar.intCenterY - secondChar.intCenterY)

    return math.sqrt((intx ** 2) + (inty ** 2))

def angleBetweenChars(firstChar, secondChar):
    fltAdj = float(abs(firstChar.intCenterX - secondChar.intCenterX))
    fltOpp = float(abs(firstChar.intCenterY - secondChar.intCenterY))

    if fltAdj != 0.0:
        fltAngleInRad = math.atan(fltOpp / fltAdj)
    else:
        fltAngleInRad = 1.5708

    fltAngleInDeg = fltAngleInRad * (180.0 / math.pi)

    return fltAngleInDeg

def detectPlatesInScene(imgOriginalScene):
    listOfPossiblePlates = []

    height, width, numChannels = imgOriginalScene.shape

    imgGrayscaleScene = np.zeros((height, width, 1), np.uint8)
    imgThreshScene = np.zeros((height, width, 1), np.uint8)
    imgContours = np.zeros((height, width, 3), np.uint8)

    #cv2.destroyAllWindows()

    if showSteps == True:
        cv2.imshow("1", imgOriginalScene)

    imgGrayscaleScene, imgThreshScene = preprocess(imgOriginalScene)
    if showSteps == True:
        cv2.imshow("2", imgGrayscaleScene)
        cv2.imshow("3", imgThreshScene)

    listOfPossibleCharsInScene ,contour= findPossibleCharsInScene(imgThreshScene)

    if showSteps == True:

        imgContours = np.zeros((height, width, 3), np.uint8)

        contours = []

        for possibleChar in listOfPossibleCharsInScene:
            contours.append(possibleChar.contour)

        cv2.drawContours(imgContours, contours, -1,(255.0,255.0,255.0))
        cv2.imshow("4", imgContours)

    listOfListsOfMatchingCharsInScene = findListOfListsOfMatchingChars(listOfPossibleCharsInScene)

    if showSteps == True:

        imgContours = np.zeros((height, width, 3), np.uint8)

        for listOfMatchingChars in listOfListsOfMatchingCharsInScene:
            intRandomBlue = random.randint(0, 255)
            intRandomGreen = random.randint(0, 255)
            intRandomRed = random.randint(0, 255)

            contours = []

            for matchingChar in listOfMatchingChars:
                contours.append(matchingChar.contour)

            cv2.drawContours(imgContours, contours, -1, (intRandomBlue, intRandomGreen, intRandomRed))

        cv2.imshow("5", imgContours)

    for listOfMatchingChars in listOfListsOfMatchingCharsInScene:
        possiblePlate = extractPlate(imgOriginalScene, listOfMatchingChars)

        if possiblePlate.imgPlate is not None:
            listOfPossiblePlates.append(possiblePlate)

    if showSteps == True:
        print("\n")
        cv2.imshow("6", imgContours)

        for i in range(0, len(listOfPossiblePlates)):
            p2rectPoints = cv2.boxPoints(listOfPossiblePlates[i].rrLocationOfPlateInScene)

            cv2.line(imgContours, tuple(p2rectPoints[0]), tuple(p2rectPoints[1]), (0.0,0.0,255.0), 2)
            cv2.line(imgContours, tuple(p2rectPoints[1]), tuple(p2rectPoints[2]), (0.0,0.0,255.0), 2)
            cv2.line(imgContours, tuple(p2rectPoints[2]), tuple(p2rectPoints[3]), (0.0,0.0,255.0), 2)
            cv2.line(imgContours, tuple(p2rectPoints[3]), tuple(p2rectPoints[0]), (0.0,0.0,255.0), 2)

            cv2.imshow("7", imgContours)

            #cv2.imshow("8", listOfPossiblePlates[i].imgPlate)

        listofMatchingCropChars=detectCharsInPlates(listOfPossiblePlates)

        print("\nplate detection complete.\n")
        #cv2.waitKey(0)

    return listofMatchingCropChars

def extractPlate(imgOriginal, listOfMatchingChars):
    possiblePlate = PossiblePlate()

    listOfMatchingChars.sort(key = lambda matchingChar: matchingChar.intCenterX)

    fltPlateCenterX = (listOfMatchingChars[0].intCenterX + listOfMatchingChars[len(listOfMatchingChars) - 1].intCenterX) / 2.0
    fltPlateCenterY = (listOfMatchingChars[0].intCenterY + listOfMatchingChars[len(listOfMatchingChars) - 1].intCenterY) / 2.0

    ptPlateCenter = fltPlateCenterX, fltPlateCenterY

    intPlateWidth = int((listOfMatchingChars[len(listOfMatchingChars) - 1].intBoundingRectX + listOfMatchingChars[len(listOfMatchingChars) - 1].intBoundingRectWidth - listOfMatchingChars[0].intBoundingRectX) * PLATE_WIDTH_PADDING_FACTOR)

    intTotalOfCharHeights = 0

    for matchingChar in listOfMatchingChars:
        intTotalOfCharHeights = intTotalOfCharHeights + matchingChar.intBoundingRectHeight

    fltAverageCharHeight = intTotalOfCharHeights / len(listOfMatchingChars)

    intPlateHeight = int(fltAverageCharHeight * PLATE_HEIGHT_PADDING_FACTOR)

    fltOpposite = listOfMatchingChars[len(listOfMatchingChars) - 1].intCenterY - listOfMatchingChars[0].intCenterY
    fltHypotenuse = distanceBetweenChars(listOfMatchingChars[0], listOfMatchingChars[len(listOfMatchingChars) - 1])
    fltCorrectionAngleInRad = math.asin(fltOpposite / fltHypotenuse)
    fltCorrectionAngleInDeg = fltCorrectionAngleInRad * (180.0 / math.pi)

    possiblePlate.rrLocationOfPlateInScene = ( tuple(ptPlateCenter), (intPlateWidth, intPlateHeight), fltCorrectionAngleInDeg )

    rotationMatrix = cv2.getRotationMatrix2D(tuple(ptPlateCenter), fltCorrectionAngleInDeg, 1.0)

    height, width, numChannels = imgOriginal.shape

    imgRotated = cv2.warpAffine(imgOriginal, rotationMatrix, (width, height))

    imgCropped = cv2.getRectSubPix(imgRotated, (intPlateWidth, intPlateHeight), tuple(ptPlateCenter))

    possiblePlate.imgPlate = imgCropped

    return possiblePlate

def detectCharsInPlates(listOfPossiblePlates):
    intPlateCounter = 0
    imgContours = None
    contours = []

    if len(listOfPossiblePlates) == 0:
        return listOfPossiblePlates

    for possiblePlate in listOfPossiblePlates:

        possiblePlate.imgGrayscale, possiblePlate.imgThresh = preprocess(possiblePlate.imgPlate)

        if showSteps == True:
            cv2.imshow("9", possiblePlate.imgPlate)
            cv2.imshow("10", possiblePlate.imgGrayscale)
            cv2.imshow("11", possiblePlate.imgThresh)

        possiblePlate.imgThresh = cv2.resize(possiblePlate.imgThresh, (0, 0), fx = 1.6, fy = 1.6)

        thresholdValue, possiblePlate.imgThresh = cv2.threshold(possiblePlate.imgThresh, 0.0, 255.0, cv2.THRESH_BINARY | cv2.THRESH_OTSU)

        if showSteps == True:
            cv2.imshow("12", possiblePlate.imgThresh)
            text=pytesseract.image_to_string(possiblePlate.imgThresh)
            print("image using tesseract",text)

        listOfPossibleCharsInPlate = findPossibleCharsInPlate(possiblePlate.imgGrayscale, possiblePlate.imgThresh)

        possiblePlate.strChars = recognizeCharsInPlate(possiblePlate.imgThresh, listOfPossibleCharsInPlate)

        if showSteps == True:
            print("chars found in plate number " + str(
	           intPlateCounter) + " = " + possiblePlate.strChars)
            intPlateCounter = intPlateCounter + 1

    if showSteps == True:

        cv2.waitKey(0)
        cv2.destroyAllWindows()

    return listOfPossiblePlates

def recognizeCharsInPlate(imgThresh, listOfMatchingChars):
    strChars = ""

    height, width = imgThresh.shape

    imgThreshColor = np.zeros((height, width, 3), np.uint8)

    listOfMatchingChars.sort(key = lambda matchingChar: matchingChar.intCenterX)

    cv2.cvtColor(imgThresh, cv2.COLOR_GRAY2BGR, imgThreshColor)

    for currentChar in listOfMatchingChars:
        pt1 = (currentChar.intBoundingRectX, currentChar.intBoundingRectY)
        pt2 = ((currentChar.intBoundingRectX + currentChar.intBoundingRectWidth), (currentChar.intBoundingRectY + currentChar.intBoundingRectHeight))
        cv2.rectangle(imgThreshColor, pt1, pt2, (0.0,255.0,0.0), 2)

        imgROI = imgThresh[currentChar.intBoundingRectY : currentChar.intBoundingRectY + currentChar.intBoundingRectHeight,
                           currentChar.intBoundingRectX : currentChar.intBoundingRectX + currentChar.intBoundingRectWidth]

        imgROIResized = cv2.resize(imgROI, (RESIZED_CHAR_IMAGE_WIDTH, RESIZED_CHAR_IMAGE_HEIGHT))
        npaROIResized = imgROIResized.reshape((1, RESIZED_CHAR_IMAGE_WIDTH * RESIZED_CHAR_IMAGE_HEIGHT))

        npaROIReized = np.float32(npaROIResized)

        npaResults=(npaROIResized)
        retval, npaResults, neigh_resp, dists = kNearest.findNearest(npaROIReized, k = 50)

        strCurrentChar = str(chr(int(npaResults[0][0])))

        strChars = strChars + strCurrentChar

    if showSteps == True:
        cv2.imshow("13", imgThreshColor)

    return strChars
def main():

    impath1="/home/lol/Desktop/project/dataset/"
    print("Your Image is Located at",impath1)
    file_source=input("Enter the image name \n")
    impath=impath1+file_source+".jpg"
    #print(impath)

    blnKNNTrainingSuccessful = loadKNNDataAndTrainKNN()

    if blnKNNTrainingSuccessful == False:
        print("\nerror: KNN traning was not successful\n")
        return

    img=cv2.imread(impath)

    imgGrayscale = detectPlatesInScene(img)

if __name__ == '__main__':
    main()


text:
import numpy as np

from . import registry
from .common import _limit_contact, RETURN_KEYS

@registry("CMMID_better")  # noqa: C901
def CMMID_strategy_better(  # noqa: C901
    case,
    contacts,
    rng,
    do_individual_isolation,
    do_household_isolation,
    do_manual_tracing,
    do_app_tracing,
    do_pop_testing,
    do_symptom_testing,
    go_to_school_prob,
    manual_home_trace_prob,
    manual_work_trace_prob,
    manual_othr_trace_prob,
    met_before_w,
    met_before_s,
    met_before_o,
    max_contacts,
    wfh_prob,
    app_cov,
    p_pop_test,
    policy_adherence,
):
    """
    This is the implementation of the Kucharski approach (see cmmid.py), with all typos fixed and some other improvements
    """

    if case.under18:
        wfh = rng.uniform() < 1 - go_to_school_prob
        met_before_w = met_before_s
    else:
        wfh = rng.uniform() < wfh_prob

    # Assume testing all symptomatic. tested if symptomatic AND comply with policy. TODO: Assumes no test lag
    got_symptom_tested = (
        (rng.uniform() < policy_adherence) and case.symptomatic and do_symptom_testing
    )
    symptom_test_day = case.day_noticed_symptoms

    # get randomly tested if doing random testing
    got_random_tested = (rng.uniform() < p_pop_test) and do_pop_testing
    # Generate a random test day to be tested on. TODO: include incubation period
    random_test_day = rng.randint(6)  # TODO: current infective period is 5 days.

    # For speed pull the shape of these arrays once
    # home_contacts = contacts.home[:, 1]
    work_contacts = contacts.work[:, 1]
    othr_contacts = contacts.other[:, 1]

    # home_infected_day = contacts.home[
    #     :, 0
    # ]  # need to work out if isolating prevents an in home infection.
    home_infections = (contacts.home[:, 0] >= 0).astype(bool)
    work_infections = (contacts.work[:, 0] >= 0).astype(bool)
    othr_infections = (contacts.other[:, 0] >= 0).astype(bool)

    n_home = home_infections.shape[0]
    n_work = work_infections.shape[0]
    n_othr = othr_infections.shape[0]

    # If the person got tested
    if got_symptom_tested or got_random_tested:
        # TRACING
        # If policy to do app tracing
        if do_app_tracing:
            has_app = rng.uniform() < app_cov
            # If has app, we can trace contacts through the app. Assume home contacts not needed to trace this way
            if has_app:
                # Trace contacts based on app coverage
                work_contacts_trace_app = rng.binomial(
                    n=1, p=app_cov, size=n_work
                ).astype(bool)
                othr_contacts_trace_app = rng.binomial(
                    n=1, p=app_cov, size=n_othr
                ).astype(bool)
            else:
                work_contacts_trace_app = np.zeros(shape=n_work, dtype=bool)
                othr_contacts_trace_app = np.zeros(shape=n_othr, dtype=bool)
        else:
            work_contacts_trace_app = np.zeros(shape=n_work, dtype=bool)
            othr_contacts_trace_app = np.zeros(shape=n_othr, dtype=bool)

        # If policy of manual tracing
        if do_manual_tracing:
            # Prob of manual tracing is a base chance, modified by the chance the person knows who the contact is.
            work_contacts_trace_manual = rng.binomial(
                n=1, p=manual_work_trace_prob * met_before_w, size=n_work
            ).astype(bool)
            othr_contacts_trace_manual = rng.binomial(
                n=1, p=manual_othr_trace_prob * met_before_o, size=n_othr
            ).astype(bool)
        else:
            # home_contacts_trace_manual = np.zeros(shape=n_home, dtype=bool)
            work_contacts_trace_manual = np.zeros(shape=n_work, dtype=bool)
            othr_contacts_trace_manual = np.zeros(shape=n_othr, dtype=bool)

        # TODO: Different from Kucharski
        # If we have the household isolation policy, (TODO: makes no sense without the other tracing policies)
        # Isolate the household. TODO: Doesn't distinguish between prevented infections and traced
        if do_household_isolation:
            home_contacts_traced = np.ones(shape=n_home, dtype=bool)
        else:
            home_contacts_traced = np.zeros(shape=n_home, dtype=bool)

        # Traced if traced either way
        work_contacts_traced = work_contacts_trace_app | work_contacts_trace_manual
        othr_contacts_traced = othr_contacts_trace_app | othr_contacts_trace_manual

        # Compute trace statistics
        manual_traces = (
            work_contacts_trace_manual.sum() + othr_contacts_trace_manual.sum()
        )
        # app_traces = work_contacts_trace_app.sum() + othr_contacts_trace_app.sum()

        # Work out if each contact will adhere to the policy
        home_contacts_adherence = rng.binomial(
            n=1, p=policy_adherence, size=n_home
        ).astype(bool)
        work_contacts_adherence = rng.binomial(
            n=1, p=policy_adherence, size=n_work
        ).astype(bool)
        othr_contacts_adherence = rng.binomial(
            n=1, p=policy_adherence, size=n_othr
        ).astype(bool)

        # Compute which contact will isolate because of the contact trace
        # TODO: Assumes traced contact does not count towards R
        home_contacts_isolated = home_contacts_traced & home_contacts_adherence
        work_contacts_isolated = work_contacts_traced & work_contacts_adherence
        othr_contacts_isolated = othr_contacts_traced & othr_contacts_adherence

        # ISOLATING

        # Find the day isolated on to determine when removed from contact.
        # TODO: Simplify this logic.
        if got_symptom_tested and got_random_tested:
            isolate_day = np.minimum(symptom_test_day, random_test_day)
        elif got_symptom_tested:
            isolate_day = symptom_test_day
        elif got_random_tested:
            isolate_day = random_test_day

        # Prevent contact after isolation day
        if do_individual_isolation:
            home_contacts_prevented = (home_infections >= isolate_day).astype(bool)
            work_contacts_prevented = (work_contacts >= isolate_day).astype(bool)
            othr_contacts_prevented = (othr_contacts >= isolate_day).astype(bool)
        else:
            home_contacts_prevented = np.zeros(shape=n_home, dtype=bool)
            work_contacts_prevented = np.zeros(shape=n_work, dtype=bool)
            othr_contacts_prevented = np.zeros(shape=n_othr, dtype=bool)
    else:
        # No tracing took place if they didn't get tested positive.
        home_contacts_isolated = np.zeros(shape=n_home, dtype=bool)
        work_contacts_isolated = np.zeros(shape=n_work, dtype=bool)
        othr_contacts_isolated = np.zeros(shape=n_othr, dtype=bool)

        # Default cases prevented (none)
        home_contacts_prevented = np.zeros(shape=n_home, dtype=bool)
        work_contacts_prevented = np.zeros(shape=n_work, dtype=bool)
        othr_contacts_prevented = np.zeros(shape=n_othr, dtype=bool)

        manual_traces = 0
        # app_traces = 0

    # Compute reduction in contacts due to contact limiting policy. Independent of test status.
    othr_contacts_limited = ~_limit_contact(othr_contacts, max_contacts)

    # Compute reduction in contacts due to wfh. Independent of test status.
    if wfh:
        work_contacts_wfh_limited = np.ones_like(work_contacts).astype(bool)
    else:
        work_contacts_wfh_limited = np.zeros_like(work_contacts).astype(bool)

    # Compute the base reproduction rate
    base_rr = home_infections.sum() + work_infections.sum() + othr_infections.sum()

    # Compute the reproduction rate due to the policy
    # Remove infections due to case isolation
    home_infections_post_policy = home_infections & ~(home_contacts_prevented)
    work_infections_post_policy = work_infections & ~(work_contacts_prevented)
    othr_infections_post_policy = othr_infections & ~(othr_contacts_prevented)

    # Count traced contacts as not included in the R TODO: make a proportion
    home_infections_post_policy = home_infections_post_policy & ~home_contacts_isolated
    work_infections_post_policy = work_infections_post_policy & ~work_contacts_isolated
    othr_infections_post_policy = othr_infections_post_policy & ~othr_contacts_isolated

    # Remove contacts not made due to work from home
    work_infections_post_policy = (
        work_infections_post_policy & ~work_contacts_wfh_limited
    )

    # Remove other contact limiting contacts
    othr_infections_post_policy = othr_infections_post_policy & ~othr_contacts_limited

    # Count the reduced infection rate
    reduced_rr = (
        home_infections_post_policy.sum()
        + work_infections_post_policy.sum()
        + othr_infections_post_policy.sum()
    )

    return {
        RETURN_KEYS.base_r: base_rr,
        RETURN_KEYS.reduced_r: reduced_rr,
        RETURN_KEYS.man_trace: manual_traces,
    }

ext: py

reason: import numpy


text:
import multiprocessing as mp
import os
from argparse import ArgumentParser

import librosa
import numpy as np
import soundfile as sf
from numpy import ndarray
from tqdm import tqdm

from hparams import hp

def save_feature(i_speech: int, s_path_speech: str, speech: ndarray) -> tuple:
    spec_clean = np.ascontiguousarray(librosa.stft(speech, **hp.kwargs_stft))
    mag_clean = np.ascontiguousarray(np.abs(spec_clean)[..., np.newaxis])
    signal_power = np.mean(np.abs(speech)**2)
    list_dict = []
    list_snr_db = []
    for _ in enumerate(args.num_snr):
        snr_db = -6*np.random.rand()
        list_snr_db.append(snr_db)
        snr = librosa.db_to_power(snr_db)
        noise_power = signal_power / snr
        noisy = speech + np.sqrt(noise_power) * np.random.randn(len(speech))
        spec_noisy = librosa.stft(noisy, **hp.kwargs_stft)
        spec_noisy = np.ascontiguousarray(spec_noisy)

        list_dict.append(
            dict(spec_noisy=spec_noisy,
                 spec_clean=spec_clean,
                 mag_clean=mag_clean,
                 path_speech=s_path_speech,
                 length=len(speech),
                 )
        )
    return i_speech, list_snr_db, list_dict

if __name__ == "__main__":
    parser = ArgumentParser()

    parser.add_argument('kind_data', choices=('TRAIN', 'TEST'))
    args = hp.parse_argument(parser, print_argument=False)
    args.kind_data = args.kind_data.lower()

    path_speech_folder = hp.dict_path[f'speech_{args.kind_data}']
    flist_speech = (list(path_speech_folder.glob('**/*.WAV')) +
                    list(path_speech_folder.glob('**/*.wav')))

    path_feature = hp.dict_path[f'feature_{args.kind_data}']
    os.makedirs(path_feature, exist_ok=True)

    pool = mp.Pool(
        processes=mp.cpu_count()//2 - 1,
        initializer=lambda: np.random.seed(os.getpid()),
    )
    results = []
    for i_speech, path_speech in enumerate(flist_speech):
        speech = sf.read(str(path_speech))[0].astype(np.float32)
        results.append(
            pool.apply_async(save_feature,
                             (i_speech, str(path_speech), speech),
                             )
        )

    pbar = tqdm(results, dynamic_ncols=True)
    for result in pbar:
        i_speech, list_snr_db, list_dict = result.get()
        for snr_db, dict_result in zip(list_snr_db, list_dict):
            np.savez(path_feature / hp.form_feature.format(i_speech, snr_db),
                     **dict_result,
                     )

max_stars_repo_name: Sytronik/deep-griffinlim-iteration

max_issues_repo_name: Sytronik/deep-griffinlim-iteration

max_forks_repo_name: Sytronik/deep-griffinlim-iteration

reason: import numpy,from numpy


text:
import numpy as np
import random
import torch
import torchvision
from torch.autograd import Variable
from torchvision import transforms, models
import torch.nn.functional as F
from model import *
from Resnet import *

def cosine_anneal_schedule(t, nb_epoch, lr):
    cos_inner = np.pi * (t % (nb_epoch))  # t - 1 is used when t has 1-based indexing.
    cos_inner /= (nb_epoch)
    cos_out = np.cos(cos_inner) + 1

    return float(lr / 2 * cos_out)

def load_model(model_name, pretrain=True, require_grad=True):
    print('==> Building model..')
    if model_name == 'resnet50_pmg':
        net = resnet50(pretrained=pretrain)
        for param in net.parameters():
            param.requires_grad = require_grad
        net = PMG(net, 512, 200)

    return net

def model_info(model):  # Plots a line-by-line description of a PyTorch model
    n_p = sum(x.numel() for x in model.parameters())  # number parameters
    n_g = sum(x.numel() for x in model.parameters() if x.requires_grad)  # number gradients
    print('\n%5s %50s %9s %12s %20s %12s %12s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma'))
    for i, (name, p) in enumerate(model.named_parameters()):
        name = name.replace('module_list.', '')
        print('%5g %50s %9s %12g %20s %12.3g %12.3g' % (
            i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))
    print('Model Summary: %g layers, %g parameters, %g gradients\n' % (i + 1, n_p, n_g))

def jigsaw_generator(images, n):
    l = []
    for a in range(n):
        for b in range(n):
            l.append([a, b])
    block_size = 448 // n
    rounds = n ** 2
    random.shuffle(l)
    jigsaws = images.clone()
    for i in range(rounds):
        x, y = l[i]
        temp = jigsaws[..., 0:block_size, 0:block_size].clone()
        jigsaws[..., 0:block_size, 0:block_size] = jigsaws[..., x * block_size:(x + 1) * block_size,
                                                y * block_size:(y + 1) * block_size].clone()
        jigsaws[..., x * block_size:(x + 1) * block_size, y * block_size:(y + 1) * block_size] = temp

    return jigsaws

def test(net, criterion, batch_size):
    net.eval()
    use_cuda = torch.cuda.is_available()
    test_loss = 0
    correct = 0
    correct_com = 0
    total = 0
    idx = 0
    device = torch.device("cuda:0,1")

    transform_test = transforms.Compose([
        transforms.Scale((550, 550)),
        transforms.CenterCrop(448),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    ])
    testset = torchvision.datasets.ImageFolder(root='./bird/test',
                                               transform=transform_test)
    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True, num_workers=4)

    for batch_idx, (inputs, targets) in enumerate(testloader):
        idx = batch_idx
        if use_cuda:
            inputs, targets = inputs.to(device), targets.to(device)
        inputs, targets = Variable(inputs, volatile=True), Variable(targets)
        output_1, output_2, output_3, output_concat= net(inputs)
        outputs_com = output_1 + output_2 + output_3 + output_concat

        loss = criterion(output_concat, targets)

        test_loss += loss.item()
        _, predicted = torch.max(output_concat.data, 1)
        _, predicted_com = torch.max(outputs_com.data, 1)
        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()
        correct_com += predicted_com.eq(targets.data).cpu().sum()

        if batch_idx % 50 == 0:
            print('Step: %d | Loss: %.3f | Acc: %.3f%% (%d/%d) |Combined Acc: %.3f%% (%d/%d)' % (
            batch_idx, test_loss / (batch_idx + 1), 100. * float(correct) / total, correct, total, 100. * float(correct_com) / total, correct_com, total))

    test_acc = 100. * float(correct) / total
    test_acc_en = 100. * float(correct_com) / total
    test_loss = test_loss / (idx + 1)

    return test_acc, test_acc_en, test_loss


text:
# standard
import datetime as dt
from dateutil.parser import parse
import requests

# third party
import bs4
from bs4 import BeautifulSoup
import numpy as np
import pandas as pd

def download_winter_data(weather_station_id, year=2019):
    datas = []
    granularity = "daily"
    year = year
    months = range(11, 13)
    days = range(1, 32)
    for month in months:
        for day in days:
            try:
                url = f"https://www.wunderground.com/dashboard/pws/{weather_station_id}/table/{year}-{month}-{day}/{year}-{month}-{day}/{granularity}"
                print(url)
                body = get_and_parse_url(url)
                df = build_dataset(body, granularity)
                df["date"] = dt.date(year, month, day)
                print(df.shape)
                datas.append(df)
            except Exception as e:
                print(e)

    year += 1
    months = range(1, 3)
    days = range(1, 32)
    for month in months:
        for day in days:
            try:
                url = f"https://www.wunderground.com/dashboard/pws/{weather_station_id}/table/{year}-{month}-{day}/{year}-{month}-{day}/{granularity}"
                print(url)
                body = get_and_parse_url(url)
                df = build_dataset(body, granularity)
                df["date"] = dt.date(year, month, day)
                print(df.shape)
                datas.append(df)
            except Exception as e:
                print(e)

    df = pd.concat(datas)
    datetimeys = []
    for date, time in zip(df.date.values, df.time.values):
        datetimey = dt.datetime.combine(date, parse(time).time())
        datetimeys.append(datetimey)
    df["datetime"] = datetimeys
    df = df.sort_values("datetime").reset_index(drop=True)

    return df

def get_and_parse_url(url):
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html.parser')
    html = list(soup.children)[1]
    body = list(html.children)[2] # list(body.children)[1]

    return body

def build_dataset(body, granularity):
    class_ = "ng-star-inserted"
    inserts = [
        p.get_text().split("\xa0")[0]
        for p in body.find_all(class_=class_)
    ]

    if granularity == "monthly":
        for ix, insert in enumerate(inserts):
            if insert == last_day:
                break
        inserts = inserts[ix + 1:-3]

        # 1 date col, 15 feature cols, 15 unit cols
        df = pd.DataFrame(
            np.array(inserts).reshape((int(len(inserts)/31), 31)),
            columns = [
                "date",
                "temperature_f_high",
                "temperature_f_high_unit",
                "temperature_f_avg",
                "temperature_f_avg_unit",
                "temperature_f_low",
                "temperature_f_low_unit",
                "dew_point_f_high",
                "dew_point_f_high_unit",
                "dew_point_f_avg",
                "dew_point_f_avg_unit",
                "dew_point_f_low",
                "dew_point_f_low_unit",
                "humidity_percent_high",
                "humidity_percent_high_unit",
                "humidity_percent_avg",
                "humidity_percent_avg_unit",
                "humidity_percent_low",
                "humidity_percent_low_unit",
                "windspeed_mph_high",
                "windspeed_mph_high_unit",
                "windspeed_mph_avg",
                "windspeed_mph_avg_unit",
                "windspeed_mph_low",
                "windspeed_mph_low_unit",
                "pressure_in_high",
                "pressure_in_high_unit",
                "pressure_in_low",
                "pressure_in_low_unit",
                "precip_accum_in_sum",
                "precip_accum_in_sum_unit",
            ]
        )
        df["date"] = [date.split(year)[0]+year for date in df["date"]]
        df = df[[col for col in df.columns if "unit" not in col]]
        df = df.set_index(df["date"])

    if granularity == "daily":
        last_time = _get_last_time_str(inserts)
        for ix, insert in enumerate(inserts):
            if insert == last_time:
                break
        inserts = inserts[ix + 1:-3]

        # 1 time col, 8 feature cols, 8 unit cols
        df = pd.DataFrame(
            np.array(inserts).reshape((int(len(inserts)/17), 17)),
            columns = [
                "time",
                "temperature_f",
                "temperature_f_unit",
                "dew_point_f",
                "dew_point_f_unit",
                "humidity_percent",
                "humidity_percent_unit",
                "windspeed_mph",
                "windspeed_mph_unit",
                "windspeed_gust_mph",
                "windspeed_gust_mph_unit",
                "pressure_in",
                "pressure_in_unit",
                "precip_rate_in",
                "precip_rate_in_unit",
                "precip_accum_in",
                "precip_accum_in_unit",
            ]
        )
        df["time"] = [t.split("M")[0]+"M" for t in df["time"]]
        df = df[[col for col in df.columns if "unit" not in col]]

    for col in df.columns:
        if col not in ["date", "time"]:
            df[col] = df[col].replace("--", np.nan).astype(float)

    return df

def _get_last_time_str(inserts):
    time_inserts = []
    for ix, insert in enumerate(inserts):
        if any(("AM" in insert, "PM" in insert)):
            if len(insert) < 10:
                time_inserts.append(insert)

    dts = [parse(t.split("M")[0] + "M") for t in time_inserts]
    return max(dts).strftime("%-I:%M %p")

# for chill hours calculation
def get_chill_hours(df, MAX_MINUTES):
    # total hours below 45
    chill_hours_sub_df = df.loc[df.temperature_f < 45]
    chill_hours = chill_hours_sub_df.duration.sum() / 3600
    # confirmed chill hours = confirmed durations (<MIN_MINUTES dur)
    # + first MIN_MINUTES mins of every unconfirmed duration
    chill_hours_confd = get_confirmed_chill_hours(chill_hours_sub_df, MAX_MINUTES)
    return chill_hours, chill_hours_confd

def get_confirmed_chill_hours(chill_hours_sub_df, MAX_MINUTES):
    return (
        chill_hours_sub_df.loc[
            df.duration <= (60 * MAX_MINUTES)
        ].duration.sum() / 3600
        + chill_hours_sub_df.loc[
            df.duration > (60 * MAX_MINUTES)
        ].duration.count() * (MAX_MINUTES * 60) / 3600
    )

def get_modified_chill_hours(df, MAX_MINUTES):
    # hours between 32 and 45
    chill_hours_sub_df = df.loc[
        (df.temperature_f >= 32) & (df.temperature_f < 45)
    ]
    mod_chill_hours = chill_hours_sub_df.duration.sum() / 3600
    # confirmed hours between 32 and 45
    mod_chill_hours_confd = get_confirmed_chill_hours(chill_hours_sub_df, MAX_MINUTES)
    return mod_chill_hours, mod_chill_hours_confd

def get_utah_chill_units(df):
    # utah model
    # 1 hour below 34°F = 0.0 chill unit
    # 1 hour 34.01 - 36°F = 0.5 chill unit
    # 1 hour 36.01 - 48°F = 1.0 chill unit
    # 1 hour 48.01 - 54°F = 0.5 chill unit
    # 1 hour 54.01 - 60°F = 0.0 chill unit
    # 1 hour 60.01 - 65°F = -0.5 chill unit
    # 1 hour >65.01°F = -1.0 chill unit
    sub_34_df = df.loc[df.temperature_f < 34]
    btw_34_and_36_df = df.loc[(df.temperature_f >= 34) & (df.temperature_f < 36)]
    btw_36_and_48_df = df.loc[(df.temperature_f >= 36) & (df.temperature_f < 48)]
    btw_48_and_54_df = df.loc[(df.temperature_f >= 48) & (df.temperature_f < 54)]
    btw_54_and_60_df = df.loc[(df.temperature_f >= 54) & (df.temperature_f < 60)]
    btw_60_and_65_df = df.loc[(df.temperature_f >= 60) & (df.temperature_f < 65)]
    over_65_df = df.loc[df.temperature_f >= 65]
    chill_units = (
        sub_34_df.duration.sum() / 3600 * 0
        + btw_34_and_36_df.duration.sum() / 3600 * 0.5
        + btw_36_and_48_df.duration.sum() / 3600 * 1
        + btw_48_and_54_df.duration.sum() / 3600 * 0.5
        + btw_54_and_60_df.duration.sum() / 3600 * 0
        + btw_60_and_65_df.duration.sum() / 3600 * -0.5
        + over_65_df.duration.sum() / 3600 * -1
    )
    chill_units_confd = (
        get_confirmed_chill_hours(sub_34_df, MAX_MINUTES) * 0
        + get_confirmed_chill_hours(btw_34_and_36_df, MAX_MINUTES) * 0.5
        + get_confirmed_chill_hours(btw_36_and_48_df, MAX_MINUTES) * 1
        + get_confirmed_chill_hours(btw_48_and_54_df, MAX_MINUTES) * 0.5
        + get_confirmed_chill_hours(btw_54_and_60_df, MAX_MINUTES) * 0
        + get_confirmed_chill_hours(btw_60_and_65_df, MAX_MINUTES) * -0.5
        + get_confirmed_chill_hours(over_65_df, MAX_MINUTES) * -1
    )

    return chill_units, chill_units_confd


text:
#!/usr/bin/env python2

import subprocess
import atexit
from math import pi
from chips.api.api import Chip, Stimulus, Response, Wire, Component

try:
    from matplotlib import pyplot
except ImportError:
    print "You need matplotlib to run this script!"
    exit(0)
try:
    from numpy import arange
except ImportError:
    print "You need numpy to run this script!"
    exit(0)

def test():
    run_c("taylor.c")
    x = [float(i) for i in open("x")]
    sin_x = [float(i) for i in open("sin_x")]
    cos_x = [float(i) for i in open("cos_x")]

def test():

    chip = Chip("taylor")

    stimulus = arange(-2*pi, 2.0*pi, pi/25)
    x = Stimulus(chip, "x", "double", stimulus)
    sin_x = Response(chip, "sin_x", "double")
    cos_x = Response(chip, "cos_x", "double")

    #create a filter component using the C code
    sqrt = Component("taylor.c")

    #add an instance to the chip
    sqrt(
        chip,
        inputs = {
            "x":x,
        },
        outputs = {
            "sin_x":sin_x,
            "cos_x":cos_x,
        },
    )

    #run the simulation
    chip.simulation_reset()
    while len(cos_x) < len(x):
        chip.simulation_step()

    x = list(x)
    sin_x = list(sin_x)[:100]
    cos_x = list(cos_x)[:100]

    pyplot.xticks(
        [-2.0*pi, -pi, 0, pi,  2.0*pi],
        [r'$-2\pi$', r"$-\pi$", r'$0$', r'$\pi$', r'$2\pi$'])
    pyplot.plot(x, sin_x, label="sin(x)")
    pyplot.plot(x, cos_x, label="cos(x)")
    pyplot.ylim(-1.1, 1.1)
    pyplot.xlim(-2.2 * pi, 2.2 * pi)
    pyplot.title("Trig Functions")
    pyplot.xlabel("x (radians)")
    pyplot.legend(loc="upper left")
    pyplot.savefig("../docs/source/examples/images/example_2.png")
    pyplot.show()

def indent(lines):
    return "\n    ".join(lines.splitlines())

def generate_docs():

    documentation = """

Approximating Sine and Cosine functions using Taylor Series
===========================================================

In this example, we calculate an approximation of the cosine functions using
the `Taylor series <http://en.wikipedia.org/wiki/Taylor_series>`_:

.. math::

    \\cos (x) = \\sum_{n=0}^{\\infty} \\frac{(-1)^n}{(2n)!} x^{2n}

The following example uses the Taylor Series approximation to generate the Sine
and Cosine functions. Successive terms of the taylor series are calculated
until successive approximations agree to within a small degree. A Sine
function is also synthesised using the identity :math:`sin(x) \\equiv cos(x-\\pi/2)`

.. code-block:: c

    %s

A simple test calculates Sine and Cosine for the range :math:`-2\\pi <= x <= 2\\pi`.

.. image:: images/example_2.png

"""%indent(open("taylor.c").read())

    document = open("../docs/source/examples/example_2.rst", "w").write(documentation)

test()
generate_docs()

reason: from numpy


text:
"""Test cases for the GNN metacontroller approach."""

import numpy as np
import pytest
from gym.spaces import Box

from predicators.src import utils
from predicators.src.approaches import ApproachFailure, create_approach
from predicators.src.approaches.gnn_metacontroller_approach import \
    GNNMetacontrollerApproach
from predicators.src.datasets import create_dataset
from predicators.src.envs import create_new_env
from predicators.src.gnn.gnn_utils import get_single_model_prediction
from predicators.src.option_model import _OptionModelBase
from predicators.src.settings import CFG
from predicators.src.structs import Action, Dataset, LowLevelTrajectory, \
    ParameterizedOption, Predicate, State, Task, Type

class _MockGNNMetacontrollerApproach(GNNMetacontrollerApproach):
    """A mock approach that exposes some methods and fields for testing."""

    def graphify_single_input(self, state, atoms, goal):
        """Expose self._graphify_single_input()."""
        return self._graphify_single_input(state, atoms, goal)

    def extract_output_from_graph(self, graph_output, object_to_node):
        """Expose self._extract_output_from_graph()."""
        return self._extract_output_from_graph(graph_output, object_to_node)

    @property
    def gnn(self):
        """Expose self._gnn."""
        return self._gnn

class _MockOptionModel(_OptionModelBase):
    """A mock option model that raises an EnvironmentFailure."""

    def __init__(self, simulator):
        self._simulator = simulator

    def get_next_state_and_num_actions(self, state, option):
        raise utils.EnvironmentFailure("Mock env failure")

@pytest.mark.parametrize("env_name,num_epochs", [("cover", 100),
                                                 ("cover_typed_options", 20)])
def test_gnn_metacontroller_approach_with_envs(env_name, num_epochs):
    """Tests for GNNMetacontrollerApproach class on environments."""
    utils.reset_config({
        "env": env_name,
        "num_train_tasks": 3,
        "num_test_tasks": 3,
        "gnn_num_epochs": num_epochs,
        "gnn_do_normalization": True,
        "sampler_learner": "oracle",
        "horizon": 10
    })
    env = create_new_env(env_name)
    train_tasks = env.get_train_tasks()
    approach = create_approach("gnn_metacontroller", env.predicates,
                               env.options, env.types, env.action_space,
                               train_tasks)
    dataset = create_dataset(env, train_tasks, env.options)
    assert approach.is_learning_based
    task = env.get_test_tasks()[0]
    with pytest.raises(AssertionError):  # haven't learned yet!
        approach.solve(task, timeout=CFG.timeout)
    approach.learn_from_offline_dataset(dataset)
    policy = approach.solve(task, timeout=CFG.timeout)
    # Try executing the policy.
    if num_epochs > 50:
        policy(task.init)  # should be able to sample an option
        # Cover the case where a sampled option leads to an EnvironmentFailure.
        approach._option_model = _MockOptionModel(env.simulate)  # pylint: disable=protected-access
        policy = approach.solve(task, timeout=CFG.timeout)
        with pytest.raises(ApproachFailure) as e:
            policy(task.init)
        assert "GNN metacontroller could not sample an option" in str(e)
    else:
        with pytest.raises(ApproachFailure) as e:
            policy(task.init)  # should NOT be able to sample an option
        assert "GNN metacontroller could not sample an option" in str(e)
    # Test loading.
    approach2 = create_approach("gnn_metacontroller", env.predicates,
                                env.options, env.types, env.action_space,
                                train_tasks)
    approach2.load(online_learning_cycle=None)

@pytest.mark.parametrize("min_data_for_nsrt", [0, 2])
def test_gnn_metacontroller_approach_special_cases(min_data_for_nsrt):
    """Tests for special cases of the GNNMetacontrollerApproach class."""
    utils.reset_config({
        "env": "cover",
        "gnn_num_epochs": 20,
        "gnn_use_validation_set": False,
        "horizon": 10,
        "min_data_for_nsrt": min_data_for_nsrt,
    })
    cup_type = Type("cup_type", ["feat1"])
    bowl_type = Type("bowl_type", ["feat1"])
    cup = cup_type("cup")
    bowl = bowl_type("bowl")

    action_space = Box(0, 1, (1, ))
    params_space = Box(0, 1, (0, ))

    def _policy(_1, _2, _3, p):
        return Action(p)

    def _initiable(s, _1, _2, _3):
        return s[cup][0] > 0.25

    def _solved_classifier(s, o):
        return s[o[0]][0] < 0.5

    Move = ParameterizedOption("Move", [cup_type], params_space, _policy,
                               _initiable, lambda _1, _2, _3, _4: True)
    Dump = ParameterizedOption("Dump", [cup_type], params_space, _policy,
                               _initiable, lambda _1, _2, _3, _4: True)
    Solved = Predicate("Solved", [cup_type], _solved_classifier)
    state = State({cup: [0.3]})
    action1 = Move.ground([cup], np.array([])).policy(state)
    next_state1 = State({cup: [0.3]})
    action2 = Dump.ground([cup], np.array([])).policy(state)
    next_state2 = State({cup: [0.8]})
    action3 = Dump.ground([cup], np.array([])).policy(state)
    next_state3 = State({cup: [0.3]})
    train_tasks = [Task(state, set())]
    # Note: from test_task.init, both Move and Dump are always non-initiable.
    test_task = Task(State({cup: [0.0]}), set())

    approach = _MockGNNMetacontrollerApproach({Solved}, {Move, Dump},
                                              {cup_type}, action_space,
                                              train_tasks)
    approach.learn_from_offline_dataset(
        Dataset([
            LowLevelTrajectory([state, next_state1], [action1],
                               _is_demo=True,
                               _train_task_idx=0),
            LowLevelTrajectory([state, next_state2], [action2],
                               _is_demo=True,
                               _train_task_idx=0),
            LowLevelTrajectory([state, next_state3], [action3],
                               _is_demo=True,
                               _train_task_idx=0),
            # For coverage, this is not a demo, so it will be ignored.
            LowLevelTrajectory([state, next_state1], [action1]),
        ]))

    # Cover the case where the approach can't sample an initiable option.
    policy = approach.solve(test_task, timeout=CFG.timeout)
    with pytest.raises(ApproachFailure) as e:
        policy(test_task.init)
    assert "GNN metacontroller could not sample an option" in str(e)
    # Cover the case where the approach can't select an object.
    in_graph, _ = approach.graphify_single_input(test_task.init, set(), set())
    hacked_object_to_node = {bowl: 0}
    out_graph = get_single_model_prediction(approach.gnn, in_graph)
    with pytest.raises(ApproachFailure) as e:
        approach.extract_output_from_graph(out_graph, hacked_object_to_node)
    assert "GNN metacontroller could not select an object" in str(e)


text:
# bokeh basics
import pandas as pd
import numpy as np
import datetime
from math import pi
from itertools import chain

from datetime import date
from collections import defaultdict
from bokeh.layouts import column, row
from bokeh.plotting import figure
from bokeh.models import ColumnDataSource, HoverTool, Select, Panel, DateRangeSlider, Div, Title
from bokeh.transform import cumsum

from bokeh.palettes import Category10_7, Turbo256

from scripts.plot_style import style
from scripts.analyser import ConvoStats

def daily_stats_tab(convoStats, convoSelection):

    # Daily by-party and total message counts
    def make_timeseries_datasets(convoTitle, startDate=None, endDate=None):
        convo: analyser.ConvoStats = next(
            (x for x in convoStats if x.title == convoTitle))
        participants = convo.participants
        participantToId = {x: i for i, x in enumerate(participants)}
        totalsId = len(participants)
        participantToId['Total'] = totalsId

        xs = [[] for _ in participants] + [[]]
        ys = [[] for _ in participants] + [[]]
        color = Category10_7 if len(participants) < 7 else Turbo256
        colors = [color[i] for i in range(len(participants)+1)]
        labels = sorted(participants) + ['Total']

        for date in convo.dailyCountsBySender.keys():
            convertedDate = pd.to_datetime(date)
            if startDate is not None and endDate is not None and (convertedDate < startDate or convertedDate > endDate):
                continue

            for i, (sender, count) in enumerate(convo.dailyCountsBySender[date].items()):
                participantId = participantToId[sender]

                xs[participantId].append(convertedDate)
                ys[participantId].append(count)

            xs[totalsId].append(convertedDate)
            ys[totalsId].append(sum(convo.dailyCountsBySender[date].values()))

        # I need an invisible scatterplot for nice tooltips, because multiline tooltips don't work well
        totalX = list(chain.from_iterable(xs))
        totalY = list(chain.from_iterable(ys))
        totalLabels = list(chain.from_iterable(
            [[x]*len(xs[participantToId[x]]) for x in labels]))

        return (ColumnDataSource(data={'x': xs, 'y': ys, 'color': colors, 'label': labels}),
                ColumnDataSource(data={'x': totalX, 'y': totalY, 'label': totalLabels}))

    def make_piechart_dataset(convoTitle, startDate=None, endDate=None):
        convo: analyser.ConvoStats = next(
            (x for x in convoStats if x.title == convoTitle))

        df = pd.DataFrame(columns=[
            'sender', 'messageCount', 'messageCountAngle', 'f_messageCount',
            'wordCount', 'wordCountAngle', 'f_wordCount', 'initiationCount', 'initiationCountAngle', 'f_initiationCount', 'color'])
        color = Category10_7 if len(convo.participants) <= 7 else Turbo256

        allMessages = convo.messages
        if startDate is not None and endDate is not None:
            allMessages = list(filter(lambda m: m.datetime.date() >=
                                      startDate and m.datetime.date() <= endDate, allMessages))
        totalWordCount = sum(len(x.content.split()) for x in allMessages)
        participantCount = len(convo.participants)

        initiationsBySender = defaultdict(int)
        curConvoParticipants = set()
        lastMessage = ''
        for i, message in enumerate(allMessages):
            if i == 0:
                # first message, so conversation initiated
                initiationsBySender[message.sender] += 1
            else:
                timeDiff = message.datetime - allMessages[i-1].datetime
                # It is assumed that if 4h passed since last message, a new conversation has been initiated
                hoursPassed = timeDiff.total_seconds() // (60*60)
                # Extra conditions: if the last convo only had one participant or the last message was a question, don't count a new initiation
                # TODO: Perhaps I should apply the same checks when calculating conversation stats, though for durations between messages etc just the time check is probably better
                if hoursPassed >= 4 and '?' not in lastMessage and len(curConvoParticipants) > 1:
                    initiationsBySender[message.sender] += 1
                    curConvoParticipants = set()
            lastMessage = message.content
            curConvoParticipants |= {message.sender}
        totalInitiationCount = sum(initiationsBySender.values())

        for i, participant in enumerate(sorted(convo.participants)):
            messages = list(filter(lambda m: m.sender ==
                                   participant, allMessages))

            tdf = pd.DataFrame()
            tdf['sender'] = [participant]
            tdf['messageCount'] = [len(messages)]
            # The +1/+2 is to avoid division by zero if no messages are present in the interval
            # TODO: Investigate whether I need to care about div by 0 here and in other places
            tdf['messageCountAngle'] = [
                (len(messages) + 1)/(len(allMessages) + participantCount) * 2*pi]
            tdf['f_messageCount'] = [
                f'{len(messages)} messages ({len(messages)/len(allMessages)*100:.2f}%)']
            tdf['wordCount'] = [sum(len(x.content.split()) for x in messages)]
            tdf['wordCountAngle'] = [
                (tdf['wordCount'][0] + 1) / (totalWordCount + participantCount) * 2*pi]
            tdf['f_wordCount'] = [
                f'{tdf["wordCount"][0]} words ({tdf["wordCount"][0]/totalWordCount*100:.2f}%)']
            tdf['initiationCount'] = [initiationsBySender[participant]]
            tdf['initiationCountAngle'] = [
                initiationsBySender[participant] / totalInitiationCount * 2*pi]
            tdf['f_initiationCount'] = f'{tdf["initiationCount"][0]} initations ({tdf["initiationCount"][0]/totalInitiationCount*100:.2f}%)'
            tdf['color'] = color[i]
            df = df.append(tdf)

        return ColumnDataSource(df)

    def make_messages_display(convoTitle, startDate=None, endDate=None):
        convo: analyser.ConvoStats = next(
            (x for x in convoStats if x.title == convoTitle))

        allMessages = convo.messages
        if startDate is not None and endDate is not None:
            allMessages = list(filter(lambda m: m.datetime.date() >=
                                      startDate and m.datetime.date() <= endDate, allMessages))

        # TODO: A single long word will make the div ignore width settings and overflow the window
        rez = '
'
        for i, message in enumerate(allMessages):
            if i > 500:
                break
            rez += f'{message.sender} ({message.datetime.strftime("%Y/%m/%d %H:%M")}): {message.content}
'
        rez += '
'
        return Div(text=rez, sizing_mode='stretch_width')

    # Statistics for the conversations in the selected date range like average message length
    def make_stats_text(convoTitle, startDate=None, endDate=None):
        convo: analyser.ConvoStats = next(
            (x for x in convoStats if x.title == convoTitle))

        allMessages = convo.messages
        if startDate is not None and endDate is not None:
            allMessages = list(filter(lambda m: m.datetime.date() >=
                                      startDate and m.datetime.date() <= endDate, allMessages))

        totalMessageLensWords = defaultdict(int)
        messageCountsByParticipant = defaultdict(int)
        convoDurationSum = 0
        convoLenWordsSum = 0
        pauseBetweenConvosDurationSum = 0
        pauseBetweenMessagesInConvoSum = 0

        lastConvoStart = allMessages[0].datetime
        convoWordCount = len(allMessages[0].content.split())
        convoCount = 0
        convoMessageCount = 1
        convoPauseBetweenMessagesSum = 0

        for i, message in enumerate(allMessages):
            wordCount = len(message.content.split())
            totalMessageLensWords[message.sender] += wordCount
            messageCountsByParticipant[message.sender] += 1
            totalMessageLensWords['total'] += wordCount

            if i != 0:
                timeDiff = message.datetime - allMessages[i-1].datetime
                hoursPassed = timeDiff.total_seconds() // (60*60)
                if hoursPassed >= 4:
                    # A new conversation has begun
                    convoDuration = (
                        allMessages[i-1].datetime - lastConvoStart).total_seconds()
                    convoDurationSum += convoDuration
                    convoCount += 1
                    convoLenWordsSum += convoWordCount
                    pauseDuration = timeDiff.total_seconds()
                    pauseBetweenConvosDurationSum += pauseDuration
                    pauseBetweenMessagesInConvoSum += convoPauseBetweenMessagesSum / convoMessageCount
                    convoMessageCount = 1
                    convoPauseBetweenMessagesSum = 0

                    convoWordCount = wordCount
                    lastConvoStart = message.datetime
                else:
                    convoWordCount += wordCount
                    convoPauseBetweenMessagesSum += timeDiff.total_seconds()
                    convoMessageCount += 1

        # In some edge cases there may be no messages sent to the participant
        if convoCount == 0:
            return ''

        hours, minutes = divmod(convoDurationSum/convoCount, 60*60)
        rez = '
'
        rez += f'Average conversation duration: {hours:.0f} h {minutes // 60:.0f} min
'
        if convoCount > 2:
            hours, minutes = divmod(
                pauseBetweenConvosDurationSum/(convoCount-1), 60*60)
            rez += f'Average pause between conversations duration: {hours:.0f} h {minutes // 60:.0f} min
'
        rez += f'Average conversation length: {len(allMessages) / convoCount:.1f} messages, {convoLenWordsSum // convoCount} words
'
        minutes, seconds = divmod(convoPauseBetweenMessagesSum/convoCount, 60)
        rez += f'Average time between messages in a conversation: {minutes:.1f} min {seconds:.1f} s
'
        rez += f'Average message length: {totalMessageLensWords["total"] // len(allMessages)} words
'
        for participant in convo.participants:
            if messageCountsByParticipant[participant] == 0:
                continue
            rez += f'Average length of messages from {participant}: {totalMessageLensWords[participant] // messageCountsByParticipant[participant]} words
'
        rez += '
'

        return rez

    def make_timeseries_plot(src, tooltipSrc):
        p = figure(plot_width=600, plot_height=600, title='Daily message counts by date',
                   x_axis_type='datetime', x_axis_label='Date', y_axis_label='Message count')

        p.multi_line(xs='x', ys='y', source=src, color='color',
                     line_width=3, legend_field='label', line_alpha=0.4)

        tooltipScatter = p.scatter('x', 'y', source=tooltipSrc, alpha=0)
        hover = HoverTool(tooltips=[('Message count', '@y'),
                                    ('Details', '@x{%F}, @label')],
                          formatters={'@x': 'datetime'},
                          mode='vline')  # vline means that tooltip will be shown when mouse is in a vertical line above glyph
        hover.renderers = [tooltipScatter]
        p.add_tools(hover)

        return p

    def _make_piechart(src, startAngle, endAngle, title, bottomTitle, tooltips):
        p = figure(plot_height=200, plot_width=280,
                   toolbar_location=None, title=title)

        p.wedge(x=0, y=1, radius=0.5, start_angle=startAngle,
                end_angle=endAngle, line_color='white', fill_color='color', source=src)
        p.axis.axis_label = None
        p.axis.visible = False
        p.grid.grid_line_color = None

        hover = HoverTool(tooltips=tooltips)
        p.add_tools(hover)

        p.add_layout(Title(text=bottomTitle, align="center"), "below")

        return p

    def make_piechart_plots(src):
        totalMessages = sum(src.data["messageCount"])
        p1 = _make_piechart(src, cumsum('messageCountAngle', include_zero=True), cumsum('messageCountAngle'),
                            'Messages sent by participant', f'Total messages: {totalMessages}',
                            [('Participant', '@sender'), ('Message count', '@f_messageCount')])

        totalWords = sum(src.data["wordCount"])
        p2 = _make_piechart(src, cumsum('wordCountAngle', include_zero=True), cumsum('wordCountAngle'),
                            'Word counts by participant', f'Total words: {totalWords}',
                            [('Participant', '@sender'), ('Word count', '@f_wordCount')])

        totalInitiations = sum(src.data["initiationCount"])
        p3 = _make_piechart(src, cumsum('initiationCountAngle', include_zero=True), cumsum('initiationCountAngle'),
                            'Conversations initiated by participant', f'Total conversations: {totalInitiations}',
                            [('Participant', '@sender'), ('Conversations initiated', '@f_initiationCount')])

        return column(p1, p2, p3)

    def _update_pie_bottom_labels():
        # Update the bottom titles of the piecharts
        for i, pie in enumerate(piePlots.children):
            # This is a bit hack-ish, but don't know a better way to do it
            if i == 0:
                totalMessages = sum(pieSrc.data["messageCount"])
                pie.below[1].text = f'Total messages: {totalMessages}'
            elif i == 1:
                totalWords = sum(pieSrc.data["wordCount"])
                pie.below[1].text = f'Total words: {totalWords}'
            elif i == 2:
                totalInitiations = sum(pieSrc.data["initiationCount"])
                pie.below[1].text = f'Total conversations: {totalInitiations}'

    def on_conversation_changed(attr, oldValue, newValue):
        convo: analyser.ConvoStats = next(
            (x for x in convoStats if x.title == newValue))

        # When switching to a new convo, update the date range slider to match convo data ranges
        initialDates = list(convo.dailyCountsBySender.keys())
        start = pd.to_datetime(initialDates[0]).date()
        end = pd.to_datetime(initialDates[-1]).date()
        dateSlider.start = start
        dateSlider.end = end
        dateSlider.value = (start, end)

        # TODO: There is some black magic going on here, find if there is a proper way to do this
        newScr, newTooltipSrc = make_timeseries_datasets(newValue)
        src.data.update(newScr.data)
        tooltipSrc.data.update(newTooltipSrc.data)
        newPieSrc = make_piechart_dataset(newValue)
        pieSrc.data.update(newPieSrc.data)

        _update_pie_bottom_labels()

        messageColumn.children = [make_messages_display(newValue)]

        statsDisplay.text = make_stats_text(newValue)

    def on_date_range_changed(attr, old, new):
        convoToPlot = convoSelection.value
        startDate, endDate = dateSlider.value_as_date

        # TODO: There is some black magic going on here, find if there is a proper way to do this
        new_src, newTooltipSrc = make_timeseries_datasets(
            convoToPlot, startDate, endDate)
        src.data.update(new_src.data)
        tooltipSrc.data.update(newTooltipSrc.data)
        newPieSrc = make_piechart_dataset(convoToPlot, startDate, endDate)
        pieSrc.data.update(newPieSrc.data)

        _update_pie_bottom_labels()

        messageColumn.children = [make_messages_display(
            convoToPlot, startDate, endDate)]

        statsDisplay.text = make_stats_text(convoToPlot, startDate, endDate)

    # A dropdown list to select a conversation
    conversationTitles = sorted([x.title for x in convoStats])
    convoSelection.on_change('value', on_conversation_changed)

    # A slider to select a date range for the analysis
    initialConvo: analyser.ConvoStats = next(
        (x for x in convoStats if x.title == conversationTitles[0]))
    initialDates = list(initialConvo.dailyCountsBySender.keys())
    start = pd.to_datetime(initialDates[0]).date()
    end = pd.to_datetime(initialDates[-1]).date()
    dateSlider = DateRangeSlider(
        title='Date interval', start=start, end=date.today(), value=(start, end), step=24*60*60*1000)
    dateSlider.on_change('value_throttled', on_date_range_changed)

    src, tooltipSrc = make_timeseries_datasets(
        conversationTitles[0], start, end)
    p = make_timeseries_plot(src, tooltipSrc)
    p = style(p)

    pieSrc = make_piechart_dataset(conversationTitles[0], start, end)
    piePlots = make_piechart_plots(pieSrc)

    messageContents = [make_messages_display(
        conversationTitles[0], start, end)]

    messageColumn = column(children=messageContents,
                           height=670, css_classes=['scrollable'], sizing_mode='stretch_width')

    statsDisplay = Div(text=make_stats_text(conversationTitles[0], start, end))
    statsColumn = column(children=[statsDisplay],
                         height=540, css_classes=['scrollable'])

    # create layout
    leftColumn = column(convoSelection, dateSlider, statsColumn)
    layout = row(leftColumn, p, piePlots, messageColumn)
    tab = Panel(child=layout, title='Daily statistics')

    return tab

max_stars_repo_name: Kodnot/facebook-message-analysis

max_issues_repo_name: Kodnot/facebook-message-analysis

max_forks_repo_name: Kodnot/facebook-message-analysis


text:
import shap
import pandas as pd
import numpy as np
import copy

class SHAPEstimatorWrapper():

        '''
        SHAP Wrapper for an estimator.

        This class aims to offer the coefficients (coef_) of any estimator as shapley values.

        The coefficients can be used as feature weights and help on feature selection or feature extraction.

        Tip: passing on the SHAP object prior fitting the estimator might lead to faster calculation of coefficients.

        '''

        def __init__(self,EstimatorClass,estimator_params,estimator_type,use_shap, *args, **kwargs):
            self.EstimatorClass = EstimatorClass
            self.estimator_params = estimator_params
            self.estimator = EstimatorClass(**estimator_params)

            self.estimator_type = estimator_type
            self.use_shap = use_shap
            self.shap_coef_ = None
            self.explainer = None
            self.X = None

        def fit(self, *args,**kwargs):
            self.estimator.fit(*args,**kwargs)
            self.X = args[0]

        def predict(self, *args, **kwargs):
            return self.estimator.predict(*args,**kwargs)

        def predict_proba(self, *args, **kwargs):
            return self.estimator.predict_proba(*args,**kwargs)

        def get_params(self,*args,**kwargs):
            return {
                'EstimatorClass':self.EstimatorClass,
                'estimator_params':self.estimator_params,
                'estimator_type':self.estimator_type,
                'use_shap': self.use_shap
                }

        def _get_tags(self,*args,**kwargs):
            return self.estimator._get_tags();

        @property
        def _estimator_type(self):
            return self.estimator._estimator_type;

        @property
        def coef_(self):
            if self.use_shap:
                return self.get_coef_with_shap(self.X)
            else:
                return self.estimator.coef_

        def get_coef_with_shap(self, X):

            if self.shap_coef_ is None and self.explainer is None:
                n_samples = X.shape[0] if X.shape[0] < 100 else 100;

                if self.estimator_type == 'tree':
                    self.explainer = shap.TreeExplainer(self.estimator,X,n_samples = n_samples)[0]
                else:
                    print('Initializing KernelExplainer, this might take a while... \n')
                    self.explainer = shap.KernelExplainer(self.estimator.predict_proba, X, n_samples = n_samples , link="logit")
                # check_additivity = False should be used with caution!!!
                # Check why shap_values from TreeExplainer are coming inside a list
                self.shap_coef_ = np.abs(self.explainer.shap_values(X,check_additivity=False)).sum(axis=0)
            return self.shap_coef_


text:
# Implement code for random projections here!
import numpy as np
from scipy import sparse
def random_proj(X,k):
    d=X.shape[1]
    R=np.zeros((d,k))
    s=3
    for i in range(0,d):
        for j in range(0,k):
            r=np.random.rand()
            if (r <= 1/(2*s)):
                R[i,j]=np.sqrt(s)/np.sqrt(k)
            elif (r > 1/(2*s)  and r<=1/s):
                R[i,j]=-np.sqrt(s)/np.sqrt(k)
    print(X.shape,R.shape)
    X_red=sparse.csr_matrix(X)*(sparse.csr_matrix(R)).todense()
    return X_red

max_stars_repo_name: amalroy/tipr-first-assignment

max_issues_repo_name: amalroy/tipr-first-assignment

max_forks_repo_name: amalroy/tipr-first-assignment

reason: import numpy,from scipy


text:
import os
import numpy as np
from tqdm import tqdm

from .datasetloader import DatasetLoader

class UCFSports(DatasetLoader):
    """
    UCF Sports Action Dataset
    https://www.crcv.ucf.edu/data/UCF_Sports_Action.php
    """

    classes = [
        "Diving", "Golf-Swing", "Kicking", "Lifting", "Riding-Horse", "Run",
        "SkateBoarding", "Swing-Bench", "Swing-Side", "Walk"
    ]
    splits = None

    def __init__(self, base_dir):
        """
        Parameters
        ----------
        base_dir : string
            folder with dataset on disk
        """
        self._data_cols = [
            "video-filename", "image-filenames", "bboxes", "action",
            "viewpoint"
        ]
        self._data = {
            "video-filename": [],
            "image-filenames": [],
            "bboxes": [],
            "action": [],
            "viewpoint": []
        }
        # Leave-One-Out cross validation recommended for action recognition
        # Add Action localisation split here for completeness?
        self._splits = None

        super().__init__(lazy_loading=False)

        self._length = 0
        viewpoints = ("", "-Front", "-Side", "-Back", "Angle")
        for cls_id, cls in tqdm(enumerate(UCFSports.classes)):
            for vp in viewpoints:
                cls_folder = os.path.join(base_dir, "ucf action", cls + vp)
                if os.path.exists(cls_folder):
                    video_id = "001"
                    while os.path.exists(os.path.join(cls_folder, video_id)):
                        self._length += 1
                        cur_id = self._length - 1
                        # set filename to blank here as in a few instances it
                        # doesn't exist and should be set to blank in those
                        # cases (this can be fixed with a script from this
                        # package)
                        self._data["video-filename"].append("")
                        self._data["action"].append(cls_id)
                        self._data["image-filenames"].append([])
                        self._data["bboxes"].append([])
                        if len(vp) > 0 and vp[0] == "-":
                            self._data["viewpoint"].append(vp)
                        else:
                            self._data["viewpoint"].append("")
                        filelist = sorted(
                            os.listdir(os.path.join(cls_folder, video_id)))
                        for filename in filelist:
                            if filename.endswith(".avi"):
                                self._data["video-filename"][
                                    cur_id] = os.path.join(
                                        cls_folder, video_id, filename)
                            elif filename.endswith(".jpg"):
                                self._data["image-filenames"][cur_id].append(
                                    os.path.join(cls_folder, video_id,
                                                 filename))
                            elif filename == "gt" or filename == "gt2":
                                if (len(self._data["bboxes"][cur_id]) <
                                        len(filename) - 1):
                                    # gt is read before gt2 so string len
                                    # corresponds to instances read
                                    self._data["bboxes"][self._length -
                                                         1].append([])
                                gt_folder = os.path.join(
                                    cls_folder, video_id, "gt")
                                gt_files = sorted(os.listdir(gt_folder))
                                for gt_file in gt_files:
                                    if gt_file.endswith(".txt"):
                                        with open(
                                                os.path.join(
                                                    gt_folder, gt_file),
                                                "r") as f:
                                            data = f.read()
                                            data = data.split("\t")
                                            self._data["bboxes"][cur_id][
                                                len(filename) - 2].append(
                                                    data[0:4])

                        self._data["bboxes"][cur_id] = np.array(
                            self._data["bboxes"][cur_id])
                        self._data["image-filenames"][cur_id].sort()
                        video_id = int(video_id) + 1
                        video_id = "0" * (3 -
                                          len(str(video_id))) + str(video_id)

        for key in self._data.keys():
            self._data[key] = np.array(self._data[key], dtype=object)

max_stars_repo_name: WeixinYang/DatasetLoader

max_issues_repo_name: WeixinYang/DatasetLoader

max_forks_repo_name: WeixinYang/DatasetLoader


text:
import pandas as pd
import numpy as np

def regression_merge(df_kappas, df_firm):
    firm_cols = ['permno', 'quarter', 'saleq', 'cogsq', 'normalized_l2',
                 'retail_share', 'market_cap', 'beta_BlackRock', 'beta_Vanguard', 'beta_StateStreet']
    keep_cols = ['from', 'to', 'quarter', 'kappa', 'cosine', 'retail_share', 'market_cap', 'marginsq', 'saleq', 'cogsq', 'normalized_l2',
                 'big3', 'beta_BlackRock', 'beta_Vanguard', 'beta_StateStreet']

    # Read things in and Merge
    df = pd.merge(
        df_kappas.loc[(df_kappas['from'] != df_kappas['to']) & (
            df_kappas['quarter'] <= '2017-10-01'), ['from', 'to', 'kappa', 'quarter', 'cosine']],
        df_firm[firm_cols], left_on=['from', 'quarter'], right_on=['permno', 'quarter'], how='left'
    ).reset_index(drop=True)

    # Calculate derived columns
    df['big3'] = df['beta_BlackRock'] + \
        df['beta_Vanguard'] + df['beta_StateStreet']
    df['marginsq'] = (df['saleq'] - df['cogsq']) / df['saleq']
    return df[keep_cols]

# merge it all together

def firm_info_merge(df_names2, df_fund2, firm_similarity, big4, chhi):
    df_firm2 = pd.merge(pd.merge(pd.merge(pd.merge(
        df_names2, df_fund2, on=['permno', 'quarter'], how='inner'),
        firm_similarity, on=['permno', 'quarter'], how='left'),
        big4, on=['permno', 'quarter'], how='left'),
        chhi, on=['permno', 'quarter'], how='left'
    )
    df_firm2['market_cap'] = df_firm2['shares_outstanding'] * df_firm2['price']
    df_firm2[['beta_BlackRock',
              'beta_Vanguard',
              'beta_StateStreet',
              'beta_Fidelity']] = df_firm2[['beta_BlackRock',
                                            'beta_Vanguard',
                                            'beta_StateStreet',
                                            'beta_Fidelity']].fillna(0)
    return df_firm2[(df_firm2.quarter >= '1980-01-01') &
                    (df_firm2.quarter <= '2017-10-01')].drop_duplicates()

# This block is for incoming and outgoing kappa
# note: not sure this made it into the paper (keep the code anyway)
def weighted_from(df):
    a1 = np.ma.average(df['kappa'].values, weights=df['saleq_x'].values)
    return pd.Series({'kappa_in': a1})

def weighted_to(df):
    a1 = np.ma.average(df['kappa'].values, weights=df['saleq_y'].values)
    return pd.Series({'kappa_out': a1})

def kappa_in_out(df, df_firm):
    dfk = df.loc[df['from'] != df['to'], ['from', 'to', 'quarter', 'kappa']]
    tmp = pd.merge(pd.merge(dfk,
                            df_firm[['permno', 'quarter', 'saleq']], left_on=['from', 'quarter'], right_on=['permno', 'quarter']),
                   df_firm[['permno', 'quarter', 'saleq']], left_on=['to', 'quarter'], right_on=['permno', 'quarter']
                   ).fillna(0)

    g1 = tmp.groupby(['quarter', 'to']).apply(weighted_from)
    g2 = tmp.groupby(['quarter', 'from']).apply(weighted_to)

    return pd.merge(pd.merge(df_firm,
                             g1, left_on=['quarter', 'permno'], right_on=['quarter', 'to'], how='left'),
                    g2, left_on=['quarter', 'permno'], right_on=['quarter', 'from'], how='left')

max_stars_repo_name: msinkinson/CommonOwnerReplication

max_issues_repo_name: msinkinson/CommonOwnerReplication

max_forks_repo_name: msinkinson/CommonOwnerReplication


text:
"""Module responsibel for plotting statistics."""
import io
import math

import matplotlib
import numpy as np
import pandas
from matplotlib import dates as mdates
from matplotlib import pyplot as plt  # noqa
from sqlalchemy import Date, Integer, cast, func

from pollbot.models import DailyStatistic, User, Vote

matplotlib.use("Agg")

def send_plots(session, chat):
    """Generate and send plots to the user."""
    image = get_user_activity(session)
    chat.send_document(image, caption="User statistics")
    image.close()

    image = get_vote_activity(session)
    chat.send_document(image, caption="Vote statistics")
    image.close()

    image = get_new_activity(session)
    chat.send_document(image, caption="New statistics")
    image.close()

def get_magnitude(value):
    if value == 0:
        return 0
    return int(math.floor(math.log10(abs(value))))

def image_from_figure(fig):
    """Create a pillow image from a figure."""
    io_buffer = io.BytesIO()
    plt.savefig(io_buffer, format="png")
    io_buffer.seek(0)
    plt.close(fig)

    return io_buffer

def get_user_activity(session):
    """Create a plot showing the inline usage statistics."""

    def running_window(session, subquery):
        # Create a running window which sums all users up to this point for the current millennium ;P
        users = (
            session.query(
                subquery.c.creation_date,
                cast(
                    func.sum(subquery.c.count).over(
                        partition_by=func.extract(
                            "millennium", subquery.c.creation_date
                        ),
                        order_by=subquery.c.creation_date.asc(),
                    ),
                    Integer,
                ).label("running_total"),
            )
            .order_by(subquery.c.creation_date)
            .all()
        )

        return users

    # Grid style
    plt.style.use("seaborn-whitegrid")

    creation_date = func.cast(User.created_at, Date).label("creation_date")
    # Group the started users by date
    started_users_subquery = (
        session.query(creation_date, func.count(User.id).label("count"))
        .filter(User.started.is_(True))
        .group_by(creation_date)
        .order_by(creation_date)
        .subquery()
    )
    started_users = running_window(session, started_users_subquery)
    started_users = [("started", q[0], q[1]) for q in started_users]

    # Group the started users by date
    all_users_subquery = (
        session.query(creation_date, func.count(User.id).label("count"))
        .group_by(creation_date)
        .order_by(creation_date)
        .subquery()
    )
    all_users = running_window(session, all_users_subquery)
    all_users = [("all", q[0], q[1]) for q in all_users]

    # Group the started users by date
    voting_users_subquery = (
        session.query(creation_date, func.count(User.id).label("count"))
        .filter(User.votes.any())
        .group_by(creation_date)
        .order_by(creation_date)
        .subquery()
    )
    voting_users = running_window(session, voting_users_subquery)
    voting_users = [("voted", q[0], q[1]) for q in voting_users]

    # Group the started users by date
    owning_users_subquery = (
        session.query(creation_date, func.count(User.id).label("count"))
        .filter(User.polls.any())
        .group_by(creation_date)
        .order_by(creation_date)
        .subquery()
    )
    owning_users = running_window(session, owning_users_subquery)
    owning_users = [("currently owning poll", q[0], q[1]) for q in owning_users]

    # Combine the results in a single dataframe and name the columns
    user_statistics = started_users + all_users + voting_users + owning_users
    dataframe = pandas.DataFrame(user_statistics, columns=["type", "date", "users"])

    months = mdates.MonthLocator()  # every month
    months_fmt = mdates.DateFormatter("%Y-%m")

    max_value = all_users[len(all_users) - 1][2]
    magnitude = get_magnitude(max_value)

    # Plot each result set
    fig, ax = plt.subplots(figsize=(30, 15), dpi=120)
    for key, group in dataframe.groupby(["type"]):
        ax = group.plot(ax=ax, kind="line", x="date", y="users", label=key)
        ax.xaxis.set_major_locator(months)
        ax.xaxis.set_major_formatter(months_fmt)
        ax.yaxis.set_ticks(np.arange(0, max_value, math.pow(10, magnitude - 1)))

    image = image_from_figure(fig)
    image.name = "user_statistics.png"

    return image

def get_vote_activity(session):
    """Create a plot showing the inline usage statistics."""
    creation_date = func.date_trunc("day", Vote.created_at).label("creation_date")
    votes = (
        session.query(creation_date, func.count(Vote.id).label("count"))
        .group_by(creation_date)
        .order_by(creation_date)
        .all()
    )
    total_votes = [("Total votes", q[0], q[1]) for q in votes]

    # Grid style
    plt.style.use("seaborn-whitegrid")

    # Combine the results in a single dataframe and name the columns
    dataframe = pandas.DataFrame(total_votes, columns=["type", "date", "votes"])

    months = mdates.MonthLocator()  # every month
    months_fmt = mdates.DateFormatter("%Y-%m")

    max_value = max([vote[2] for vote in total_votes])
    magnitude = get_magnitude(max_value)

    # Plot each result set
    fig, ax = plt.subplots(figsize=(30, 15), dpi=120)
    for key, group in dataframe.groupby(["type"]):
        ax = group.plot(ax=ax, kind="bar", x="date", y="votes", label=key)
        ax.xaxis.set_major_locator(months)
        ax.xaxis.set_major_formatter(months_fmt)
        ax.yaxis.set_ticks(np.arange(0, max_value, math.pow(10, magnitude - 1)))

    image = image_from_figure(fig)
    image.name = "vote_statistics.png"
    return image

def get_new_activity(session):
    """Create a plot showing new users and polls per day."""
    creation_date = func.date_trunc("week", DailyStatistic.date).label("creation_date")
    stats = (
        session.query(
            creation_date,
            func.avg(DailyStatistic.created_polls).label("new_polls"),
            func.avg(DailyStatistic.new_users).label("new_users"),
        )
        .group_by(creation_date)
        .order_by(creation_date)
        .all()
    )
    new_polls = [("New polls", stat[0], float(stat[1])) for stat in stats]
    new_users = [("New users", stat[0], float(stat[2])) for stat in stats]

    # Grid style
    plt.style.use("seaborn-whitegrid")

    combined = new_polls + new_users
    # Combine the results in a single dataframe and name the columns
    dataframe = pandas.DataFrame(combined, columns=["type", "date", "count"])

    months = mdates.MonthLocator()  # every month
    months_fmt = mdates.DateFormatter("%Y-%m")

    max_value = max([vote[2] for vote in combined])
    magnitude = get_magnitude(max_value)

    # Plot each result set
    fig, ax = plt.subplots(figsize=(30, 15), dpi=120)
    for key, group in dataframe.groupby(["type"]):
        ax = group.plot(ax=ax, kind="line", x="date", y="count", label=key)
        ax.xaxis.set_major_locator(months)
        ax.xaxis.set_major_formatter(months_fmt)
        ax.yaxis.set_ticks(np.arange(0, max_value, math.pow(10, magnitude - 1)))

    image = image_from_figure(fig)
    image.name = "new_statistics.png"
    return image

max_stars_repo_name: shubham-king/poll

max_issues_repo_name: shubham-king/poll

max_forks_repo_name: shubham-king/poll


text:
"""
Track 2011 histogram changes as moving-average cadence is changed
"""

import numpy as np
import pickle

import matplotlib.pyplot as plt
from matplotlib import cm

import chmap.data.corrections.lbcc.lbcc_utils as lbcc

image_out_path = "/Users/turtle/Dropbox/MyNACD/analysis/move-avg_cadence/"
data_path = "/Users/turtle/GitReps/CHD/test_data/"

optim_vals = ["Beta", "y", "SSE"]

v_cmap = cm.get_cmap('viridis')

# try evaluating the moving average at 2 hour cadence. Then 4, 6, ..., 24 hours; then 1.5, 2, 3, 4, 5, 6, 7 days
skip_steps = list(range(1, 13)) + [18, 24, 36, 48, 60, 72, 84]
hour_cadence = [x * 2 for x in skip_steps]

for instrument in ['AIA', "EUVI-A", "EUVI-B"]:
    print("Calculating LBCC fit parameters for " + instrument + " 2011 as the moving-average cadence varies.")
    # use only the 400 bin files
    hists_400_fname = data_path + "mu-hists-2011_400_" + instrument + ".pkl"

    # start with the 400 bin file
    f = open(hists_400_fname, 'rb')
    hists_400 = pickle.load(f)
    f.close()

    # extract histograms
    full_year_hists = hists_400['all_hists']
    # extract intensity bins
    image_intensity_bin_edges = hists_400['intensity_bin_edges']
    intensity_centers = (image_intensity_bin_edges[:-1] + image_intensity_bin_edges[1:])/2
    # extract mu bins
    mu_bin_edges = hists_400['mu_bin_edges']
    mu_bin_centers = (mu_bin_edges[1:] + mu_bin_edges[:-1])/2
    # clean-up
    del hists_400

    results = np.zeros((len(mu_bin_centers) - 1, len(skip_steps), len(optim_vals)))

    for skip_index, skip in enumerate(skip_steps):
        # using every iith image, sum over the year.
        # Note: this is not perfect. Some images are missing, so we are not getting exactly 2, 4, 6, ... hour steps
        full_year = full_year_hists[:, :, ::skip].sum(axis=2)
        # normalize in mu
        norm_hist = np.full(full_year.shape, 0.)
        row_sums = full_year.sum(axis=1, keepdims=True)
        # but do not divide by zero
        zero_row_index = np.where(row_sums != 0)
        norm_hist[zero_row_index[0]] = full_year[zero_row_index[0]]/row_sums[zero_row_index[0]]

        # determine peak bin and value for reference histogram (mu=1.)
        hist_ref = norm_hist[-1, ]
        ref_peak_index = np.argmax(hist_ref)
        ref_peak_val = hist_ref[ref_peak_index]
        # fit coefficients for each mu_bin
        for ii in range(mu_bin_centers.__len__() - 1):
            hist_fit = norm_hist[ii, ]

            # estimate correction coefs that match fit_peak to ref_peak
            fit_peak_index = np.argmax(hist_fit)
            fit_peak_val = hist_fit[fit_peak_index]
            beta_est = fit_peak_val/ref_peak_val
            y_est = image_intensity_bin_edges[ref_peak_index] - beta_est*image_intensity_bin_edges[fit_peak_index]
            init_pars = np.asarray([beta_est, y_est])

            # optimize correction coefs
            optim_result = lbcc.optim_lbcc_linear(hist_ref, hist_fit, image_intensity_bin_edges, init_pars)
            # record results
            results[ii, skip_index, 0] = optim_result.x[0]
            results[ii, skip_index, 1] = optim_result.x[1]
            results[ii, skip_index, 2] = optim_result.fun

    # save intsrument specific results
    save_object = {'mu_bins': mu_bin_edges, 'hour_cadence': hour_cadence, 'pars': optim_vals, 'result_array': results}
    file_path = image_out_path + 'lbcc-vals_2011_' + instrument + '.pkl'
    print('\nSaving LBCC coefficients to ' + file_path + '\n')
    f = open(file_path, 'wb')
    pickle.dump(save_object, f)
    f.close()

    # add plotting routines here
    n_mu = len(mu_bin_centers)
    color_dist = np.linspace(0., 1., n_mu - 1)

    par_list = ["Beta", "y"]

    for par_name in par_list:
        plt.figure(0)
        par_index = optim_vals.index(par_name)

        # first plot lbcc coeffs v image cadence
        # add lines/markers
        for ii in range(n_mu - 1):
            plt.plot(hour_cadence, results[ii, :, par_index], c=v_cmap(color_dist[ii]), marker='x',
                     label=str(mu_bin_centers[ii].round(decimals=3)))

        plt.xscale('log')
        plt.ylabel(par_name + "(mu)")
        plt.xlabel("Moving AVG Image Interval (hours)")
        plt.title("2011 " + instrument + " Image Interval Analysis")
        ax = plt.gca()
        ax.legend(loc='upper left', bbox_to_anchor=(1., 0., 0.2, 1.0), title="mu bin")
        # adjust margin to incorporate legend
        plt.subplots_adjust(right=0.8)
        plt.grid()

        plot_fname = image_out_path + 'lbcc-' + par_name + '_v_mu_' + instrument + '.pdf'
        plt.savefig(plot_fname)
        plt.close(0)

        # now plot 'convergence rate' of coeffs
        plt.figure(0)
        par_index = optim_vals.index(par_name)
        for ii in range(n_mu - 1):
            par_change = np.divide(np.abs(np.diff(results[ii, :, par_index])), np.divide(hour_cadence[:-1],
                                                                                hour_cadence[1:]))
            plt.plot(hour_cadence[0:-1], par_change, c=v_cmap(color_dist[ii]), marker='x',
                     label=str(mu_bin_centers[ii].round(decimals=3)))

        plt.yscale('log')
        plt.xscale('log')
        plt.ylabel("Relative Error in " + par_name + "(mu)")
        plt.xlabel("Moving AVG Image Interval (hours)")
        plt.title("2011 " + instrument + " Image Interval Analysis")
        ax = plt.gca()
        ax.legend(loc='upper left', bbox_to_anchor=(1., 0., 0.2, 1.0), title="mu bin")
        # adjust margin to incorporate legend
        plt.subplots_adjust(right=0.8)
        plt.grid()

        plot_fname = image_out_path + 'lbcc-rel-error' + par_name + '_v_mu_' + instrument + '.pdf'
        plt.savefig(plot_fname)
        plt.close(0)

max_stars_repo_name: predsci/CHD

max_issues_repo_name: predsci/CHD

max_forks_repo_name: predsci/CHD


text:
import numpy as numpy

import ArScreens
from rascil.data_models.polarisation import PolarisationFrame
from rascil.processing_components.image.operations import export_image_to_fits, create_image_from_array

if __name__ == '__main__':
    r0 = 5000.0
    hiono = 3e5
    bmax = 20000.0
    sampling = 100.0

    bigD = 200000.0
    m = int(bmax / sampling)
    n = int(bigD / bmax)
    npixel = n * m
    print("Number of pixels %d" % (npixel))
    pscale = bigD / (n * m)
    print("Pixel size %.3f (m)" % pscale)
    print("Field of view %.1f (m)" % (npixel * pscale))
    speed = 150000.0 / 3600.0
    direction = 90.0
    print("Ionospheric velocity %.3f (m/s) at direction %.1f (degrees)" % (speed, direction))
    rate = 1.0/10.0
    alpha_mag = 0.9999
    ntimes = 61
    lowparamcube = numpy.array([(r0, speed, direction, hiono)])
    filename = 'low_screen_%.1fr0_%.3frate.fits' % (r0, rate)

    my_screens = ArScreens.ArScreens(n, m, pscale, rate, lowparamcube, alpha_mag)
    my_screens.run(ntimes, verbose=True)

    from astropy.wcs import WCS

    nfreqwin = 1
    npol = 1
    frequency = [1e8]
    channel_bandwidth = [0.1e8]

    w = WCS(naxis=4)
    cellsize = pscale
    w.wcs.cdelt = [cellsize, cellsize, 1.0 / rate, channel_bandwidth[0]]
    w.wcs.crpix = [npixel // 2 + 1, npixel // 2 + 1, ntimes // 2 + 1, 1.0]
    w.wcs.ctype = ['XX', 'YY', 'TIME', 'FREQ']
    w.wcs.crval = [0.0, 0.0, 0.0, frequency[0]]
    w.naxis = 4
    w.wcs.radesys = 'ICRS'
    w.wcs.equinox = 2000.0
    data = numpy.zeros([nfreqwin, ntimes, npixel, npixel])
    for i, screen in enumerate(my_screens.screens[0]):
        data[:, i, ...] = screen[numpy.newaxis, ...]

    im = create_image_from_array(data, wcs=w, polarisation_frame=PolarisationFrame("stokesI"))
    print(im)
    export_image_to_fits(im, filename)

max_stars_repo_name: ska-telescope/sim-low-imaging

max_issues_repo_name: ska-telescope/sim-low-imaging

max_forks_repo_name: ska-telescope/sim-low-imaging

reason: import numpy,from astropy


text:
import matplotlib.pyplot as plt
import numpy as np
mean = np.loadtxt("fort.10")
plt.title('Mean flow profile')
plt.ylabel(r"$U(y)$, $U''(y)$")
plt.xlabel(r"$y$")
plt.tick_params(axis='x',direction='in')
plt.tick_params(axis='y',direction='in')
plt.xticks(np.arange(0,18,2))
plt.yticks(np.arange(-0.2,1.2,.2))
plt.plot(mean[:,0],mean[:,1],'b-',label=r"$U(y)$")
plt.plot(mean[:,0],mean[:,2],'r-',label=r"$U''(y)$")
plt.legend()
plt.savefig("mean.png")
plt.show(block=False)

max_stars_repo_name: sscollis/os-stab

max_issues_repo_name: sscollis/os-stab

max_forks_repo_name: sscollis/os-stab