#2 Loading Data in pandas
# Selecting columns

# Select the column item from credit_records
# Use dot notation
items = ____

# Display the results
print(items)

# Select the column item from credit_records
# Use dot notation
items = credit_records.item

# Display the results
print(items)
# One or more lines of code contain errors.
# Fix the errors so that the code runs.

# Select the location column in credit_records
location = credit_records['location']

# Select the item column in credit_records
items = credit_records.item

# Display results
print(location)

# Use info() to inspect mpr
print(mpr.info())

# Use info() to inspect mpr
print(mpr.info())

# The following code contains one or more errors
# Correct the mistakes in the code so that it runs without errors

# Select column "Dog Name" from mpr
name = mpr['Dog Name']

# Select column "Missing?" from mpr
is_missing = mpr['Missing?']

# Display the columns
print(name)
print(is_missing)

# Selecting Rows with Logic
# Is height_inches greater than 70 inches?
print(height_inches > 70)

# Is plate1 equal to "FRQ123"?
print(plate1 == "FRQ123")

# Is fur_color not equal to "brown"?
print(fur_color != "brown")

# Select the dogs where Age is greater than 2
greater_than_2 = mpr[mpr.Age > 2]
print(greater_than_2)

# Select the dogs whose Status is equal to Still Missing
still_missing = mpr[mpr.Status == 'Still Missing']
print(still_missing)

# Select all dogs whose Dog Breed is not equal to Poodle
not_poodle = mpr[mpr['Dog Breed'] != 'Poodle']
print(not_poodle)

# Select purchases from 'Pet Paradise'
purchase = credit_records[credit_records.location == 'Pet Paradise']

# Display
print(purchase)

# Plotting w/matplotlib
# From matplotlib, import pyplot under the alias plt
from matplotlib import pyplot as plt

# Plot Officer Deshaun's hours_worked vs. day_of_week
plt.plot(deshaun.day_of_week, deshaun.hours_worked)

# Display Deshaun's plot
plt.show()

# Plot Officer Deshaun's hours_worked vs. day_of_week
plt.plot(deshaun.day_of_week, deshaun.hours_worked)

# Plot Officer Aditya's hours_worked vs. day_of_week
plt.plot(aditya.day_of_week, aditya.hours_worked)

# Plot Officer Mengfei's hours_worked vs. day_of_week
plt.plot(mengfei.day_of_week,mengfei.hours_worked)

# Display all three line plots
plt.show()

# Officer Deshaun
plt.plot(deshaun.day_of_week, deshaun.hours_worked, label='Deshaun')

# Add a label to Aditya's plot
plt.plot(aditya.day_of_week, aditya.hours_worked,label='Aditya')

# Add a label to Mengfei's plot
plt.plot(mengfei.day_of_week, mengfei.hours_worked,label='Mengfei')

# Add a command to make the legend display
plt.legend()

# Display plot
plt.show()

# Lines
plt.plot(deshaun.day_of_week, deshaun.hours_worked, label='Deshaun')
plt.plot(aditya.day_of_week, aditya.hours_worked, label='Aditya')
plt.plot(mengfei.day_of_week, mengfei.hours_worked, label='Mengfei')

# Add a title
plt.title('Kidnap doggo')

# Add y-axis label
plt.ylabel('Hours Worked')

# Legend
plt.legend()
# Display plot
plt.show()

# Create plot
plt.plot(six_months.month, six_months.hours_worked)

# Add annotation "Missing June data" at (2.5, 80)
plt.text(2.5,80,"Missing June data")

# Display graph
plt.show()

# Change the color of Phoenix to `"DarkCyan"`
plt.plot(data["Year"], data["Phoenix Police Dept"], label="Phoenix", color="DarkCyan")

# Make the Los Angeles line dotted
plt.plot(data["Year"], data["Los Angeles Police Dept"], label="Los Angeles", linestyle=":")

# Add square markers to Philedelphia
plt.plot(data["Year"], data["Philadelphia Police Dept"], label="Philadelphia", marker="s")

# Add a legend
plt.legend()

# Display the plot
plt.show()

# Change the style to fivethirtyeight
plt.style.use('fivethirtyeight')

# Plot lines
plt.plot(data["Year"], data["Phoenix Police Dept"], label="Phoenix")
plt.plot(data["Year"], data["Los Angeles Police Dept"], label="Los Angeles")
plt.plot(data["Year"], data["Philadelphia Police Dept"], label="Philadelphia")

# Add a legend
plt.legend()

# Display the plot
plt.show()

# Change the style to ggplot
plt.style.use('ggplot')

# Plot lines
plt.plot(data["Year"], data["Phoenix Police Dept"], label="Phoenix")
plt.plot(data["Year"], data["Los Angeles Police Dept"], label="Los Angeles")
plt.plot(data["Year"], data["Philadelphia Police Dept"], label="Philadelphia")

# Add a legend
plt.legend()

# Display the plot
plt.show()

#dispay all style types
print(plt.style.available)

# Choose any of the styles
plt.style.use('seaborn-darkgrid')

# Plot lines
plt.plot(data["Year"], data["Phoenix Police Dept"], label="Phoenix")
plt.plot(data["Year"], data["Los Angeles Police Dept"], label="Los Angeles")
plt.plot(data["Year"], data["Philadelphia Police Dept"], label="Philadelphia")

# Add a legend
plt.legend()

# Display the plot
plt.show()

# x should be ransom.letter and y should be ransom.frequency
plt.plot(ransom.letter, ransom.frequency,
         # Label should be "Ransom"
         label="Ransom",
         # Plot the ransom letter as a dotted gray line
         linestyle=':', color='gray')

# Display the plot
plt.show()

# Plot each line
plt.plot(ransom.letter, ransom.frequency,
         label='Ransom', linestyle=':', color='gray')

# X-values should be suspect1.letter
# Y-values should be suspect1.frequency
# Label should be "Fred Frequentist"
plt.plot(suspect1.letter, suspect1.frequency, label='Fred Frequentist')

# Display the plot
plt.show()

# Plot each line
plt.plot(ransom.letter, ransom.frequency,
         label='Ransom', linestyle=':', color='gray')
plt.plot(suspect1.letter, suspect1.frequency, label='Fred Frequentist')
plt.plot(suspect2.letter, suspect2.frequency, label='Gertrude Cox')

# Add x- and y-labels
plt.xlabel("Letter")
plt.ylabel("Frequency")

# Add a legend
plt.legend()

# Display plot
plt.show()

# Different Types of plots

# Explore the data
print(cellphone.head())

# Create a scatter plot of the data from the DataFrame cellphone
plt.scatter(cellphone.x, cellphone.y)

# Add labels
plt.ylabel('Latitude')
plt.xlabel('Longitude')

# Display the plot
plt.show()

# Change the transparency to 0.1
plt.scatter(cellphone.x, cellphone.y,
           color='red',
           marker='s',
           alpha=0.1)

# Add labels
plt.ylabel('Latitude')
plt.xlabel('Longitude')

# Display the plot
plt.show()

# Display the DataFrame hours using print
print(hours)

# Create a bar plot from the DataFrame hours
plt.bar(hours.officer, hours.avg_hours_worked,
        # Add error bars
        yerr=hours.std_hours_worked)

# Display the plot
plt.show()

Introduction to Data Science in Python.py


from pathlib import Path

p = Path(__file__).with_name('d2.txt')

MAXES = {
    'red': 12,
    'green': 13,
    'blue': 14
}

def validate_game(game: str) -> int:
    gametag, draws = game.split(': ')
    _, game_id = gametag.split()
    for draw in draws.split(';'):
        if not possible(draw):
            return 0
    return int(game_id)

def possible(draw: str) -> bool:
    for color_count in draw.split(','):
        count, color = color_count.split()
        if int(count) > MAXES[color]:
            return False
    return True

lines = p.open('r').readlines()
good_games = (validate_game(line) for line in lines)
print(sum(good_games))


from django.conf.urls import url

from itkufs.accounting.views.edit import (
    new_edit_settlement,
    new_edit_transaction,
    approve_transactions,
    reject_transactions,
    transfer,
)
from itkufs.accounting.views.display import (
    SettlementDetails,
    SettlementList,
    TransactionDetails,
    TransactionList,
    APIAccountDetails,
)

urlpatterns = [
    # --- Settlements
    url(
        r"^(?P<group>[0-9a-z_-]+)/settlement/(?P<settlement>\d+)/$",
        SettlementDetails.as_view(),
        name="settlement-details",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/new-settlement/$",
        new_edit_settlement,
        name="new-settlement",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/settlement/(?P<settlement>\d+)/edit/$",
        new_edit_settlement,
        name="edit-settlement",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/settlement/$",
        SettlementList.as_view(),
        name="settlement-list",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/settlement/p(?P<page>\d+)/$",
        SettlementList.as_view(),
        name="settlement-list-page",
    ),
    # --- Transactions
    url(
        r"^(?P<group>[0-9a-z_-]+)/transaction/(?P<transaction>\d+)/$",
        TransactionDetails.as_view(),
        name="transaction-details",
    ),
    # Admin transaction actions
    url(
        r"^(?P<group>[0-9a-z_-]+)/transaction/new/$",
        new_edit_transaction,
        name="new-transaction",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/transaction/(?P<transaction>\d+)/edit/$",
        new_edit_transaction,
        name="edit-transaction",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/approve-transaction/$",
        approve_transactions,
        name="approve-transactions",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/reject-transaction/$",
        reject_transactions,
        name="reject-transactions",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/transaction/(?P<transaction>\d+)/reject/$",
        reject_transactions,
        name="reject-transaction",
    ),
    # User transaction actions
    url(
        r"^(?P<group>[0-9a-z_-]+)/account/(?P<account>[0-9a-z_-]+)/deposit/$",
        transfer,
        {"transfer_type": "deposit"},
        name="account-deposit",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/account/(?P<account>[0-9a-z_-]+)/withdraw/$",
        transfer,
        {"transfer_type": "withdraw"},
        name="account-withdraw",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/account/(?P<account>[0-9a-z_-]+)/transfer/$",
        transfer,
        {"transfer_type": "transfer"},
        name="account-transfer",
    ),
    # Group transaction lists
    url(
        r"^(?P<group>[0-9a-z_-]+)/transaction/$",
        TransactionList.as_view(),
        name="transaction-list-group",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/transaction/p(?P<page>\d+)/$",
        TransactionList.as_view(),
        name="transaction-list-group-page",
    ),
    # Account transaction lists
    url(
        r"^(?P<group>[0-9a-z_-]+)/account/"
        r"(?P<account>[0-9a-z_-]+)/transaction/$",
        TransactionList.as_view(),
        name="transaction-list-account",
    ),
    url(
        r"^(?P<group>[0-9a-z_-]+)/account/"
        r"(?P<account>[0-9a-z_-]+)/transaction/p(?P<page>\d+)/$",
        TransactionList.as_view(),
        name="transaction-list-account-page",
    ),
    url(r"^(?P<group>[0-9a-z_-]+)/api/accounts/$",
        APIAccountDetails.as_view(),
        name="account-list"
    ),
]


"""
This scripts contains the PayoffMatrix
"""
from typing import Optional
import numpy as np

class PayoffMatrix:
    """
    A class of 2 by 2 payoff matrices.

    Currently include the following games:
    The staghunt game: 'staghunt',
    The matching pennies game (coop and competive): 'penny_competive', 'penny_cooperative',
    The party dilemma: 'party',
    The Battle of the sexes: 'sexes',
    The chicken game: 'chicken',
    The deadlock: 'deadlock',
    The prisoners dilemma: 'prisoners_dilemma'.

    For custom payoff matrix supply a 2x2x2 numpy array to the predefined argument.

    Examples:
        >>> import tomsup as ts
        >>> staghunt = ts.PayoffMatrix(name="staghunt")
        >>> staghunt.payoff(action_agent0=1, action_agent1=1, agent=0)
        5
        >>> staghunt.payoff(action_agent0=1, action_agent1=0, agent=0)
        0
        >>> staghunt.payoff(action_agent0=0, action_agent1=1, agent=0)
        3
        >>> chicken = ts.PayoffMatrix(name="chicken")
        >>> chicken.payoff(0, 1, 0)
        -1
        >>> dead = ts.PayoffMatrix(name="deadlock")
        >>> dead.payoff(1, 0, 1)
        0
        >>> sexes = ts.PayoffMatrix(name="sexes")
        >>> sexes.payoff(1, 1, 0)
        5
        >>> custom = ts.PayoffMatrix(name="custom", np.array(([(10, 0), (0, 5)],
                                                        [(5, 0), (0, 10)])))
    """

    def __init__(self, name: str, predefined: Optional[np.array] = None):
        """
        Args:
            name (str): The name of the either predefined matrix or your custom matrix.
                Currently include the following games:
                The staghunt game ('staghunt'),
                the matching pennies game ('penny_competive', 'penny_cooperative'),
                the party dilemma ('party'),
                the Battle of the sexes ('sexes'), the chicken game ('chicken'),
                the deadlock ('deadlock'), nad the prisoners dilemma ('prisoners_dilemma').
            predefined (Optional[np.array], optional): A custom 2x2x2 matrix. Defaults to None.
        """
        self.name = name
        if name == "staghunt":
            self.matrix = np.array(
                (
                    [(3, 3), (0, 5)],
                    [(3, 0), (3, 5)],
                )
            )
        elif name == "prisoners_dilemma":
            self.matrix = np.array(([(1, 5), (0, 3)], [(1, 0), (5, 3)]))
        elif name == "penny_competitive":
            self.matrix = np.array(([(-1, 1), (1, -1)], [(1, -1), (-1, 1)]))
        elif name == "penny_cooperative":
            self.matrix = np.array(([(1, -1), (-1, 1)], [(1, -1), (-1, 1)]))
        elif name == "party":
            self.matrix = np.array(([(5, 0), (0, 10)], [(5, 0), (0, 10)]))
        elif name == "sexes":  # battle of the sexes
            self.matrix = np.array(([(10, 0), (0, 5)], [(5, 0), (0, 10)]))
        elif name == "chicken":
            self.matrix = np.array(([(-1000, -1), (1, 0)], [(-1000, 1), (-1, 0)]))
        elif name == "deadlock":
            self.matrix = np.array(([(1, 0), (3, 2)], [(1, 3), (0, 2)]))
        else:
            if predefined is not None:
                matrix = np.array(predefined)
                if matrix.shape == (2, 2, 2):
                    self.matrix = np.array(predefined)
                else:
                    raise TypeError(
                        "Predefined should be a valid matrix where \
                                     matrix.shape == (2, 2, 2), e.g. a 2x2x2 \
                                     matrix"
                    )
            else:
                raise TypeError(
                    "Invalid name and no predefined matrix given. \
                                 Please input a valid name or input a \
                                 predefined matrix of dimension 2x2x2."
                )

    def payoff(self, choice_agent0: int, choice_agent1: int, agent: int = 0) -> float:
        """
        Args:
            choice_agent0 (int): choice of agent 0
            choice_agent1 (int): choice of agent 1
            agent (int, optional): The perspective agent which should get the payoff, either 0 or 1.
                Defaults to 0.

        Returns:
            float: The payoff of the agent
        """

        return self.matrix[agent, choice_agent0, choice_agent1]

    def __str__(self):
        print_len = max([len(str(self().min())), len(str(self().max()))])

        def add_pl(string):
            return (print_len - len(str(string))) * " " + str(string)

        str1 = f"<Class PayoffMatrix, Name = {self.name}> "
        str2 = "The payoff matrix of agent 0"
        str3 = "       |  Choice agent 1"
        str4 = (
            "       | "
            + f"{add_pl(' ')}"
            + " | "
            + f"{add_pl(0)}"
            + " | "
            + f"{add_pl(1)}"
            + " |"
        )
        str5 = "       | " + print_len * 3 * "-" + 2 * "---" + " |"
        str6 = (
            "Choice | " + f"{add_pl(0)}" + " | " + f"{add_pl(self()[0][0,0])}" + " | "
            f"{add_pl(self()[0][0,1])}" + " |"
        )
        str7 = (
            "agent 0| " + f"{add_pl(1)}" + " | " + f"{add_pl(self()[0][1,0])}" + " | "
            f"{add_pl(self()[0][1,1])}" + " |"
        )
        str8 = " "
        str9 = "The payoff matrix of agent 1"
        str10 = "       |  Choice agent 1"
        str11 = (
            "       | "
            + f"{add_pl(' ')}"
            + " | "
            + f"{add_pl(0)}"
            + " | "
            + f"{add_pl(1)}"
            + " |"
        )
        str12 = str5
        str13 = (
            "Choice | " + f"{add_pl(0)}" + " | " + f"{add_pl(self()[1][0,0])}" + " | "
            f"{add_pl(self()[1][0,1])}" + " |"
        )
        str14 = (
            "agent 0| " + f"{add_pl(1)}" + " | " + f"{add_pl(self()[1][1,0])}" + " | "
            f"{add_pl(self()[1][1,1])}" + " |"
        )
        str15 = str8
        return "\n".join(
            [
                str1,
                str2,
                str3,
                str4,
                str5,
                str6,
                str7,
                str8,
                str9,
                str10,
                str11,
                str12,
                str13,
                str14,
                str15,
            ]
        )

    def get_matrix(self) -> np.array:
        """
        Returns:
            np.array: The payoff matrix
        """
        return self.matrix

    def __call__(self):
        return self.matrix


import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

def medianFilter(image):

    rows, columns = image.size
    m_image = np.array(image, dtype=np.uint8)
    for row in range(rows):
        for col in range(columns):
            if m_image[row, col] == np.uint8(0):
                window = np.squeeze(np.asarray(m_image[row - 1:row + 2, col - 1:col + 2]))
                m_image[row, col] = np.uint8(np.median(window))
    finalImage = Image.fromarray(m_image, 'L')

    return finalImage

def getNewPos(comp,angle,pix_x,pix_y,rot_x,rot_y): #Makes all the translation of the old components to the rotated components
    if (comp == 'x'):
        a0 = np.cos(angle)
        a1 = np.sin(angle)
        a2 = rot_x - a0 * rot_x - a1 * rot_y
        new_x = round(a0 * pix_x + a1 * pix_y + a2)
        return new_x
    else:
        b0 = -np.sin(angle)
        b1 = np.cos(angle)
        b2 = rot_y - b0 * rot_y - b1 * rot_y
        new_y = round(b0 * pix_x + b1 * pix_y + b2)
        return new_y

def rotacion(img, angle):

    imageM = np.array(img, dtype=np.uint8)
    max_rows, max_cols = img.size
    imageRot = np.zeros((max_rows, max_cols), dtype=np.uint8)
    centerX = max_rows // 2 #Get the x component center about which it will rotate
    centerY = max_cols // 2 #Get the y component center about which it will rotate

    for x in range(max_rows - 1):
        for y in range(max_cols - 1):

            x_new = getNewPos('x',angle,x,y,centerX,centerY)
            y_new = getNewPos('y', angle, x, y, centerX, centerY)

            if x_new >= 0 and y_new >= 0 : ##the transformation is not out of scope
                if x_new < max_rows and y_new < max_cols: #Checks that the image dont gets out of the matrix size
                    imageRot[x_new, y_new] = np.uint8(imageM[x, y])#Sets the components
    finalImage = Image.fromarray(imageRot, 'L')
    return finalImage

def draw():

    gf = plt.figure()
    img = Image.open('landscape.jpg', 'r').convert('L')
    angle = np.pi / 1.8
    rotatedImage = rotacion(img, angle)
    rotatedImage.save("rotationImage.jpg")
    filtertedImage = medianFilter(rotatedImage.convert('L'))
    gf.add_subplot(1, 3, 1)
    filtertedImage.save("medianFilterResult.jpg")

    plt.imshow(img.convert('L'), cmap='gray')
    plt.title("Original")
    gf.add_subplot(1, 3, 2)
    plt.imshow(rotatedImage.convert('L'), cmap='gray')

    plt.title("Rotated")
    gf.add_subplot(1, 3, 3)
    plt.imshow(filtertedImage.convert('L'), cmap='gray')
    plt.title("Filtered rotated")
    plt.show()

draw()


from django import forms

from app_jobs.models import City, Language

from .models import User

class ProfileChangeForm(forms.ModelForm):
    """Форма для личного кабинета"""

    username = forms.CharField(widget=forms.TextInput(attrs={
        'class': 'form-control'}), label='Имя пользователя')
    email = forms.EmailField(widget=forms.EmailInput(attrs={
        'class': 'form-control', 'readonly': True}), label='Email')
    city = forms.ModelChoiceField(empty_label='Изменить город',
                                  queryset=City.objects.all(),
                                  widget=forms.Select(attrs={'class': 'form-control'}),
                                  label='Город')
    language = forms.ModelChoiceField(empty_label='Изменить язык программирования',
                                      queryset=Language.objects.all(),
                                      widget=forms.Select(attrs={'class': 'form-control'}),
                                      label='Язык программирования')

    class Meta:
        model = User
        fields = ('username', 'email', 'city', 'language')


from typing import List
from typing import Optional
from typing import Text

class CardType:
	def __init__(self, num=None, special=None):
		self.num = num
		self.special = special
		assert self.num or self.special

	def number_value(self) -> Optional[int]:
		# The number value of this card.
		# Returns None if the card has no number value.
		return self.num

	def special_value(self) -> Optional[Text]:
		# The special value of this card.
		# Returns None if the card is a number card.
		# Value will be one of: "peek", "draw_two", "swap"
		return self.special

def sorted_deck_factory() -> List[CardType]:
	""" Create a deck of cards per game rules. """
	_cards = []
	for _ in range(4):
		for n in range (1,10):
			_cards.append(CardType(num=n))

	for _ in range(4):
		_cards.append(CardType(special='peek'))
		_cards.append(CardType(special='draw_two'))
		_cards.append(CardType(special='swap'))

	return _cards


from django.urls import path, include
from .views import *
urlpatterns = [
    path('',index, name='index'),
    path('hackerlist',hacker_list, name='hackerlist'),
    path('hackerdetail',user_detail, name='userdetail'),
    path('voted',vote, name='vote')

]


#coding=utf-8

from __future__ import print_function
from __future__ import division

import cv2
import tensorflow as tf
import numpy as np
import os
import colorsys

import config

from yolov3 import yolov3_body
from yolov3 import yolo_eval
from utils.get_anchors import get_anchors

flags = tf.app.flags
FLAGS = flags.FLAGS

flags.DEFINE_string('image_path', 'utils/test.jpg', 'Path to save training checkpoint.')
flags.DEFINE_string('ckpt_path', config._coco_tf_weights, 'Path to save training summary.')

flags.DEFINE_string('names_path', config._coco_names, 'Path to save training summary.')
flags.DEFINE_integer('num_classes', config._coco_classes, 'The classes number.')

with open(FLAGS.names_path, 'r') as f:
    classmap = f.readlines()

#colormap = config._coco_colormap

# Generate colors for drawing bounding boxes.
hsv_tuples = [(x / FLAGS.num_classes, 1., 1.)
              for x in range(FLAGS.num_classes)]
colormap = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
colormap = list(
    map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colormap))
np.random.seed(10101)  # Fixed seed for consistent colors across runs.
np.random.shuffle(colormap)  # Shuffle colors to decorrelate adjacent classes.
np.random.seed(None)  # Reset seed to default.

def letterbox_resize(img, new_width, new_height, interp=0):
    '''
    Letterbox resize. keep the original aspect ratio in the resized image.
    '''
    ori_height, ori_width = img.shape[:2]

    resize_ratio = min(new_width / ori_width, new_height / ori_height)

    resize_w = int(resize_ratio * ori_width)
    resize_h = int(resize_ratio * ori_height)

    img = cv2.resize(img, (resize_w, resize_h), interpolation=interp)
    image_padded = np.full((new_height, new_width, 3), 128, np.uint8)

    dw = int((new_width - resize_w) / 2)
    dh = int((new_height - resize_h) / 2)

    image_padded[dh: resize_h + dh, dw: resize_w + dw, :] = img

    return image_padded

def detect_image(image_path, i=1):
    img_ = cv2.imread(image_path)
    height, width, _ = img_.shape
    img = letterbox_resize(img_, config._input_size, config._input_size)

    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = np.asarray(img, np.float32)
    img = img[np.newaxis, :] / 255.

    input_data = tf.placeholder(tf.float32, [1, config._input_size, config._input_size, 3], name='input_data')

    with tf.variable_scope('yolov3'):

        yolo_outputs = yolov3_body(inputs=input_data, num_classes=FLAGS.num_classes, is_training=False)

    boxes_, scores_, classes_ = yolo_eval(yolo_outputs, num_classes=FLAGS.num_classes, image_shape=(height, width), max_boxes=config._max_boxes, score_threshold=.6, iou_threshold=.5)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        saver = tf.train.Saver()

        #saver.restore(sess, './data/checkpoints/yolov3.ckpt')

        ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_path)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
            print('Model restored')

        boxes, scores, ids = sess.run([boxes_, scores_, classes_], feed_dict={input_data: img})

        #print(boxes)

        for i, bbox in enumerate(boxes):
            img_ = cv2.rectangle(img_, (int(float(bbox[1])), int(float(bbox[0]))), (int(float(bbox[3])), int(float(bbox[2]))), colormap[int(ids[i])], 2)
            font = cv2.FONT_HERSHEY_COMPLEX
            cv2.putText(img_, classmap[int(ids[i])].strip(), (int(float(bbox[1])), int(float(bbox[0])) - 7), font, 0.5, (255, 255, 0), 1)
        if not os.path.exists('output'):
            os.mkdir('output')

        basename = os.path.basename(image_path).split('.')[0]

        cv2.imwrite('output/%s_pred.png'%basename, img_)

        if len(boxes) == 0:
            print("no object is detected.")
        elif len(boxes) == 1:
            print("one object is detected.")
            print(classmap[int(ids[0])].strip(), ": ",  scores[0])
        else:
            print("%d objects are detected." %len(boxes))
            for i in range(len(boxes)):
                print(classmap[ids[i]].strip(), ": ", scores[i])

        print("prediction file saved in output/%s_pred.png" %basename)

if __name__ == '__main__':
    detect_image(FLAGS.image_path)


# source: https://platform.openai.com/docs/api-reference/introduction
from pprint import pprint
import requests
from typing import Union, List
import aiohttp
import asyncio
from .config import ChatCompletionConfig
from ._return import BaseReturn,AsyncReturn
from openai.error import InvalidRequestError

class ChatGPT:
    def __init__(self, api_key:str, model:str='gpt-3.5-turbo', global_system:str= '', use_assist:bool = True):
        '''

        :param api_key: your openai api key
        :param model: model name
        :param global_system: The local_system message helps set the behavior of the assistant. In the example above, the assistant was instructed with “You are a helpful assistant.”
        '''
        self.model = model
        self.global_system = global_system
        # self.api_key = api_key
        self.headers = {
            "Content-Type": "application/json",
            "Authorization": "Bearer {}".format(api_key)
        }
        self.knowledge_base= dict()
        self.use_assist = use_assist
        self.__namer__ = -999999

    def __get_unique__name__(self):
        self.__namer__+=1
        return str(self.__namer__)

    def print_KB(self):
        pprint(self.knowledge_base)

    def __update_KB__(self, key:str, value:str):
        if key not in self.knowledge_base:
            self.knowledge_base[key] = [value]
        else:
            self.knowledge_base[key].append(value)

    def __render_KB__(self,user_name:str):
        if user_name in self.knowledge_base:
            return ', '.join(self.knowledge_base[user_name])
        else:
            return ''
    def __build_headers_data__(self,param:ChatCompletionConfig):
        # 设置请求头和参数
        headers = self.headers
        user_msg = param.user_msg
        local_system = param.local_system
        temperature = param.temparature
        top_p = param.top_p
        n = param.n
        stream = param.stream
        stop = param.stop
        max_tokens = param.max_tokens
        presence_penalty = param.presence_penalty
        frequency_penalty = param.frequency_penalty
        user_name = param.user_name if param.user_name is not None else self.__get_unique__name__()

        if param.assistant is not None:
            assistant = param.assistant
        else:
            if self.use_assist:
                assistant = self.__render_KB__(param.user_name)
            else:
                assistant = ''

        data = dict(
            model=self.model,
            messages=[
                {'role': 'system', 'content': self.global_system if local_system is None else local_system},
                {"role": "user", "content": user_msg},
                {'role': "assistant", 'content': assistant}
            ],
            temperature=temperature,
            top_p=top_p,
            n=n,
            stream=stream,
            stop=stop,
            max_tokens=max_tokens,
            presence_penalty=presence_penalty,
            frequency_penalty=frequency_penalty,
            user=user_name
        )
        return dict(
            headers = headers,
            data = data
        )

    def reply(self,
                param: ChatCompletionConfig,
              ) ->Union[BaseReturn, AsyncReturn]:

        # 1. build headers and data
        _  = self.__build_headers_data__(param)
        headers = _['headers']
        data = _['data']

        # 2. update knowledge base
        self.__update_KB__(param.user_name,param.user_msg)

        # 3. return request
        return self.__request__(headers,data,param.only_response)

    def __request__(self,headers, data, only_response):
        raise NotImplementedError()

class SyncChatGPT(ChatGPT):

    def __request__(self, headers, data,only_response,jupyter=None):
        # send request
        print('send request...')
        response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=data)

        # process response
        print('receive response...')
        if response.status_code == 200:
            result = response.json()
            if only_response:
                tmp = result['choices']
                return BaseReturn([i['message']['content'] for i in tmp])

            return BaseReturn(result)
        elif response.status_code == 400:
            raise InvalidRequestError(message=data['messages'],param=data)
        else:
            raise IOError(response.json())

class AsyncChatGPT(ChatGPT):

    def __request__(self, headers, data, only_response):
        async def request(headers,data,only_response):
            url = 'https://api.openai.com/v1/chat/completions'
            print('send request...')
            async with aiohttp.ClientSession(headers=headers) as session:
                async with session.post(url, json=data) as response:
                    if response.status == 200:
                        result = await response.json()
                        print('receive response...')
                        if only_response:
                            tmp = result['choices']
                            return [i['message']['content'] for i in tmp]
                        return result
                    elif response.status == 400:
                        raise InvalidRequestError(message=data['messages'],param=data)
                    else:
                        raise IOError(await response.json())

        return AsyncReturn(asyncio.get_event_loop().create_task(request(headers,data,only_response)))

    def multi_reply(self,
                            params: List[ChatCompletionConfig],
                          ):
        async def reply(param:ChatCompletionConfig):
            _ = self.__build_headers_data__(param)
            headers = _['headers']
            data = _['data']
            self.__update_KB__(param.user_name,param.user_msg)
            return await self.__request__(headers,data,param.only_response)

        async def m_request(params):
            return await asyncio.gather(*tuple(reply(param) for param in params))

        return AsyncReturn(asyncio.get_event_loop().create_task(m_request(params)))


import get_stock
import pandas as pd
from xlsxwriter import Workbook
from datetime import datetime,date

import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.dates import DayLocator, HourLocator, DateFormatter, drange

def main():

    start = datetime(2021,3,1)
    end = date.today()
    stock = '7203.JP'
    stock_data = get_stock.company(stock, start,end)
    #print(stock_data,name)

    #export to excel
    export_excel(stock_data,stock)

    #export to excel
    stock_graph(stock_data,stock)

def stock_graph(data,name):
    sns.set() #  コレ
    fig, ax = plt.subplots(figsize = (10,5))
    ax.plot(data['Date'],data['Open'])
    ax.set(xlabel ='Date',ylabel='Open',\
        ylim=(0,max(data['Open'])*1.2))

    from matplotlib import ticker
    ax.xaxis.set_major_locator(ticker.MultipleLocator(3))
    ax.xaxis.set_major_formatter(DateFormatter('%m-%d'))
#    plt.show()
    plt.savefig('./graph/'+name+'.png')

def export_excel(data,name):
    with pd.ExcelWriter('./data/stock_data.xlsx') as writer:
        data.to_excel(writer,sheet_name=name)

if __name__ == "__main__":
    main()


import os
from wand.image import Image
from shutil import copyfile
import glob
import pdb
import logging

modulate_presets_1 = { (-100, -200): 'dark'}
modulate_presets_2 = { (200, 300): 'default', (-1000, -1000): 'galaxy', (200, -800): 'flir'}
cwd = './defaults_test/'

def mkgif(filename):
    outfiles = ['testing_overlay/no_over.gif', 'testing_overlay/combine_prev_frame.gif']
    # outfile = tempfile.mkstemp(prefix="deepfry", suffix=".gif")[1]
    combine = False
    for outfile in outfiles:
        logging.info("About to run mkgif(%s) -> %s", filename, outfile)
        outdir = outfile.split('.')[0]
        tmpfile=outdir + '/tmp.jpg'
        if not os.path.isdir(outdir):
            os.mkdir(outdir)
        with Image(filename=filename) as original:
            zeros = len(str(len(original.sequence)))
            framecount = 0
            for frame in original.sequence:
                if not combine:
                    print('testing no composite call for result file %s' % outfile)
                    with Image(frame) as mod_frame:
                        mod_frame.modulate(200, -800)
                        mod_frame.evaluate(operator='gaussiannoise', value=0.05)
                        # mod_frame.function('sinusoid', params)
                        mod_frame.save(filename=(outdir + '/' + ('0' * (zeros - len(str(framecount)))) + str(framecount) + '.jpg'))
                    framecount += 1
                else:
                    print('TESTING COMPOSITE CALL for result file %s' % outfile)
                    do_prev = False
                    if framecount == 0:
                        print('frying first image')
                        with Image(frame) as mod_frame:
                            with Image(mod_frame) as img:
                                img.save(filename=tmpfile)
                            mod_frame.modulate(200, -800)
                            mod_frame.evaluate(operator='gaussiannoise', value=0.05)
                            # mod_frame.function('sinusoid', params)
                            mod_frame.save(filename=(outdir + '/' + ('0' * (zeros - len(str(framecount)))) + str(framecount) + '.jpg'))
                    else:
                        print('frying subsequent images: overlay with previous??')
                        with Image(tmpfile) as previous:
                            with Image(frame) as new:
                                previous.composite(new, left=0, top=0)
                                with Image(previous) as img:
                                    img.save(filename=tmpfile)
                                previous.modulate(200, -800)
                                previous.evaluate(operator='gaussiannoise', value=0.05)
                                # previous.function('sinusoid', params)
                                previous.save(filename=(outdir + '/' + ('0' * (zeros - len(str(framecount)))) + str(framecount) + '.jpg'))
                    framecount += 1
            os.remove(tmpfile)
        frames = sorted(glob.glob(outdir + '/*.jpg'))
        with Image() as final:
            for frame in frames:
                with Image(filename=frame) as img:
                    final.sequence.append(img)
            final.type='optimize'
            final.save(filename=outfile)
        combine = True
    return outfile, None

if not os.path.isdir(cwd):
    os.mkdir(cwd)
else:
    for filename in glob.glob(cwd + 'modulate*.jpg'):
        os.remove(filename)

def main():
    mkgif('./testing_overlay/signal-2018-12-15-201849.gif')

main()


import torch
import torchvision
from ssd.container import Container
import numpy as np

class PostProcessor:
    def __init__(self, cfg):
        super().__init__()
        self.cfg = cfg
        self.width = cfg.INPUT.IMAGE_SIZE[1]
        self.height = cfg.INPUT.IMAGE_SIZE[0]

    def __call__(self, detections):
        """
        Post processor of detections when we are doing inference.
        Applies two filter operations to remove all prior boxes that does not include an object:
        Confidence thresholding and non-maximum suppression.
        """
        batches_scores, batches_boxes = detections
        device = batches_scores.device
        batch_size = batches_scores.size(0)
        results = []
        for batch_id in range(batch_size):
            scores, boxes = batches_scores[batch_id], batches_boxes[batch_id]  # (N, #CLS) (N, 4)
            num_boxes = scores.shape[0]
            num_classes = scores.shape[1]

            boxes = boxes.view(num_boxes, 1, 4).expand(num_boxes, num_classes, 4)
            labels = torch.arange(num_classes, device=device)
            labels = labels.view(1, num_classes).expand_as(scores)

            # remove predictions with the background label
            boxes = boxes[:, 1:]
            scores = scores[:, 1:]
            labels = labels[:, 1:]

            # batch everything, by making every class prediction be a separate instance
            boxes = boxes.reshape(-1, 4)
            scores = scores.reshape(-1)
            labels = labels.reshape(-1)

            # remove low scoring boxes
            indices = torch.nonzero(scores > self.cfg.TEST.CONFIDENCE_THRESHOLD).squeeze(1)
            boxes, scores, labels = boxes[indices], scores[indices], labels[indices]

            boxes[:, 0::2] *= self.width
            boxes[:, 1::2] *= self.height

            keep = batched_nms(boxes, scores, labels, self.cfg.TEST.NMS_THRESHOLD, self.cfg.SOFT_NMS)
            # keep only topk scoring predictions
            keep = keep[:self.cfg.TEST.MAX_PER_IMAGE]
            boxes, scores, labels = boxes[keep], scores[keep], labels[keep]

            container = Container(boxes=boxes, labels=labels, scores=scores)
            container.img_width = self.width
            container.img_height = self.height
            results.append(container)
        return results

def batched_nms(boxes, scores, idxs, iou_threshold, soft_nms):
    """
    Performs non-maximum suppression in a batched fashion.

    Each index value correspond to a category, and NMS
    will not be applied between elements of different categories.

    Parameters
    ----------
    boxes : Tensor[N, 4]
        boxes where NMS will be performed. They
        are expected to be in (x1, y1, x2, y2) format
    scores : Tensor[N]
        scores for each one of the boxes
    idxs : Tensor[N]
        indices of the categories for each one of the boxes.
    iou_threshold : float
        discards all overlapping boxes
        with IoU < iou_threshold

    Returns
    -------
    keep : Tensor
        int64 tensor with the indices of
        the elements that have been kept by NMS, sorted
        in decreasing order of scores
    """
    if boxes.numel() == 0:
        return torch.empty((0,), dtype=torch.int64, device=boxes.device)
    # strategy: in order to perform NMS independently per class.
    # we add an offset to all the boxes. The offset is dependent
    # only on the class idx, and is large enough so that boxes
    # from different classes do not overlap
    max_coordinate = boxes.max()
    offsets = idxs.to(boxes) * (max_coordinate + 1)
    boxes_for_nms = boxes + offsets[:, None]
    if soft_nms:
        keep = soft_nms_pytorch(boxes_for_nms, scores, thresh=iou_threshold)
    else:
        keep = torchvision.ops.nms(boxes_for_nms, scores, iou_threshold)
    return keep

def soft_nms_pytorch(dets, box_scores, sigma=0.5, thresh=0.001, cuda=1):
    """
    # Author:Richard Fang, https://github.com/DocF/Soft-NMS
    Build a pytorch implement of Soft NMS algorithm.
    # Augments
        dets:        boxes coordinate tensor (format:[y1, x1, y2, x2])
        box_scores:  box score tensors
        sigma:       variance of Gaussian function
        thresh:      score thresh
        cuda:        CUDA flag
    # Return
        the index of the selected boxes
    """

    # Indexes concatenate boxes with the last column
    N = dets.shape[0]
    if cuda:
        indexes = torch.arange(0, N, dtype=torch.float).cuda().view(N, 1)
    else:
        indexes = torch.arange(0, N, dtype=torch.float).view(N, 1)
    dets = torch.cat((dets, indexes), dim=1)

    # The order of boxes coordinate is [y1,x1,y2,x2]
    y1 = dets[:, 0]
    x1 = dets[:, 1]
    y2 = dets[:, 2]
    x2 = dets[:, 3]
    scores = box_scores
    areas = (x2 - x1 + 1) * (y2 - y1 + 1)

    for i in range(N):
        # intermediate parameters for later parameters exchange
        tscore = scores[i].clone()
        pos = i + 1

        if i != N - 1:
            maxscore, maxpos = torch.max(scores[pos:], dim=0)
            if tscore < maxscore:
                dets[i], dets[maxpos.item() + i + 1] = dets[maxpos.item() + i + 1].clone(), dets[i].clone()
                scores[i], scores[maxpos.item() + i + 1] = scores[maxpos.item() + i + 1].clone(), scores[i].clone()
                areas[i], areas[maxpos + i + 1] = areas[maxpos + i + 1].clone(), areas[i].clone()

        # IoU calculate
        yy1 = np.maximum(dets[i, 0].to("cpu").numpy(), dets[pos:, 0].to("cpu").numpy())
        xx1 = np.maximum(dets[i, 1].to("cpu").numpy(), dets[pos:, 1].to("cpu").numpy())
        yy2 = np.minimum(dets[i, 2].to("cpu").numpy(), dets[pos:, 2].to("cpu").numpy())
        xx2 = np.minimum(dets[i, 3].to("cpu").numpy(), dets[pos:, 3].to("cpu").numpy())

        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)
        inter = torch.tensor(w * h).cuda() if cuda else torch.tensor(w * h)
        ovr = torch.div(inter, (areas[i] + areas[pos:] - inter))

        # Gaussian decay
        weight = torch.exp(-(ovr * ovr) / sigma)
        scores[pos:] = weight * scores[pos:]

    # select the boxes and keep the corresponding indexes
    keep = dets[:, 4][scores > thresh]
    return keep.to(torch.int64) #customized by us


"""
Created on 2022/8/8

This file was

Images are divided by inner radii and outer radii
A HoughCircles algorithm is run to obtain outer radii from the droplets
A Canny Edge Detection algorithm is run to obtain the inner radii from the condensates

Once these radii are obtained, each condensate is matched with its respective droplet and
its data will be matched containing each droplet's [inner radius, outer radius, concentration, temp]

Scientist notes:
Needs a cleanup

@author: Marco
"""
#%%
param1 = 12
param2 = 100
mindist= 100

dparam1 = 25
dparam2 = 60

import matplotlib.pyplot as plt
import pandas as pd
import math
import DropletRadiusDetection as drd
from matplotlib_scalebar.scalebar import ScaleBar
import skimage.filters.thresholding
from skimage import filters, io, feature, data
from scipy import ndimage as ndi
from skimage.morphology import remove_small_objects
from skimage.filters import try_all_threshold
from skimage.util import img_as_ubyte
from skimage.measure import label, regionprops, regionprops_table
import numpy as np
import cv2 as cv
import glob
from skimage import io
import DropletRadiusDetection
from DropletRadiusDetection import showFig

path = "C:/Users/mborj/Documents/Rogers Lab/22-08-05 LLPS Data Analysis/Trial 1 0.6mM/"
conc = "0.6mM"
files = (glob.glob(path + "*.tif"))
finalcsv = pd.DataFrame()

for i in range (20,80):
    temp = i
    list = []
    for filename in files:
        if str(temp) in filename:
            list.append(filename)
    if len(list) == 2:
        condensates = io.imread(list[0])
        droplets = io.imread(list[1])

        droplets = droplets[5:1195, 5:1915]
        condensates = condensates[5:1195, 5:1915]

        # Remove 5 pixels from each image edge
        droplets = droplets[5:1195, 5:1915]
        condensates = condensates[5:1195, 5:1915]

        # Apply a Gaussian Filter of Sigma 3 on inner and outer images
        sig = 3
        gauss = filters.gaussian(droplets, sigma=sig)
        condgauss = filters.gaussian(condensates, sigma = sig-2)

        cv_img = img_as_ubyte(condgauss)
        circles = cv.HoughCircles(cv_img, cv.HOUGH_GRADIENT, 1, mindist, param1=param1, param2=param2, minRadius=2, maxRadius=100)

        cv_img_drop = img_as_ubyte(gauss)
        circles_drop = cv.HoughCircles(cv_img_drop, cv.HOUGH_GRADIENT, 1, 200, param1=dparam1, param2=dparam2, minRadius=50, maxRadius=0)

        if circles is not None and circles_drop is not None:
            props = pd.DataFrame(circles[0])
            props = props.rename(columns = {0:"cond_x", 1:"cond_y", 2:"cond_radius" } )
            props_drop = pd.DataFrame(circles_drop[0])
            props_drop = props_drop.rename(columns = {0:"drop_x", 1:"drop_y", 2:"drop_radius" })

            numCond = props.shape[0]
            drop_x= pd.Series(float("NaN"), range(numCond))
            drop_y = pd.Series(float("NaN"), range(numCond))
            drop_radius = pd.Series(float("NaN"), range(numCond))

            complete = pd.DataFrame(columns = ["cond_x", "cond_y", "cond_radius", "drop_x", "drop_y", "drop_radius"])
            for i, row in props.iterrows():
                x = row["cond_x"]
                y = row["cond_y"]

                for j, drow in props_drop.iterrows():
                    dx = drow["drop_x"]
                    dy = drow["drop_y"]
                    radius = drow["drop_radius"]
                    # if(drd.isContained(x,y,dx,dy, radius)):
                    #     print("TRUE")
                    #     print("dropx: " + str(dx) + " condx: " + str(x))
                    #     print("dropy: " + str(dy) + " condy: " + str(y))
                    if drd.isContained(x,y,dx,dy,radius):
                        complete.loc[len(complete.index)] = [x,y,row["cond_radius"],dx,dy,radius]
                        # drop_x[j] = dx
                        # drop_y[j] = dy
                        # drop_radius[j] = radius

            # props['drop_x'] = drop_x
            # props['drop_y'] = drop_y
            # props['drop_radius'] = drop_radius
            complete['temp'] = temp
            complete['conc'] = conc

            cv_img, complete = drd.encircleProps(cv_img, complete)

            complete['drop_radius'] = 100/345 * complete['drop_radius']
            complete['cond_radius'] = 100/345 * complete['cond_radius']
            complete['drop_volume'] = 4.4332 * complete['drop_radius'] ** 2.89346
            complete['cond_volume'] = 4 / 3 * math.pi * complete['cond_radius'] ** 3

            finalcsv = pd.concat([finalcsv, complete])

        showFig(cv_img, title = str(temp) + "C")

finalcsv.to_csv(path + conc + " Matched Final.csv")
        #
        #
        # detectionTests = []
        # for i in [0.25*i for i in range(1,5)]:
        #     detectionTests.append(feature.canny(condensates, sigma = i))
        #
        # fig, axes = plt.subplots(figsize=(15, 6), nrows=2, ncols=2, sharex=True, sharey=True)
        #
        # for i in range(4):
        #     axes.flat[i].imshow(detectionTests[i], cmap=plt.cm.gray)
        #     axes.flat[i].set_axis_off()
        #     axes.flat[i].set_title(str(temp) + "C Sigma = {}".format(0.25 * i), fontsize=16)
        #
        # fig.tight_layout()
        # plt.show()
        # Use a Canny edge detector to detected condensate edges
        #
        # edges = feature.canny(condensates, sigma=0.25, low_threshold=25, high_threshold= 50)
        # fill_edges = ndi.binary_fill_holes(edges)
        # fill_edges = remove_small_objects(fill_edges, min_size=200)
        #
        # kernel = np.ones((5, 5), 'uint8')
        # bigEdges = ndi.binary_dilation(edges)
        # showFig(condensates+(bigEdges*255), title=str(temp) + "C inner")
        #
    # temp = filename[-7:-5]
    # print(temp)
    # showFig(io.imread(filename), title = "temp")


import pygame
from pygame.locals import *

# physics
def CollisionTest(Object1,ObjectList):
    CollisionList = []
    for Object in ObjectList:
        if Object.colliderect(Object1):
            CollisionList.append(Object)
    return CollisionList

class PhysicsObject(object):

    def __init__(self,x,y,x_size,y_size):
        self.width = x_size
        self.height = y_size
        self.rect = pygame.Rect(x,y,self.width,self.height)
        self.x = x
        self.y = y

    def move(self,Movement,platforms):
        self.x += Movement[0]
        self.rect.x = int(self.x)
        block_hit_list = CollisionTest(self.rect,platforms)
        collision_types = {'top':False,'bottom':False,'right':False,'left':False}
        for block in block_hit_list:
            if Movement[0] > 0:
                self.rect.right = block.left
                collision_types['right'] = True
            elif Movement[0] < 0:
                self.rect.left = block.right
                collision_types['left'] = True
            self.x = self.rect.x
        self.y += Movement[1]
        self.rect.y = int(self.y)
        block_hit_list = CollisionTest(self.rect,platforms)
        for block in block_hit_list:
            if Movement[1] > 0:
                self.rect.bottom = block.top
                collision_types['bottom'] = True
            elif Movement[1] < 0:
                self.rect.top = block.bottom
                collision_types['top'] = True
            self.change_y = 0
            self.y = self.rect.y
        return collision_types

    def Draw(self):
        pygame.draw.rect(screen,(0,0,255),self.rect)

    def CollisionItem(self):
        CollisionInfo = [self.rect.x,self.rect.y,self.width,self.height]
        return CollisionInfo

def flip(img,boolean=True):
    return pygame.transform.flip(img,boolean,False)

class entity(object):
    global animation_database

    def __init__(self,x,y,size_x,size_y):
        self.x = x
        self.y = y
        self.size_x = size_x
        self.size_y = size_y
        self.obj = PhysicsObject(x,y,size_x,size_y)
        self.animation = None
        self.image = None
        self.animation_frame = 0
        self.animation_tags = []
        self.flip = False
        self.offset = [0,0]

    def set_pos(self,x,y):
        self.x = x
        self.y = y
        self.obj.x = x
        self.obj.y = y
        self.obj.rect.x = x
        self.obj.rect.y = y

    def move(self,momentum,platforms):
        collisions = self.obj.move(momentum,platforms)
        self.x = self.obj.x
        self.y = self.obj.y
        return collisions

    def rect(self):
        return pygame.Rect(self.x,self.y,self.size_x,self.size_y)

    def set_flip(self,boolean):
        self.flip = boolean

    def set_animation_tags(self,tags):
        self.animation_tags = tags

    def set_animation(self,sequence):
        self.animation = sequence
        self.animation_frame = 0

    def clear_animation(self):
        self.animation = None

    def set_image(self,image):
        self.image = image

    def set_offset(self,offset):
        self.offset = offset

    def set_frame(self,amount):
        self.animation_frame = amount

    def change_frame(self,amount):
        self.animation_frame += amount
        if self.animation != None:
            while self.animation_frame < 0:
                if 'loop' in self.animation_tags:
                    self.animation_frame += len(self.animation)
                else:
                    self.animation = 0
            while self.animation_frame >= len(self.animation):
                if 'loop' in self.animation_tags:
                    self.animation_frame -= len(self.animation)
                else:
                    self.animation_frame = len(self.animation)-1

    def get_current_img(self):
        if self.animation == None:
            if self.image != None:
                return flip(self.image,self.flip)
            else:
                return None
        else:
            return flip(animation_database[self.animation[self.animation_frame]],self.flip)

    def display(self,surface,scroll):
        if self.animation == None:
            if self.image != None:
                surface.blit(flip(self.image,self.flip),(int(self.x)-scroll[0]+self.offset[0],int(self.y)-scroll[1]+self.offset[1]))
        else:
            surface.blit(flip(animation_database[self.animation[self.animation_frame]],self.flip),(int(self.x)-scroll[0]+self.offset[0],int(self.y)-scroll[1]+self.offset[1]))

# animation stuff
global animation_database
animation_database = {}

# a sequence looks like [[0,1],[1,1],[2,1],[3,1],[4,2]]
# the first numbers are the image name(as integer), while the second number shows the duration of it in the sequence
def animation_sequence(sequence,base_path,colorkey=(255,255,255),transparency=255):
    global animation_database
    result = []
    for frame in sequence:
        image_id = base_path + str(frame[0])
        image = pygame.image.load(image_id + '.png').convert()
        image.set_colorkey(colorkey)
        image.set_alpha(transparency)
        animation_database[image_id] = image.copy()
        for i in range(frame[1]):
            result.append(image_id)
    return result

def get_frame(ID):
    global animation_database
    return animation_database[ID]


import threading
import zmq
from msgpack_numpy import dumps as dump
from msgpack_numpy import loads as load
from tornado.concurrent import Future
from six.moves import queue
import time
import random
import os
from tqdm import tqdm
from queue import Queue

from intelActNet import Network
from config import config
from util import *

class MasterProcess(threading.Thread):
	def __init__(self, pipe_c2s=config.ip_addr+':8100', pipe_s2c=config.ip_addr+':8101'):
		super(MasterProcess, self).__init__()
		self.context = zmq.Context()
		self.c2s_socket = self.context.socket(zmq.PULL)
		self.c2s_socket.bind(pipe_c2s)
		self.s2c_socket = self.context.socket(zmq.ROUTER)
		self.s2c_socket.bind(pipe_s2c)
		self.start_time = time.time()
		self.network = Network()
		if config.continue_training:
			self.network.restore()

		self.predict_queue = queue.Queue(maxsize = config.agent_num)
		self.training_queue = queue.Queue(maxsize=config.agent_num*100)

		self.predictor = PredictThread(self.predict_queue, self.network)
		self.trainer = TrainingThread(self.training_queue, self.network)
		self.client_memory = [Queue(maxsize=35) for _ in range(config.agent_num)]

		self.predictor.start()
		self.trainer.start()

	def _put_predict_task(self, observation, callback):
		f = Future()
		f.add_done_callback(callback)
		if observation is not None:
			self.predict_queue.put([observation, f])
		else:
			f.set_result(None)
		return f

	def parse_memory(self, ident, observation, predicting_result):
		raise(NotImplementedError())

	def _on_state(self, index, obs):
		def cb(output):
			predicting_result = output.result()
			if predicting_result is not None:
				self.s2c_socket.send_multipart([nameClient(index), dump(predicting_result['action'])])
			training_data = self.parse_memory(index, obs, predicting_result)
			if training_data is not None:
				self.training_queue.put(training_data)

		self._put_predict_task(obs, cb)  # add one dimension as time axis

	def run(self):
		while True:
			for _ in tqdm(range(config.save_freq)):
				client_id, observations = load(self.c2s_socket.recv(copy = False).bytes)
				self._on_state(client_id, observations)

			self.network.save()
			config.update()

class PredictThread(threading.Thread):
	def __init__(self, predicting_queue, network):
		super(PredictThread, self).__init__()
		self.recv_queue = predicting_queue
		self.network = network

	def run(self):
		while True:
			predicting_batch = []
			future_batch = []
			observation, future = self.recv_queue.get()
			predicting_batch.append(observation)
			future_batch.append(future)
			while len(predicting_batch) < config.batch_size:
				try:
					observation, future = self.recv_queue.get_nowait()
					predicting_batch.append(observation)
					future_batch.append(future)
				except queue.Empty:
					break

			predicting_results = self.network.predict(predicting_batch)

			for c in range(predicting_results.shape[0]):
				predicting_result = predicting_results[c]
				if random.random() > config.p_explore:
					rtn_action = np.where(predicting_result == max(predicting_result))[0][0]
				else:
					rtn_action = np.random.choice(range(config.num_actions))

				final_result = dict()
				final_result['action'] = rtn_action
				future_batch[c].set_result(final_result)

class TrainingThread(threading.Thread):
	def __init__(self, training_queue, network):
		super(TrainingThread, self).__init__()
		self.recv_queue = training_queue
		self.network = network

	def run(self):
		while True:
			training_batch = []
			training_data = self.recv_queue.get()
			training_batch.append(training_data)
			while len(training_batch) < config.batch_size:
				try:
					training_data = self.recv_queue.get_nowait()
					training_batch.append(training_data)
				except queue.Empty:
					break

			self.network.train(training_batch)


"""
Insert the reviews file into the DB.
"""
from tqdm import tqdm

from cmt_statistics_tool.helper import fillna_strs, get_or_add, read_original_revision
from cmt_statistics_tool.tables import RevisionReview, SubmissionReview, async_session

async def insert_reviews(file: str) -> None:
    original, revision = read_original_revision(file)
    original = fillna_strs(
        original,
        [
            "Q6 (Three (or more) strong points about the paper. Please be precise and explicit; clearly explain the value and nature of the contribution.)",
            "Q7 (Three (or more) weak points about the paper. Please clearly indicate whether the paper has any mistakes, missing related work, or results that cannot be considered a contribution; write it so that the authors can understand what is seen as negative.)",
            "Q13 (Detailed Evaluation (Contribution, Pros/Cons, Errors); please number each point and please provide as constructive feedback as possible.)",
            "Q14 (Supplemental material. If the authors have provided supplemental material (data, code, etc.,), is the information likely to be sufficient to understand and to reproduce the experiments? Note that we do not expect actual reproducibility experiments, but rather a verification that the files are in fact there and are reasonable in scope and content.)",
            "Q15 (Revision. If revision is required, list specific required revisions you seek from the authors. Please number each point.)",
            "Q17 (Confidential comments for the PC Chairs. Please add any information that may help us reach a decision.)",
            "Q18 (Name and affiliation of external expert (!) reviewer (if applicable).)",
        ],
    )
    revision = fillna_strs(
        revision,
        [
            "Q6 (Additional comments to the authors on the revised version of the paper)",
            "Q18 (Confidential Comments for the PC Chairs. Please add any information that may help us reach a decision.)",
        ],
    )
    async with async_session() as session:
        for _, row in tqdm(  # Add reviews on submissions
            original.iterrows(), desc="Reviews Submissions", total=len(original)
        ):
            async with session.begin():
                submission = SubmissionReview(
                    reviewer=(
                        await get_or_add(
                            session,
                            row["Reviewer Name"].strip(),
                            row["Reviewer Email"].strip(),
                            "",
                        )
                    ),
                    submission_id=row["Paper ID"],
                    overall_rating=row["Q1 (Overall Rating)"],
                    relevance=row["Q2 (Relevant for PVLDB)"],
                    revision_possible=row[
                        "Q3 (Are there specific revisions that could raise your overall rating?)"
                    ],
                    paper_flavor=row[
                        "Q4 (Flavor of Regular Research Paper. Please indicate which flavor or flavors best describe the paper.)"
                    ],
                    summary=row[
                        "Q5 (Summary of the paper (what is being proposed and in what context) and a brief justification of your overall recommendation. One solid paragraph.)"
                    ],
                    strengths=row[
                        "Q6 (Three (or more) strong points about the paper. Please be precise and explicit; clearly explain the value and nature of the contribution.)"
                    ],
                    weaknesses=row[
                        "Q7 (Three (or more) weak points about the paper. Please clearly indicate whether the paper has any mistakes, missing related work, or results that cannot be considered a contribution; write it so that the authors can understand what is seen as negative.)"
                    ],
                    novelty=row[
                        "Q8 (Novelty. Please give a high novelty ranking to papers on new topics, opening new fields, or proposing truly new ideas; assign medium ratings to delta papers and papers on well-known topics but still with some valuable contribution.)"
                    ],
                    significance=row["Q9 (Significance)"],
                    technical_depth=row["Q10 (Technical Depth and Quality of Content)"],
                    experiments=row["Q11 (Experiments)"],
                    presentation=row["Q12 (Presentation)"],
                    details=row[
                        "Q13 (Detailed Evaluation (Contribution, Pros/Cons, Errors); please number each point and please provide as constructive feedback as possible.)"
                    ],
                    reproducibility=row[
                        "Q14 (Supplemental material. If the authors have provided supplemental material (data, code, etc.,), is the information likely to be sufficient to understand and to reproduce the experiments? Note that we do not expect actual reproducibility experiments, but rather a verification that the files are in fact there and are reasonable in scope and content.)"
                    ],
                    revision_items=row[
                        "Q15 (Revision. If revision is required, list specific required revisions you seek from the authors. Please number each point.)"
                    ],
                    confidence=row["Q16 (Rate your confidence in this review.)"],
                    confidential_comments=row[
                        "Q17 (Confidential comments for the PC Chairs. Please add any information that may help us reach a decision.)"
                    ],
                    external_reviewer=row[
                        "Q18 (Name and affiliation of external expert (!) reviewer (if applicable).)"
                    ],
                    trainee_agreement=row[
                        "Q19 (I understand that I am allowed to discuss a paper submission with a trainee for the purpose of teaching them how to review papers. I understand that (a) I am responsible to ensure that there is no COI according to the rules published at PVLDB.org between the trainee and any of the authors of the paper. (b) I have informed the trainee about the confidentiality of the content of the paper. (c) I am solely responsible for the final review. [If the trainee contributed significantly to the paper review, please list them above as external reviewer].)"
                    ],
                )
                session.add(submission)
        for _, row in tqdm(  # Add reviews on revisions
            revision.iterrows(), desc="Reviews Revisions", total=len(revision)
        ):
            async with session.begin():
                revision = RevisionReview(
                    reviewer=(
                        await get_or_add(
                            session,
                            row["Reviewer Name"].strip(),
                            row["Reviewer Email"].strip(),
                            "",
                        )
                    ),
                    revision_id=row["Paper ID"],
                    recommendation=row["Q1 (Final and Overall Recommendation)"],
                    revision_addressed=row[
                        "Q3 (Did the authors satisfactorily address the revision requirements identified in the meta-review of the original submission?)"
                    ],
                    justification=row[
                        "Q5 (Justify your answer to the above question by briefly addressing key revision items.)"
                    ],
                    comments_authors=row[
                        "Q6 (Additional comments to the authors on the revised version of the paper)"
                    ],
                    confidential_comments=row[
                        "Q18 (Confidential Comments for the PC Chairs. Please add any information that may help us reach a decision.)"
                    ],
                )
                session.add(revision)


import json
import random
import boto3
import time

topic = ""
region = ""
endpoint = ""

iot = boto3.client('iot-data', region_name=region,
                   endpoint_url=endpoint)

def lambda_handler(event, context):
    data = {
        "timestamp": time.time(),
        "temperature": random.uniform(0.0, 100.0),
        "humidity": random.uniform(0.0, 100.0),
        "pressure": random.uniform(0.0, 100.0)
    }

    try:
        response = iot.publish(
            topic=topic,
            qos=1,
            payload=json.dumps(data)
        )
        print(response)

        return {
            'statusCode': 200,
            'body': json.dumps('Dummy data published to IoT topic!')
        }
    except Exception as e:
        return {
            'statusCode': 500,
            'body': f'Error: {str(e)}'
        }


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.13.8
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# # Supplementary Video 2

# +
import joblib as jl
import numpy as np

import matplotlib.pyplot as plt
import matplotlib as mpl
import matplotlib.animation as animation
from mpl_toolkits.mplot3d.art3d import Line3DCollection
from matplotlib import cm
import matplotlib.gridspec as gridspec

from IPython.display import HTML
from collections import deque

from scipy.interpolate import make_interp_spline, BSpline

# -

# ! mkdir -p allen_movie_frames

# +
data = pd.read_hdf("../data/SupplVideo2.h5")
decoding = data["decoding"]
ca_train = decoding["ca_train"]
ca_test = decoding["ca_test"]
np_train = decoding["np_train"]
np_test = decoding["np_test"]
ca_decode = decoding["ca_decode"]
np_decode = decoding["np_decode"]

allen_ex_neural = data["np"]
allen_ex_ca = data["ca"]
ca_neural = allen_ex_ca["ca_trace"][:80]
np_neural = allen_ex_neural["np_data"]

movie = jl.load("../data/movie1_frames.jl")["frames"]

spl_ca = make_interp_spline(np.arange(900), ca_test[:, :3], k=3)
ca_test_smooth = spl_ca(np.arange(900))

spl_np = make_interp_spline(np.arange(3600), np_test[:, :3], k=3)
np_test_smooth = spl_np(np.arange(3600))

# +
start_idx = 0
length = 900
history_len = 200

def make_white(ax):
    ax.spines["left"].set_color("white")
    ax.spines["bottom"].set_color("white")
    ax.tick_params(axis="x", colors="white")
    ax.tick_params(axis="y", colors="white")

fig = plt.figure(figsize=(28, 8), dpi=150)

plt.subplots_adjust(wspace=0.015)

G = gridspec.GridSpec(2, 3, width_ratios=[1, 1, 0.9])

ax1_ca = fig.add_subplot(G[0, 0:1], xlim=(0, length + start_idx))
ax1_np = fig.add_subplot(
    G[1, 0:1],
)
ax2 = fig.add_subplot(G[:, 1], projection="3d")
ax2_ca = fig.add_subplot(G[0, 2:])
ax2_np = fig.add_subplot(G[1, 2:])

for t in ca_neural:
    ax1_ca.plot(t, c="darkgray")

ax1_np.imshow(
    np_neural.T.reshape(1000, 900, -1).mean(axis=-1)[:50, :],
    aspect="auto",
    cmap="gray",
    vmin=0,
    vmax=1.5,
)

ax1_ca.spines["right"].set_visible(False)
ax1_ca.spines["top"].set_visible(False)
ax1_ca.spines["left"].set_visible(False)
ax1_ca.spines["bottom"].set_visible(False)
ax1_ca.set_ylim([0, 4])
ax1_ca.axis("off")

ax1_ca.text(750, 4, "2-Photon", fontsize=20, color="white")

ax1_np.spines["right"].set_visible(False)
ax1_np.spines["top"].set_visible(False)
ax1_np.spines["left"].set_visible(False)
ax1_np.spines["bottom"].set_visible(False)
ax1_np.set_yticks([])
ax1_np.set_xticks(np.linspace(0, 899, 4), np.linspace(0, 30, 4, dtype=int), fontsize=15)
ax1_np.set_xlabel("Time (s)", fontsize=20, color="white")
ax1_np.text(700, -5, "Neuropixels", fontsize=20, color="white")

make_white(ax1_ca)
make_white(ax1_np)

cmap_gray = plt.cm.gray
cmap_magma = plt.cm.magma
cmap_viridis = plt.cm.viridis

z_list_ca = np.tile(np.arange(900), 9)
z_list_ca_test = np.arange(900)
z_list_np = np.tile(np.repeat(np.arange(900), 4), 9)
z_list_np_test = np.repeat(np.arange(900), 4)

colors_gray_dim = cmap_gray(z_list_np / z_list_np.max())
colors_ca_test = cmap_viridis(z_list_ca_test / z_list_ca_test.max())
colors_np_test = cmap_magma(z_list_np_test / z_list_np_test.max())
colors_gray_dim[:, -1] = np.tile(np.ones(3600), 9) / 10

idx1, idx2, idx3 = (0, 1, 2)

emb = ax2.scatter(
    np_train[:, idx1],
    np_train[:, idx2],
    np_train[:, idx3],
    c=colors_gray_dim[-500],
    s=15,
    depthshade=False,
)

test_scat_np = ax2.scatter([], [], [], s=1, c=[], cmap="magma", vmax=3600, vmin=0)
test_scat_ca = ax2.scatter([], [], [], s=1, c=[], cmap="viridis", vmax=900, vmin=0)

norm = mpl.colors.Normalize(vmin=0, vmax=900)
cb_ax = fig.add_axes([0.64, 0.15, 0.008, 0.6])
cb = mpl.colorbar.ColorbarBase(cb_ax, cmap=cm.magma, norm=norm)
cb.ax.set_title("Frame", fontsize=20, loc="center", y=-0.1, color="white")
cb.ax.yaxis.set_ticks_position("left")
cb.ax.tick_params(labelsize=15)
cb.ax.tick_params(colors="white")

ax2.set_title("CEBRA", fontsize=20, color="white")
ax2.set_xlim([-0.13, 0.13])
ax2.set_ylim([-0.08, 0.13])
ax2.set_zlim([-0.13, 0.13])

movie_ca = ax2_ca.imshow(movie[start_idx], aspect="auto", cmap="gray")
movie_np = ax2_np.imshow(movie[np_decode[start_idx * 4]], aspect="auto", cmap="gray")

ax2_ca.spines["right"].set_visible(False)
ax2_ca.spines["top"].set_visible(False)
ax2_ca.spines["left"].set_visible(False)
ax2_ca.spines["bottom"].set_visible(False)
ax2_ca.text(280, -10, "True Video", fontsize=20, color="white")
ax2_ca.axis("off")
ax2_np.spines["right"].set_visible(False)
ax2_np.spines["top"].set_visible(False)
ax2_np.spines["left"].set_visible(False)
ax2_np.spines["bottom"].set_visible(False)
ax2_np.text(250, -15, "Predicted Video", fontsize=20, color="white")
ax2_np.axis("off")

(ca_neural_pointer,) = ax1_ca.plot([0, 0], [0, 3.5], linewidth=5, c="green", alpha=0.5)
(np_neural_pointer,) = ax1_np.plot([0, 0], [0, 50], linewidth=5, c="purple", alpha=0.5)

ax2.axis("off")

def animate_position(i):
    rate = 1
    frame_step = 5
    frame_ratio = 4

    start_idx = 0
    """
    ## For Ca trace
    ca_points = ca_test_smooth[start_idx:start_idx+i*rate,:3][np.arange(0, i*rate, frame_step)].reshape(-1, 1, 3)
    ca_seg = np.concatenate([ca_points[:-1], ca_points[1:]], axis=1)
    ca_coll = Line3DCollection(ca_seg, colors = colors_ca_test[start_idx:(start_idx+i*rate)][np.arange(0, i*rate, frame_step)], linewidth = 3)
    ax2.add_collection(ca_coll)
    """

    ## For NP trace
    np_points = np_test_smooth[
        start_idx * frame_ratio : (start_idx + i * rate) * frame_ratio, :3
    ][np.arange(0, i * rate * frame_ratio, frame_step)].reshape(-1, 1, 3)
    np_seg = np.concatenate([np_points[:-1], np_points[1:]], axis=1)
    np_coll = Line3DCollection(
        np_seg,
        colors=colors_np_test[
            start_idx * frame_ratio : (start_idx + i * rate) * frame_ratio
        ][np.arange(0, i * rate * frame_ratio, frame_step)],
        linewidth=3,
    )
    ax2.add_collection(np_coll)

    ca_neural_pointer.set_data([i * rate + start_idx, i * rate + start_idx], [0, 2.5])
    np_neural_pointer.set_data([i * rate + start_idx, i * rate + start_idx], [0, 50])

    movie_ca.set_data(movie[i * rate + start_idx])
    movie_np.set_data(movie[np_decode[(i * rate + start_idx) * frame_ratio]])

    ## Set the folder you want to save ###

    fig.savefig(f"./allen_movie_frames//{i:03d}.png", transparent=True)
    print(i)
    plt.show()

ani = animation.FuncAnimation(fig, animate_position, 900, interval=50)


import os
import sys
import logging
import subprocess
from pathlib import Path
from ioco import util

this = sys.modules[__name__]
this.part = None
this.fs_type = None
this.config = None

def main(config):
    this.config = config
    if this.config.get("block"):
        timing_key = "oci block"
        util.start_timing(timing_key)

        logging.debug("Processing block command")
        this.part = this.config.get("--block-disk") + "1"
        this.fs_type = "xfs"

        # If no specific oci block options are set, then set all
        all = not this.config['--make-file-system'] \
            and not this.config['--mount']

        if all or this.config.get('--make-file-system'):
            make_file_system()
        if all or this.config.get('--mount'):
            mount_file_system()
        util.end_timing(timing_key)

def make_file_system():
    logging.debug("Making file system for block volume")
    timing_key = "oci block make_file_system"
    util.start_timing(timing_key)

    # Create partition, if needed
    if os.path.exists(this.part):
        logging.debug("Partition on " + this.config.get("--block-disk") + " is already created")
    else:
        logging.debug("Creating partition on " + this.config.get("--block-disk"))
        try:
            fdisk_tmp_sh = config.get("--home") + "/ioco-oci-block-partition.sh"
            f = open(fdisk_tmp_sh,"w")
            f.write("sudo fdisk " + this.config.get("--block-disk") + " <<EOF\nn\np\n\n\n\nw\nEOF")
            f.close()
        except:
            logging.error("Issue creating temp partition setup script")
            util.error_timings(timing_key)

        try:
            subprocess.run(["sudo","sh", fdisk_tmp_sh], check=True)
        except:
            logging.error("Issue creating partion")
            util.error_timings(timing_key)

    # Create file system, if needed
    logging.debug("Creating file sytem using " + this.part)
    try:
        subprocess.run(["sudo","mkfs." + this.fs_type, this.part], check=True)
    except:
        logging.info("File system did NOT create, assuming already created")
        util.error_timings(timing_key)

    util.end_timing(timing_key)

def mount_file_system():
    logging.debug("Mounting")
    timing_key = "oci block mount"
    util.start_timing(timing_key)

    try:
        Path(config.get('--block-path')).mkdir(parents=True, exist_ok=True)

        mount_tmp_sh = config.get("--home") + "/ioco-oci-block-mount.sh"
        f = open(mount_tmp_sh,"w")
        f.write("#!/bin/bash\n")
        f.write("if ! grep -q '" + part + "' /etc/fstab ; then\n")
        f.write("    echo '# ioco-oci-block-mount' >> /etc/fstab\n")
        f.write("    echo '" + part + " " + config.get("--block-path") + " " + fs_type + " defaults 0 2' >> /etc/fstab\n")
        f.write("fi")
        f.close()

        subprocess.run(["sudo","sh", mount_tmp_sh], check=True)
        subprocess.run(["sudo","mount", "-a"], check=True)
    except:
        logging.error("Issue mounting partion")
        util.error_timings(timing_key)
        raise

    util.end_timing(timing_key)


from ...data import EntityBase

class ConfigParameterBase(EntityBase):

    def __init__(self,
                 Name: str = None,
                 Type: str = None,
                 Value: str = None,
                 Description: str = None,
                 *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.Name: str = Name
        self.Type: str = Type
        self.Value: str = Value
        self.Description: str = Description


import logging
from typing import Any, Dict, List, Mapping, Optional

from langchain_core.callbacks import CallbackManagerForLLMRun
from langchain_core.language_models.llms import LLM
from langchain_core.pydantic_v1 import Extra, Field, SecretStr, root_validator
from langchain_core.utils import convert_to_secret_str, get_from_dict_or_env

logger = logging.getLogger(__name__)

class GooseAI(LLM):
    """GooseAI large language models.

    To use, you should have the ``openai`` python package installed, and the
    environment variable ``GOOSEAI_API_KEY`` set with your API key.

    Any parameters that are valid to be passed to the openai.create call can be passed
    in, even if not explicitly saved on this class.

    Example:
        .. code-block:: python

            from langchain_community.llms import GooseAI
            gooseai = GooseAI(model_name="gpt-neo-20b")

    """

    client: Any

    model_name: str = "gpt-neo-20b"
    """Model name to use"""

    temperature: float = 0.7
    """What sampling temperature to use"""

    max_tokens: int = 256
    """The maximum number of tokens to generate in the completion.
    -1 returns as many tokens as possible given the prompt and
    the models maximal context size."""

    top_p: float = 1
    """Total probability mass of tokens to consider at each step."""

    min_tokens: int = 1
    """The minimum number of tokens to generate in the completion."""

    frequency_penalty: float = 0
    """Penalizes repeated tokens according to frequency."""

    presence_penalty: float = 0
    """Penalizes repeated tokens."""

    n: int = 1
    """How many completions to generate for each prompt."""

    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not explicitly specified."""

    logit_bias: Optional[Dict[str, float]] = Field(default_factory=dict)
    """Adjust the probability of specific tokens being generated."""

    gooseai_api_key: Optional[SecretStr] = None

    class Config:
        """Configuration for this pydantic config."""

        extra = Extra.ignore

    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = {field.alias for field in cls.__fields__.values()}

        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name not in all_required_field_names:
                if field_name in extra:
                    raise ValueError(f"Found {field_name} supplied twice.")
                logger.warning(
                    f"""WARNING! {field_name} is not default parameter.
                    {field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        values["model_kwargs"] = extra
        return values

    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        gooseai_api_key = convert_to_secret_str(
            get_from_dict_or_env(values, "gooseai_api_key", "GOOSEAI_API_KEY")
        )
        values["gooseai_api_key"] = gooseai_api_key
        try:
            import openai

            openai.api_key = gooseai_api_key.get_secret_value()
            openai.api_base = "https://api.goose.ai/v1"
            values["client"] = openai.Completion
        except ImportError:
            raise ImportError(
                "Could not import openai python package. "
                "Please install it with `pip install openai`."
            )
        return values

    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling GooseAI API."""
        normal_params = {
            "temperature": self.temperature,
            "max_tokens": self.max_tokens,
            "top_p": self.top_p,
            "min_tokens": self.min_tokens,
            "frequency_penalty": self.frequency_penalty,
            "presence_penalty": self.presence_penalty,
            "n": self.n,
            "logit_bias": self.logit_bias,
        }
        return {**normal_params, **self.model_kwargs}

    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {**{"model_name": self.model_name}, **self._default_params}

    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "gooseai"

    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call the GooseAI API."""
        params = self._default_params
        if stop is not None:
            if "stop" in params:
                raise ValueError("`stop` found in both the input and default params.")
            params["stop"] = stop

        params = {**params, **kwargs}

        response = self.client.create(engine=self.model_name, prompt=prompt, **params)
        text = response.choices[0].text
        return text


import numpy as np

from fastai.text import SortSampler, SortishSampler

def test_sort_sampler_sorts_all_descending():
    bs = 4
    n = bs*100
    data = 2 * np.arange(n)
    samp = list(SortSampler(data, lambda i: data[i]))

    # The sample is a permutation of the indices.
    assert sorted(samp) == list(range(n))
    # And that "permutation" is for descending data order.
    assert all(s1 > s2 for s1, s2 in zip(samp, samp[1:]))

def test_sortish_sampler_sorts_each_batch_descending():
    bs = 4
    n = bs*100
    data = 2 * np.arange(n)
    samp = list(SortishSampler(data, lambda i: data[i], bs))

    # The sample is a permutation of the indices.
    assert sorted(samp) == list(range(n))
    # And that permutation is kind of reverse sorted.
    assert all(
        s1 > s2 or (i+1) % bs == 0  # don't check batch boundaries
        for i, (s1, s2) in enumerate(zip(samp, samp[1:]))
    )
    assert samp[0] == max(samp)


import gobject, gtk, pango

from plugin.Interface import Interface
from utils.HIGDialog import HIGDialog

#
# Dialog for inspecting controls.
#
class ControlInspector(HIGDialog):

    def __init__(self, ctrlclass):

        HIGDialog.__init__(self, buttons=(gtk.STOCK_CLOSE, gtk.RESPONSE_CLOSE))
        name = ctrlclass.__name__
        self.set_property("title", _("%s Control") % name)

        align = gtk.Alignment(0.0, 0.0, 0.0, 0.0)
        align.set_property("border-width", 6)
        frame = gtk.Frame()
        self.__scrollw = gtk.ScrolledWindow()
        frame.add(self.__scrollw)
        align.add(frame)

        self.__label = gtk.Label()
        self.__label.set_property("selectable", True)

        self.vbox.pack_start(align, False, False, 0)
        self.vbox.pack_start(self.__label, False, False, 0)

        # list the interfaces and their descriptions
        texts = Interface.gui_describe(ctrlclass)

        # create the listview now, with the content
        self.__create_listview(texts)

        def f(*args): self.destroy()
        self.connect("response", f)

        self.vbox.show_all()

    def __create_listview(self, texts):

        self.__label.set_property("label", texts[0][0])

        treemodel = gtk.TreeStore(gobject.TYPE_STRING, gobject.TYPE_STRING,
                                  gobject.TYPE_STRING, gobject.TYPE_STRING)
        treeview = gtk.TreeView(treemodel)

        textrenderer0 = gtk.CellRendererText()
        textrenderer1 = gtk.CellRendererText()

        column0 = gtk.TreeViewColumn(_("Interface Name"), textrenderer0)
        column0.add_attribute(textrenderer0, "text", 0)
        treeview.append_column(column0)
        column1 = gtk.TreeViewColumn(_("Property Name"), textrenderer1)
        column1.add_attribute(textrenderer1, "text", 1)
        treeview.append_column(column1)
        column2 = gtk.TreeViewColumn(_("Property Access"), textrenderer1)
        column2.add_attribute(textrenderer1, "text", 2)
        treeview.append_column(column2)
        column3 = gtk.TreeViewColumn(_("Property Description"), textrenderer1)
        column3.add_attribute(textrenderer1, "text", 3)
        treeview.append_column(column3)

        textrenderer0.set_property("weight", pango.WEIGHT_BOLD)
        textrenderer0.set_property("wrap-width", 250)
        textrenderer0.set_property("wrap-mode", pango.WRAP_WORD)
        textrenderer1.set_property("style", pango.STYLE_ITALIC)
        textrenderer1.set_property("wrap-width", 500)
        textrenderer1.set_property("wrap-mode", pango.WRAP_WORD)

        for (iface, items) in texts:

            myiter = treemodel.insert_before(None, None,
                                             (iface.split(":")[0], None, None,
                                              None))

            for (name, access, description) in items:
                treemodel.insert_before(myiter, None,
                                        (None, name, access, description))

        treeview.expand_all()
        treeview.columns_autosize()
        w, h = treeview.size_request()
        self.__scrollw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
        self.__scrollw.set_size_request(max(w + 20, 600), min(h, 400))
        self.__scrollw.add(treeview)


import torchvision
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
import torch

def load_fasterRCNN(max_size, min_size, num_classes, device):
  model = torchvision.models.detection.fasterrcnn_resnet50_fpn(weights="DEFAULT", max_size=max_size, min_size=min_size)
  in_features = model.roi_heads.box_predictor.cls_score.in_features
  model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
  model = model.to(device)
  return model

def get_model(max_size, min_size, num_classes, device, path_to_model=""):

  model = load_fasterRCNN(max_size, min_size, num_classes, device)

  if len(path_to_model) != 0:
    try:
      model.load_state_dict(torch.load(path_to_model, map_location = device)['model_state_dict'])
      print('Loaded saved model')
    except FileNotFoundError:
      print('File not found, loaded default model')

  else:
    print('Loaded default model')

  return model


"""
Example of "constant expression" propagation.
A "constant expression" is an expression based on constants or init regs.

"""

from argparse import ArgumentParser

from miasm.analysis.machine import Machine
from miasm.analysis.binary import Container
from miasm.analysis.cst_propag import propagate_cst_expr
from miasm.analysis.data_flow import DeadRemoval, \
    merge_blocks, remove_empty_assignblks
from miasm.expression.simplifications import expr_simp
from miasm.core.locationdb import LocationDB

parser = ArgumentParser(description="Constant expression propagation")
parser.add_argument('filename', help="File to analyze")
parser.add_argument('address', help="Starting address for disassembly engine")
parser.add_argument('-s', "--simplify", action="store_true",
                    help="Apply simplifications rules (liveness, graph simplification, ...)")

args = parser.parse_args()

machine = Machine("x86_32")

loc_db = LocationDB()
cont = Container.from_stream(open(args.filename, 'rb'), loc_db)
mdis = machine.dis_engine(cont.bin_stream, loc_db=loc_db)
lifter = machine.lifter_model_call(mdis.loc_db)
addr = int(args.address, 0)
deadrm = DeadRemoval(lifter)

asmcfg = mdis.dis_multiblock(addr)
ircfg = lifter.new_ircfg_from_asmcfg(asmcfg)
entry_points = set([mdis.loc_db.get_offset_location(addr)])

init_infos = lifter.arch.regs.regs_init
cst_propag_link = propagate_cst_expr(lifter, ircfg, addr, init_infos)

if args.simplify:
    ircfg.simplify(expr_simp)
    modified = True
    while modified:
        modified = False
        modified |= deadrm(ircfg)
        modified |= remove_empty_assignblks(ircfg)
        modified |= merge_blocks(ircfg, entry_points)

open("%s.propag.dot" % args.filename, 'w').write(ircfg.dot())


# -*- coding: utf-8 -*-
"""
Created on Thu May  9 19:12:25 2019

@author: zsl
"""

from math import log
import operator
import treePlotter
from sklearn.datasets import load_iris
import numpy as np
import pandas as pd

def calEntropy(dataset):
    '''
    计算熵
    '''
    data_dic ={}
    total = len(dataset)
    ent = 0.0
    for line in dataset:
        label = line[-1]
        if label not in data_dic.keys():
            data_dic[label] = 0
        data_dic[label] += 1
    for key in data_dic:
        prob = float(data_dic[key])/total
        ent -= prob*log(prob,2)
    return ent

def splitData(dataset,i,value):
    '''
     以第i列值为value的进行划分，得到划分结果为value的数据集，但是此时不包含第i列
    '''
    redataset =[]
    for line in dataset:
        if line[i] == value:
            reline = line[:i]
            reline.extend(line[i+1:])
            redataset.append(reline)
    return redataset

def majorityCnt(classList):
    '''
      在遇到属性都用完了之后，还存在不一致的情况  投票选出分类结果
    '''
    classCount = {}
    for vote in classList:
        if vote not in classCount.keys():
            classCount[vote] =0
        classCount[vote] += 1
    sortedClassCount = sorted(classCount.items(),key=operator.itemgetter(1),reverse=True)
    return sortedClassCount[0][0]

#CART算法
def CART_chooseBestFeatureToSplit(dataset):

    numFeatures = len(dataset[0]) - 1
    bestGini = 999999.0
    bestFeature = -1
    for i in range(numFeatures):
        featList = [example[i] for example in dataset]
        uniqueVals = set(featList)
        gini = 0.0
        for value in uniqueVals:
            subdataset=splitData(dataset,i,value)
            p=len(subdataset)/float(len(dataset))
            subp = len(splitData(subdataset, -1, '0')) / float(len(subdataset))
        gini += p * (1.0 - pow(subp, 2) - pow(1 - subp, 2))
#        print(u"CART中第%d个特征的基尼值为：%.3f"%(i,gini))
        if (gini < bestGini):
            bestGini = gini
            bestFeature = i
    return bestFeature

def CART_createTree(dataset,labels):
    classList=[example[-1] for example in dataset]
    if classList.count(classList[0]) == len(classList):
        # 类别完全相同，停止划分
        return classList[0]
    if len(dataset[0]) == 1:
        # 遍历完所有特征时返回出现次数最多的
        return majorityCnt(classList)
    bestFeat = CART_chooseBestFeatureToSplit(dataset)
    #print(u"此时最优索引为："+str(bestFeat))
    bestFeatLabel = labels[bestFeat]
#    print(u"此时最优索引为："+(bestFeatLabel))
    CARTTree = {bestFeatLabel:{}}
    del(labels[bestFeat])
    # 得到列表包括节点所有的属性值
    featValues = [example[bestFeat] for example in dataset]
    uniqueVals = set(featValues)
    for value in uniqueVals:
        subLabels = labels[:]
        CARTTree[bestFeatLabel][value] = CART_createTree(splitData(dataset, bestFeat, value), subLabels)
    return CARTTree

def classify(inputTree, featLabels, testVec):
    """
    输入：决策树，分类标签，测试数据
    输出：决策结果
    描述：跑决策树
    """
    firstStr = list(inputTree.keys())[0]
    secondDict = inputTree[firstStr]
    featIndex = featLabels.index(firstStr)
    classLabel = '0'
    for key in secondDict.keys():
        if testVec[featIndex] == key:
            if type(secondDict[key]).__name__ == 'dict':
                classLabel = classify(secondDict[key], featLabels, testVec)
            else:
                classLabel = secondDict[key]
    return classLabel

def classifytest(inputTree, featLabels, testDataSet,target_test):
    """
        测试
    """
    i =0
    cnt = 0
    for testVec in testDataSet:
        pre = classify(inputTree, featLabels, testVec)
        if pre == target_test[i]:
            cnt +=1
    return cnt/len(target_test)

if __name__ == '__main__':
    data=load_iris()
    dataset = data.data
    labels = data.feature_names
    target = data.target
    dataset[:,0]=pd.cut(dataset[:,0],2,labels=[1,2])
    dataset[:,1]=pd.cut(dataset[:,1],2,labels=[1,2])
    dataset[:,2]=pd.cut(dataset[:,2],2,labels=[1,2])
    dataset[:,3]=pd.cut(dataset[:,3],2,labels=[1,2])

    for i in range(len(target)):
        if target[i] == 2:
            target[i] = 0

    # 使数据集乱序
    num_example = dataset.shape[0]
    array = np.arange(num_example)
    np.random.shuffle(array)
    dataset = dataset[array]
    target = target[array]
    # 训练集: 验证集 = 9: 1
    sample = np.int(num_example * 0.9)
    x_train = dataset[: sample]
    y_train = target[: sample]
    x_test = dataset[sample:]
    y_test = target[sample:]

    dataset = x_train.tolist()
    target = y_train.tolist()
    for i in range(len(dataset)):
        dataset[i].extend([target[i]])

    dataset_test = x_test.tolist()
    target_test = y_test.tolist()
    for i in range(len(dataset_test)):
        dataset_test[i].extend([target_test[i]])

    res=[]
    for j in range(10):
        pri = []
        tmp = 0
        prob = 0.1
        for i in range(10):
            labels_tmp = labels[:] # 拷贝，createTree会改变labels
            CARTdesicionTree = CART_createTree(dataset[tmp:np.int(sample*prob)],labels_tmp[tmp:np.int(sample*prob)])
    #        print('CARTdesicionTree:\n', CARTdesicionTree)
    #        treePlotter.CART_Tree(CARTdesicionTree)
            pre = classifytest(CARTdesicionTree, labels, dataset_test,target_test)
            pri.append(pre)
            tmp=np.int(sample*prob)+1
            prob += 0.1
        res.append(sum(pri)/len(pri))
    print('CART分类树，10次十折交叉验证结果:',sum(pri)/len(pri))


from Bio import SeqIO
import pandas as pd
import multiprocessing
import numpy as np

# Translates the DNA sequence into AA sequence

# USearch v11 was used to merge and filter deep sequence reads.
# The zotus and otutab files are .txt files generated from USearch analysis.
# Outfile is a pklfile with DNA and AA sequences and sequence counts.

def translate(dna):
    transdic={"TTT":"F","TTC":"F","TTA":"L","TTG":"L",
              "CTT":"L","CTC":"L","CTA":"L","CTG":"L",
              "ATT":"I","ATC":"I","ATA":"I","ATG":"M",
              "GTT":"V","GTC":"V","GTA":"V","GTG":"V",
              "TCT":"S","TCC":"S","TCA":"S","TCG":"S",
              "CCT":"P","CCC":"P","CCA":"P","CCG":"P",
              "ACT":"T","ACC":"T","ACA":"T","ACG":"T",
              "GCT":"A","GCC":"A","GCA":"A","GCG":"A",
              "TAT":"Y","TAC":"Y","TAA":"Z","TAG":"Z",
              "CAT":"H","CAC":"H","CAA":"Q","CAG":"Q",
              "AAT":"N","AAC":"N","AAA":"K","AAG":"K",
              "GAT":"D","GAC":"D","GAA":"E","GAG":"E",
              "TGT":"C","TGC":"C","TGA":"Z","TGG":"W",
              "CGT":"R","CGC":"R","CGA":"R","CGG":"R",
              "AGT":"S","AGC":"S","AGA":"R","AGG":"R",
              "GGT":"G","GGC":"G","GGA":"G","GGG":"G"}
    AAseq=[]
    if len(dna)%3!=0:
        return "FRAMESHIFT"
    for i in range(0,len(dna),3):
        AAseq.append(transdic[str(dna[i:i+3])])
    AAseq=''.join(AAseq)
    return AAseq

def main(zotus, otutab, outfile):
	#first part loads zotus and translates to AA
    num_cores = multiprocessing.cpu_count()
    pool=multiprocessing.Pool(processes=num_cores) # to run data in parallel, based on number of cpus available
    DNAseq,name=[],[]
    with open(zotus) as indna:        # zotus = 'zotus.fasta'
        parser= SeqIO.parse(indna,'fasta')       # SeqIO.parse can upload a fasta file and read it
        for i in parser:
            DNAseq.append(str(i.seq))
            name.append(i.id)
    (AAseq)=pool.map(translate,DNAseq)

    name_df=pd.DataFrame([name,DNAseq,AAseq])  # pd.DataFrame sets up table where name, DNAseq, AAseq are headers to columns
    name_df=name_df.transpose()              # rows to columns and columns to rows
    name_df.columns=['Zotu_name','DNA','AA']

    #this part merges name_df with the otu-table
    otu_table=otu_table=pd.read_csv(otutab,sep='\t',header=0)    #reading '\t' tab-delimited file
                                                                                #header=0 keeps headers from

    name_df['BC'] = name_df['AA'].str.split('YRITY').str[0]
    name_df['BC'].replace('', np.nan, inplace=True)

    name_df['DE'] = name_df['AA'].str.extract('TVPG(.*)ATIS')
    name_df['DE'].replace('', np.nan, inplace=True)

    name_df['FG'] = name_df['AA'].str.split('TITVYAV').str[-1]
    name_df['FG'].replace('', np.nan, inplace=True)

    name_df['AA'].replace('FRAMESHIFT', np.nan, inplace=True)
    name_df=name_df.dropna() #this removes short reads that lack paratope and frameshifts

    #pd.merge = amazing! Even though zotus.fasta was in random zotu order, still lined up correctly
    merged_otu=pd.merge(name_df,otu_table,how='left',left_on='Zotu_name',right_on='#OTU ID')   #where to merge
    merged_otu=merged_otu.dropna() #this removes short reads that lack paratope and frameshifts
    merged_otu.to_pickle(outfile)   # 'merged_df.pkl'

if __name__ == '__main__':
    main()