import numpy as np
np.set_printoptions(threshold=np.inf)
import logging
import glob
import pickle
from functools import reduce
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import SimpleRNN
from keras.layers import LSTM
from keras.preprocessing import sequence
from keras.utils import to_categorical
from keras.optimizers import SGD, RMSprop
from utils.constants import Constants

maximum_length = 1000

def build_keras_rnn():
    model = Sequential()
    model.add(LSTM(100, input_shape=(maximum_length, 1)))
    model.add(Dense(1000, activation='sigmoid'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    print(model.summary())
    return model

def train_keras_rnn(model):
    # load prepare the data
    #train_speakers_dataset, test_speaker_dataset = load_all_speakers()
    pickle_files = glob.glob('*-set.pickle')
    with open(pickle_files[1], 'rb') as train_file:
        train_speakers_dataset = pickle.load(train_file)
    with open(pickle_files[0], 'rb') as test_file:
        test_speaker_dataset = pickle.load(test_file)

    X_train = train_speakers_dataset.X
    y_train = train_speakers_dataset.y
    X_test = test_speaker_dataset.X
    y_test = test_speaker_dataset.y
    logging.debug('Total size of train dataset: ' + str(np.shape(X_train)))
    logging.debug('Total size of test dataset: ' + str(np.shape(X_test)))

    # old padding strategy. leaving it here for reference.
    #X_train_w = sequence.pad_sequences(X_train, maxlen=maximum_length)
    #X_test_w = sequence.pad_sequences(X_test, maxlen=maximum_length)

    # pad sequences
    X_temp = np.zeros((np.shape(X_train)[0], maximum_length))
    i = 0
    for x in X_train:
        X_temp[i] = x[0:maximum_length]
        i += 1
    X_train = X_temp

    X_temp = np.zeros((np.shape(X_test)[0], maximum_length))
    i = 0
    for x in X_test:
        X_temp[i] = x[0:maximum_length]
        i += 1
    X_test = X_temp

    logging.debug('Total size of train dataset: ' + str(np.shape(X_train)))
    logging.debug('Total size of test dataset: ' + str(np.shape(X_test)))

    X_train = np.reshape(X_train, (X_train.shape[0], maximum_length, 1))
    X_test = np.reshape(X_test, (X_test.shape[0], maximum_length, 1))

    logging.debug('Total size of train dataset: ' + str(np.shape(X_train)))
    logging.debug('Total size of test dataset: ' + str(np.shape(X_test)))

    y_train = to_categorical(y_train)
    y_test = to_categorical(y_test)
    model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=100, batch_size=64)
    return model

def test_keras_rnn(trained_model):
    pass

def load_all_speakers():
    dataset_list = []
    for speaker in Constants.AVAILABLE_SPEAKERS:
        temp_dataset = OSXSpeakerDataset(speaker)
        temp_dataset.load()
        dataset_list.append(temp_dataset)
    return reduce((lambda x, y: x + y), dataset_list[:-1]), dataset_list[-1]

if __name__ == '__main__':
    np.random.seed(7)
    model = build_keras_rnn()
    trained_model = train_keras_rnn(model)
    model_path = 'adam-epochs100-batchsize64-seqlength1000.pickle'
    model.save(model_path)
    test_keras_rnn(trained_model)


import sys
import time
import pytest

import ray
from ray import serve
from ray._private.test_utils import wait_for_condition
from ray.serve.backend_state import ReplicaState
from ray.serve.config import AutoscalingConfig
from ray.serve.autoscaling_policy import calculate_desired_num_replicas

class TestCalculateDesiredNumReplicas:
    def test_bounds_checking(self):
        num_replicas = 10
        max_replicas = 11
        min_replicas = 9
        config = AutoscalingConfig(
            max_replicas=max_replicas,
            min_replicas=min_replicas,
            target_num_ongoing_requests_per_replica=100)

        desired_num_replicas = calculate_desired_num_replicas(
            autoscaling_config=config,
            current_num_ongoing_requests=[150] * num_replicas)
        assert desired_num_replicas == max_replicas

        desired_num_replicas = calculate_desired_num_replicas(
            autoscaling_config=config,
            current_num_ongoing_requests=[50] * num_replicas)
        assert desired_num_replicas == min_replicas

        for i in range(50, 150):
            desired_num_replicas = calculate_desired_num_replicas(
                autoscaling_config=config,
                current_num_ongoing_requests=[i] * num_replicas)
            assert min_replicas <= desired_num_replicas <= max_replicas

    def test_scale_up(self):
        config = AutoscalingConfig(
            min_replicas=0,
            max_replicas=100,
            target_num_ongoing_requests_per_replica=1)
        num_replicas = 10
        num_ongoing_requests = [2.0] * num_replicas
        desired_num_replicas = calculate_desired_num_replicas(
            autoscaling_config=config,
            current_num_ongoing_requests=num_ongoing_requests)
        assert 19 <= desired_num_replicas <= 21  # 10 * 2 = 20

    def test_scale_down(self):
        config = AutoscalingConfig(
            min_replicas=0,
            max_replicas=100,
            target_num_ongoing_requests_per_replica=1)
        num_replicas = 10
        num_ongoing_requests = [0.5] * num_replicas
        desired_num_replicas = calculate_desired_num_replicas(
            autoscaling_config=config,
            current_num_ongoing_requests=num_ongoing_requests)
        assert 4 <= desired_num_replicas <= 6  # 10 * 0.5 = 5

    def test_smoothing_factor(self):
        config = AutoscalingConfig(
            min_replicas=0,
            max_replicas=100,
            target_num_ongoing_requests_per_replica=1,
            smoothing_factor=0.5)
        num_replicas = 10

        num_ongoing_requests = [4.0] * num_replicas
        desired_num_replicas = calculate_desired_num_replicas(
            autoscaling_config=config,
            current_num_ongoing_requests=num_ongoing_requests)
        assert 24 <= desired_num_replicas <= 26  # 10 + 0.5 * (40 - 10) = 25

        num_ongoing_requests = [0.25] * num_replicas
        desired_num_replicas = calculate_desired_num_replicas(
            autoscaling_config=config,
            current_num_ongoing_requests=num_ongoing_requests)
        assert 5 <= desired_num_replicas <= 8  # 10 + 0.5 * (2.5 - 10) = 6.25

@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
def test_e2e_basic_scale_up_down(serve_instance):
    """Send 100 requests and check that we autoscale up, and then back down."""

    @serve.deployment(
        _autoscaling_config={
            "metrics_interval_s": 0.1,
            "min_replicas": 1,
            "max_replicas": 2,
            "look_back_period_s": 0.2
        },
        # We will send over a lot of queries. This will make sure replicas are
        # killed quickly during cleanup.
        _graceful_shutdown_timeout_s=1,
        max_concurrent_queries=1000,
        version="v1")
    class A:
        def __call__(self):
            time.sleep(1)

    A.deploy()
    handle = A.get_handle()
    [handle.remote() for _ in range(100)]

    controller = serve_instance._controller

    def get_num_running_replicas():
        replicas = ray.get(
            controller._dump_replica_states_for_testing.remote("A"))
        running_replicas = replicas.get([ReplicaState.RUNNING])
        return len(running_replicas)

    wait_for_condition(lambda: get_num_running_replicas() >= 2)

    # As the queue is drained, we should scale back down.
    wait_for_condition(lambda: get_num_running_replicas() <= 1)

if __name__ == "__main__":
    import sys
    import pytest
    sys.exit(pytest.main(["-v", "-s", __file__]))


import numpy as np
import pandas as pd
import random
import matplotlib.pyplot as plt
import glob as gb
import efficientnet.tfkeras as efn
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import load_img
from tensorflow.keras.applications import EfficientNetB4
from tensorflow.keras.optimizers.legacy import Adam
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, TensorBoard, ModelCheckpoint
from tensorflow.keras.utils import plot_model
from IPython.display import SVG, Image
import cv2
from sklearn.preprocessing import MultiLabelBinarizer
import os
import hashlib

from PIL import Image
import albumentations as A

print(tf.__version__)

# tf.config.set_visible_devices([], 'GPU')

gpus = tf.config.list_physical_devices('GPU'); print(gpus)
if len(gpus)==1: strategy = tf.distribute.OneDeviceStrategy(device="/gpu:0")
else: strategy = tf.distribute.MirroredStrategy()

tf.config.optimizer.set_experimental_options({"auto_mixed_precision": True})
print('Mixed precision enabled')

def load_image(filename):
    image = load_img("./data/img_sz_256/" + filename)
    plt.imshow(image)

def load_random_image(filenames):
    sample = random.choice(filenames)
    image = load_img("./data/img_sz_256/" + sample)
    plt.imshow(image)

def load_image_for_augmentation(image_path):
    image = cv2.imread(image_path)
    if image.shape[-1] == 1:
        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)

    image = np.array(image)
    return image

# adding path we will use in this nothebook
def format_resized_image_path_gcs(st):
    return GCS_DS_PATH + '/img_sz_256/' + st
def format_tpu_path(st):
    return './data/img_sz_256/' + st

train_df = pd.read_csv("./train.csv")

RESIZED_IMAGE_PATH = "./data/img_sz_256/"

train_df.labels.value_counts()

initial_length = len(train_df)

# create dictionary to store hashes and paths
hashes = {}

duplicates = []
originals = []

# loop over rows in the dataframe
for index, row in train_df.iterrows():
    # get the filename of the image
    filename = row['image']

    # compute the hash of the image
    with open(os.path.join(RESIZED_IMAGE_PATH, filename), 'rb') as f:
        hash = hashlib.md5(f.read()).hexdigest()

    # check if hash already exists in dictionary
    if hash in hashes:
        duplicates.append(filename)

        originals.append(hashes[hash])
        # delete duplicate row from dataframe
        train_df.drop(index, inplace=True)
    else:
        # add hash and path to dictionary
        hashes[hash] = filename

# print number of duplicates found
print(f"Number of duplicates found: {initial_length-len(hashes)}")

# Extract the labels column from the dataframe and store it in a list
labels = train_df['labels'].tolist()

# Create a set of unique labels
unique_labels = set()

# Iterate through each label in the list of labels and add it to the set of unique labels
for label in labels:
    unique_labels.update(label.split())

# Print the number of unique labels
print(unique_labels, "suma:", len(unique_labels))

# Select the six most common unique labels
common_labels = [label[0] for label in pd.Series(labels).str.split(expand=True).stack().value_counts()[:6].items()]
# Create a MultiLabelBinarizer object with the six common labels
mlb = MultiLabelBinarizer(classes=common_labels)

# Transform the labels into a binary matrix with one-hot encoding for the six common labels
label_matrix = mlb.fit_transform(train_df['labels'].str.split())

# Create a new dataframe with the one-hot encoded labels
label_df = pd.DataFrame(label_matrix, columns=common_labels)

# fixes train_df row number error
train_df.reset_index(drop=True, inplace=True)
label_df.reset_index(drop=True, inplace=True)

# Concatenate the new label dataframe with the original dataframe
new_df = pd.concat([train_df, label_df], axis=1)

# Drop the original labels column from the new dataframe
train_df = new_df.drop('labels', axis=1)
train_df.head()

# Training and validation split

#===============================================================
#===============================================================
# X = train_df.image.apply(format_resized_image_path_gcs).values
X = train_df.image.apply(format_tpu_path).values
y = np.float32(train_df.loc[:, 'healthy':'scab'].values)

df = train_df

# Get the hot encoded label columns
label_cols = [col for col in df.columns if not col.startswith('image')]

# Split the dataframe into features (X) and labels (y)
# X = df.drop(label_cols, axis=1)
y = df[label_cols]

# Determine the size of the validation set
validation_size = 0.2

# Split the data into training and validation sets while ensuring that the label distribution is balanced in both sets
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=validation_size, stratify=y, random_state=25)

arr_df = pd.DataFrame(X_train, columns=['image'])

# Reset the index of arr_df
arr_df = arr_df.reset_index(drop=True)
y_train = y_train.reset_index(drop=True)

train_df = pd.concat([arr_df, y_train], axis=1)

def cutout_image(image):
    image_array = np.array(image)
    augmented_array = A.CoarseDropout(max_holes=8, max_height=12, max_width=12, always_apply=True)(image=image_array)['image']

    return Image.fromarray(augmented_array)

def elastic_transform_image(image):
    image_array = np.array(image)
    augmented_array = A.ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03, always_apply=True)(image=image_array)['image']

    return Image.fromarray(augmented_array)

def gaussian_noise_image(image):
    image_array = np.array(image)
    augmented_array = A.GaussNoise(var_limit=(15.0, 60.0), always_apply=True)(image=image_array)['image']

    return Image.fromarray(augmented_array)

df = train_df

labels = unique_labels

# Get the counts of each label
label_counts = df[list(unique_labels)].sum()

# Calculate the number of rows to be added for each label
total_rows_added = int(len(df) * 0.03)
rows_added_per_label = (total_rows_added * label_counts / label_counts.sum()).astype(int)

# Find the maximum value
max_value = rows_added_per_label.max()

# Make the smallest value the largest
rows_added_per_label = max_value - rows_added_per_label + max_value

print(rows_added_per_label)

# Create a new DataFrame for the augmented images
cutout_df = pd.DataFrame(columns=df.columns)
elastic_transform_df = pd.DataFrame(columns=df.columns)
gaus_df = pd.DataFrame(columns=df.columns)

# Create a new directory to save augmented images
cutout_dir = 'cutout_images'
os.makedirs(cutout_dir, exist_ok=True)

# Create a new directory to save augmented images
elastic_dir = 'elastic_images'
os.makedirs(elastic_dir, exist_ok=True)

# Create a new directory to save augmented images
gaus_dir = 'gaus_images'
os.makedirs(gaus_dir, exist_ok=True)

# Track the selected indices
selected_indices_set = set()

# Augment the data for each label
for label in labels:
    # Get the indices of rows with the current label
    label_indices = df[df[label] == 1].index
    # Convert label indices to a NumPy array
    label_indices = np.array(label_indices)

    # Shuffle the label indices randomly
    np.random.shuffle(label_indices)

    # Get the number of rows to be added for the current label
    rows_to_add = rows_added_per_label[label]

    # Select the indices for augmentation, excluding the ones already selected
    selected_indices = np.random.choice(
    np.setdiff1d(label_indices, list(selected_indices_set)),
    size=min(rows_to_add, len(label_indices)),
    replace=False)

    # Update the set of selected indices
    selected_indices_set.update(selected_indices)

    # Augment the selected rows and update the image paths in the dataframe
    for index in selected_indices:
        image_path = df.at[index, 'image']

        image = load_image_for_augmentation(image_path)

        # Apply cutout
        augmented_image = cutout_image(image)

        # Save the augmented image
        augmented_image_path = os.path.join(cutout_dir, f'cutout_{label}_{index}.png')
        # Save the augmented image
        augmented_image.save(augmented_image_path)

        new_row = df.iloc[index].copy()
        new_row['image'] = augmented_image_path

        # save to cutout dataframe
        cutout_df.loc[len(cutout_df)] = new_row.values

        # Apply elastic transformations
        augmented_image = elastic_transform_image(image)

        # Save the augmented image
        augmented_image_path = os.path.join(elastic_dir, f'elastic_{label}_{index}.png')
        augmented_image.save(augmented_image_path)

        new_row = df.iloc[index].copy()
        new_row['image'] = augmented_image_path

        # save to elastic dataframe
        elastic_transform_df.loc[len(elastic_transform_df)] = new_row.values

        # Apply gaussian noise
        augmented_image = gaussian_noise_image(image)

        # Save the augmented image
        augmented_image_path = os.path.join(gaus_dir, f'gaus_{label}_{index}.png')
        augmented_image.save(augmented_image_path)

        new_row = df.iloc[index].copy()
        new_row['image'] = augmented_image_path

        # save to gaus dataframe
        gaus_df.loc[len(gaus_df)] = new_row.values

# Save the cutout DataFrame to a new CSV file
augmented_dataframe_path = 'cutout_df.csv'
cutout_df.to_csv(augmented_dataframe_path, index=False)

# Save the elastic DataFrame to a new CSV file
augmented_dataframe_path = 'elastic_transform_df.csv'
elastic_transform_df.to_csv(augmented_dataframe_path, index=False)

# Save the elastic DataFrame to a new CSV file
augmented_dataframe_path = 'gaus_df.csv'
gaus_df.to_csv(augmented_dataframe_path, index=False)

# Reset the index
train_df.reset_index(drop=True, inplace=True)
cutout_df.reset_index(drop=True, inplace=True)
elastic_transform_df.reset_index(drop=True, inplace=True)

# Concatenate the two dataframes vertically
combined_df = pd.concat([train_df, cutout_df, elastic_transform_df, gaus_df], axis=0)

# Reset the index of the combined dataframe
combined_df.reset_index(drop=True, inplace=True)

print("Original dataframe length:", len(train_df),
      "\nCutout dataframe length:", len(cutout_df),
      "\nElastic transform dataframe length:", len(elastic_transform_df),
      "\nGaussian noise dataframe length:", len(gaus_df))

print("------------------------------------------------------------")
print("Combined length:", len(combined_df))
print("____________________________________________________________")
print("Added length:", len(combined_df) - len(train_df))

X_train = combined_df[['image']]
y_train = combined_df.drop('image', axis=1)
y_train = y_train.to_numpy()
y_train = y_train.astype(np.float32)

# Convert DataFrame column to NumPy array and change shape
X_train = X_train['image'].values.squeeze()

import tensorflow as tf
AUTO = tf.data.experimental.AUTOTUNE
try:
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
    print('Running on TPU ', tpu.master())
except ValueError:
    tpu = None

if tpu:
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
    strategy = tf.distribute.get_strategy()

print("REPLICAS: ", strategy.num_replicas_in_sync)

if(strategy.num_replicas_in_sync != 1 ):
    print('TPU used')
    BATCH_SIZE = 48 * strategy.num_replicas_in_sync
else:
    BATCH_SIZE = 1

print("Batch size:", BATCH_SIZE)

STEPS_PER_EPOCH = y_train.shape[0] // BATCH_SIZE
VALIDATION_STEPS = y_val.shape[0] // BATCH_SIZE

print ("Steps per epoch: ", STEPS_PER_EPOCH, "\nValidation steps: ", VALIDATION_STEPS)

image_height = 380
image_width = 380

def decode_image(filename, label=None, image_size=(image_height, image_width)):
    bits = tf.io.read_file(filename)
    image = tf.image.decode_jpeg(bits, channels=3)
    image = tf.cast(image, tf.float32) / 255.0
    image = tf.image.resize(image, image_size)

    if label is None:
        return image
    else:
        return image, label

def data_augment(image, label=None):
    image = tf.image.random_flip_left_right(image)
    image = tf.image.random_flip_up_down(image)
    image = tf.image.random_brightness(image, max_delta=0.3)
    image = tf.image.random_hue(image, max_delta=0.3)

    if label is None:
        return image
    else:
        return image, label

train_dataset = (
    tf.data.Dataset.from_tensor_slices((X_train, y_train))
    .map(decode_image, num_parallel_calls = AUTO)
    .map(data_augment, num_parallel_calls = AUTO)
    .repeat()
    .shuffle(256)
    .batch(BATCH_SIZE)
    .prefetch(AUTO)
)

valid_dataset = (
    tf.data.Dataset
    .from_tensor_slices((X_val, y_val))
    .map(decode_image, num_parallel_calls = AUTO)
    .cache()
    .batch(BATCH_SIZE)
    .prefetch(AUTO)
)

tensorboard = TensorBoard(log_dir = 'logs')

checkpoint = ModelCheckpoint("effnet.h5", monitor="val_accuracy",save_best_only=True,
                             mode="auto", verbose=1)
reduce_lr = ReduceLROnPlateau(monitor = 'val_accuracy', factor = 0.1, patience = 2, min_delta = 0.001,
                              mode='auto', verbose=1, min_lr=1e-9)

# Choosing to monitor val_accuracy since it is classification task and accuracy is more important
early_stop=EarlyStopping(monitor='val_accuracy', restore_best_weights= True,
                             patience=7, verbose=1)

filename='history.csv'
history_logger=tf.keras.callbacks.CSVLogger(filename, separator=",", append=True)

callback_options = [tensorboard, checkpoint, reduce_lr, early_stop, history_logger]

effnet = Sequential([efn.EfficientNetB4(input_shape=(image_height, image_width,3), weights='noisy-student', include_top=False)])
# effnet.trainable = False

model = effnet.output
model = tf.keras.layers.GlobalAveragePooling2D()(model)
model = tf.keras.layers.Dense(256,activation='relu')(model)
model = tf.keras.layers.Dense(128,activation='relu')(model)
model = tf.keras.layers.BatchNormalization()(model)
model = tf.keras.layers.Dropout(rate=0.2)(model)
model = tf.keras.layers.Dense(64,activation='relu')(model)
model = tf.keras.layers.BatchNormalization()(model)
model = tf.keras.layers.Dropout(rate=0.2)(model)
model = tf.keras.layers.Dense(6,activation='sigmoid')(model)
model = tf.keras.models.Model(inputs=effnet.input, outputs = model)

model.summary()

import json

# Count the number of occurrences for each label
label_counts = np.sum(y_train, axis=0)

# Calculate the class weights based on label frequencies
total_samples = y_train.shape[0]
class_weights = total_samples / (len(label_counts) * label_counts)

class_weights_dict = dict(zip(range(len(label_counts)), class_weights))

print(class_weights_dict)

model.compile(loss='binary_crossentropy',optimizer = 'Adam', metrics= ['accuracy'])

history = model.fit(train_dataset,
                    epochs=30,
                    callbacks = callback_options,
                    class_weight=class_weights_dict,
                    steps_per_epoch=STEPS_PER_EPOCH,
                    validation_data=valid_dataset,
                    validation_steps=VALIDATION_STEPS)

history_df = pd.DataFrame(history.history)

# Save the training history as a CSV file
history_df.to_csv('training_history.csv', index=False)

plt.figure()
fig,(ax1, ax2)=plt.subplots(1,2,figsize=(19,7))
ax1.plot(history.history['loss'])
ax1.plot(history.history['val_loss'])
ax1.legend(['training','validation'])
ax1.set_title('loss')
ax1.set_xlabel('epoch')

ax2.plot(history.history['accuracy'])
ax2.plot(history.history['val_accuracy'])
ax2.legend(['training','validation'])
ax2.set_title('Acurracy')
ax2.set_xlabel('epoch')


import math
import multiprocessing as mp
import os
import time

import numpy as np
from mathutils import Quaternion
import tqdm

import circleselector.cv_utils as cv_utils
from .datatypes import PointDict, CameraCenters

class Metrics(object):
    """
    class that can :
        - run on prexisting point_dcts list
        - select what errors to calculate
        - calculate all errors in parallel (if required)
    """

    def __init__(self, points=None, point_dcts=None, errors=None, verbose=False, dataset_path=None):
        self.points = points
        self.point_dcts = None
        self.errors = ["endpoint_error",
                       "flatness_error",
                       "perimeter_error",
                       "pairwise_distribution",
                       "ssim",
                       "psnr"]
        if point_dcts is not None:
            self.point_dcts = point_dcts
        if errors is not None:
            self.errors = errors

        self.verbose = verbose
        self.dataset_path = dataset_path
        self.rel_input_image_path = 'Input'
        self.__save_of__ = False

    def run_on_interval(self, interval: tuple) -> dict:

        if self.points is None:
            raise ValueError("need to initialise points before running calculations.")

        point_dict = {"interval": interval}
        sub_arr = np.array(self.points[interval[0]:interval[1]])
        centroid = np.mean(sub_arr,axis=0)
        if "flatness_error" in self.errors:
            points_centered = sub_arr - centroid
            u, sigma, _ = np.linalg.svd(points_centered.T)
            error = min(sigma)
            point_dict["flatness_error"] = error

        std = None
        path_length = None

        if "perimeter_error" in self.errors:
            # calculate radius as average distance of all points to the COM
            radius = np.sum(np.linalg.norm(sub_arr - centroid,axis=1)) / len(sub_arr)
            exp_perimeter = 2 * np.pi * radius
            path_length, std = self.find_path_length(interval)

            point_dict["perimeter_error"] = abs(1 - exp_perimeter / path_length)

        if "pairwise_distribution" in self.errors:
            if std is None:
                _, std = self.find_path_length(interval)

            point_dict["pairwise_distribution"] = std

        if "endpoint_error" in self.errors:
            n = len(sub_arr)

            if path_length is None:
                path_length = self.find_path_length(interval)[0]

            endpoint_gap = np.linalg.norm(sub_arr[0] - sub_arr[-1])
            point_dict["endpoint_error"] = endpoint_gap / path_length

        psnr = None
        savedir = None
        lookatang = None
        if self.__save_of__:
            savedir = os.path.join(self.dataset_path, "CircleFittingResults", "interval_" + str(interval))
            os.makedirs(savedir)
        if "ssim" in self.errors:
            lookatang = self.calculate_angle(centroid, interval)
            ssim, psnr = cv_utils.calculate_metrics(interval, self.dataset_path, savedir, self.rel_input_image_path,
                                                    lookatang)
            point_dict["ssim"] = ssim

        if "psnr" in self.errors:
            if lookatang is None:
                lookatang = self.calculate_angle(centroid, interval)
            if psnr is None:
                _, psnr = cv_utils.calculate_metrics(interval, self.dataset_path, savedir, self.rel_input_image_path,
                                                     lookatang)
            point_dict["psnr"] = psnr

        return point_dict

    def run_on_pair_lst(self, point_dicts) -> PointDict:
        if point_dicts is None and self.point_dcts is None:
            self.init_data()
            point_dicts = self.point_dcts
        for enum, dct in enumerate(point_dicts):
            dct.update(self.run_on_interval(dct["interval"]))
        return point_dicts

    def create_pairs(self):
        n = len(self.points)
        pairs = []
        for i in range(n):
            for j in range(i + 2, n):
                if 10 > abs(i - j):
                    continue
                pairs.append((i, j))
        return pairs

    def init_data(self):
        if self.points is None:
            raise ValueError("need to initialise points before running calculations.")

        if self.point_dcts is None or len(self.point_dcts) == 0:
            pairs = self.create_pairs()
            self.point_dcts = [dict(interval=pair) for pair in pairs]

    def run_map(self,dct):
        """
        Used by imap_async (multiprocessing)
        """
        dct.update(self.run_on_interval(dct["interval"]))
        return dct

    def run(self):
        if self.__save_of__:
            path = os.path.join(self.dataset_path, "CircleFittingResults")
            if not os.path.exists(path):
                os.makedirs(path)
        self.init_data()

        pool = mp.Pool(mp.cpu_count())
        if self.verbose:
            print("time started", time.ctime(time.time()))

        res = []

        # Note: pool.map_async(self.run_map, self.point_dcts), supposedly the most optimised asyncronous mapping,
        # is only marginally faster than the call below.

        # create a progress bar and update it every 10% of the iterations.
        for iter in tqdm.tqdm(pool.imap(self.run_map, self.point_dcts,chunksize=len(self.point_dcts)//10), total=len(self.point_dcts)):
            res.append(iter)

        pool.close()
        pool.join()

        if self.verbose:
            print("time ended", time.ctime(time.time()))
        self.point_dcts = PointDict(res)

    def find_path_length(self, interval: tuple) -> (float, float):
        """
        :param interval: tuple, interval on self.points
        :return: float,float
        """

        # select what set of points we want
        points = self.points[interval[0]:interval[1]]

        diffs = np.diff(points,axis=0).tolist()

        # wrap around
        diffs.append(points[0] - points[-1])

        norms = np.linalg.norm(diffs,axis=1)

        return np.sum(norms), np.std(norms)

    def calculate_angle(self, centroid: np.array, interval: tuple) -> float:
        """
        Calculates the mean of the outward looking angle for the first and last camera in the path.
        """
        angs = []
        orientations = self.points.orientations
        for idx in interval:
            c_prime = self.points[idx] - centroid
            transform = Quaternion(orientations[idx]).to_matrix()
            proj_c = np.dot(transform, c_prime)
            ang = math.atan2(proj_c[0], proj_c[2])
            angs.append(ang)
        return np.mean(angs)

    def calculate_centroid(self, arr: np.array):
        return np.mean(arr, axis=0)

    def __str__(self):
        parameters = []
        parameters.append("errors:         " + str(self.errors))
        parameters.append("dataset_path:   " + str(self.dataset_path))
        parameters.append("len point dict: " + str(len(self.point_dcts)))
        parameters.append("len points:     " + str(len(self.points)))
        return "\n".join(parameters)

def decorator(Metrics):
    def calc(points: CameraCenters,
             point_dcts=None,
             errors=None,
             verbose=False,
             dataset_path=None,
             rel_input_image_path='Input',
             mp=True,
             interval=None,
             save_of=False) -> [dict]:
        """
        Calculates the metrics on the given points. By default will perform all metrics on all the possible intervals
        in the camera path provided. WARNING: this will take a long time.

        :param points: array like. ~1K camera centers in a camera path. NOTE: ssim and psnr calculations will fail
        if camera orientations not provided. (type CameraCenters from loader.load)
        :param point_dcts: listof(dict). If this parameter is passed the
        metrics will be calculated on only the intervals present in this dict.
        :param errors: list. names of metrics that should be calculated.
        :param verbose:
        :param dataset_path: path to rootdir of dataset. eg. path/to/GenoaCathedral
        :param mp: calculated in parallel (using multiprocessing) or not
        :param rel_input_image_path: path to images for cv relative to dataset_path
        :param interval: overrides all other behaviour and will calculate on given interval of points.
        :param: save_of: will save the Optical Flow results to dataset_path/CircleFittingResults
        :return: PointDict
        """

        metrics = Metrics(points, point_dcts, errors, verbose, dataset_path)
        metrics.rel_input_image_path = rel_input_image_path
        if save_of and os.path.exists(str(dataset_path)):
            metrics.__save_of__ = True
        for error in ["ssim", "psnr"]:
            if error in metrics.errors and metrics.dataset_path is None:
                raise Exception(error + " being calculated but dataset_path was not set.")

        if interval is not None:
            return [metrics.run_on_interval(interval)]
        if not mp:
            return PointDict(metrics.run_on_pair_lst(point_dcts))
        metrics.run()
        if verbose:
            print(metrics)
        return PointDict(metrics.point_dcts)

    return calc

calc = decorator(Metrics)


# 丙组 2020年3月
# 1.
n = int(input())
m = n%5
if n%5 == 4 or n%5 == 0:
    print('Lying')
else:
	print('Fishing')

# 2.
n = list(str(input()))
n[0], n[3] = n[3], n[0]
n[2], n[1] = n[1], n[2]
r1 = 9-int(n[0])
r2 = 9-int(n[1])
r3 = 9-int(n[2])
r4 = 9-int(n[3])
number = str(r1)+str(r2)+str(r3)+str(r4)
print(number)

#3.
# num = int(input())
import sys
import math
def isPrime(n):
    if (n==1) or (n==2) or (n==3) or (n==5) or (n==7):
        return True

    if (n%2==0) or (n%3==0) or (n%5==0) or (n%7==0):
        return False
    else:
        for i in range(2,int(math.sqrt(n)+1),1):
            if n%i == 0:
                return False
            else:
                return True

def main():
    t=0
    s,e = map(int, sys.stdin.readline().split())
    for r in range(s,e+1):
        if isPrime(r) and isPrime(int(r%10)):
            print(r)
            t += 1
        else: pass
    if t==0:
        print('None')

if __name__ == "__main__":
    main()

# 3. sample answer from https:#iai.sh.cn/contribution/21
import sys
a,b = map(int, sys.stdin.readline().split())
def prime(n):
	if n == 2:
		return True
	elif n == 1 or n%2 == 0:
		return False
	else: pass

	for j in range(2,int(pow(n,0.5))+2):
		if n%j == 0:
			return False
	else:
		return True
count = 0
for i in range(a,b+1):
	if prime(i) and prime(i%10):
		print(i)
		count += 1
if count == 0:
	print(None)

# 4.
i = int(input())
li = list(map(int,str(input()).split()))
from functools import reduce
n = reduce(lambda a,b:a*b,li)
for i in range(len(li)):
    print(int((n/int(li[i]))%10000))
i = int(input())
li = str(input()).split()

# 4. sample answer from https:#iai.sh.cn/contribution/41
n = int(input())
a = list(map(int,str(input()).split()))
l = [0]*1000100
r = [0]*1000100
l[0] = r[n + 1] = 1
for i in range(1,n):
    l[i] = l[i - 1] * a[i] % 10000
for i in range(n,0,-1):
    r[i] = r[i + 1] * a[i] % 10000
for i in range(1,n):
    print(l[i - 1] * r[i + 1] % 10000)

# 5.
import sys
def ts(tq):
    tq = list(map(int,tq.split(':')))
    h,m = tq[0],tq[1]
    ts = h*60+m
    return ts
n = int(input())
inputs = sys.stdin.readlines()
result = []
for n,t in enumerate(inputs):
    t = t.strip().split()
    st,dt = ts(t[0]),ts(t[1])
    result.append([st,st+dt,n+1])
result.sort()
for x in result: print(x[2])

# 5. sample ansser from https://iai.sh.cn/contribution/33
n = int(input())
t = [[0,0,0] for _ in range(1005)]# 出发时间,路程时间,队伍编号
tmp = t[::]
'''
int depart;	//	出发时间 [0]
int arrival;//	路程时间 [1]
int x;		//	队伍编号 [2]
'''
for i in range(1,n):
    #输入出发时间，并转换为整数形式
    hour,mintue = list(map(int,input().split(':')))
    t[i][0] = hour * 60 + mintue
    #输入路程时间，并转换为到达时间的整数形式
    t[i][1] = t[i][0] + hour * 60 + mintue
    #变量 i 的值就是队伍的编号
    t[i][2] = i

	#用选择排序
for i in range(1,n):
    k = i
    for j in range(i+1,n+1):
        #首先挑选到达时间靠前的队伍
        if t[j][1] < t[k][1]: k = j
        elif t[j][1] == t[k][1]:
            #在到达时间相同的情况下，挑选出发时间靠前的队伍
            if t[j][0] < t[k][0]: k = j
            elif t[j][0] == t[k][0]:
                #如果出发时间仍然相同，则编号较小的队伍排在前列
                if t[j][2] < t[k][2]: k = j
    #交换
    tmp = t[i]
    t[i] = t[k]
    t[k] = tmp

#输出排序后的队伍编号
for i in range(1,n+1):
    print(t[i][2])

YACS Python Solution/iai.sh 2020.3 丙组.py


from copy import deepcopy
from typing import List

import time

def atualiza_nascimento_timestamp_para_date_string(data: List[dict]) -> List[dict]:
    """
    This function takes a object dict type, unpacks it,
    transforms the nasicmento attribute, which is in type timestamp,
    and transforms it into a calendar date each person listed "
    """

    try:
        data = deepcopy(data)

        for pessoa in data:

            birth_timestamp = pessoa['nascimento']
            birth_converted = time.strftime('%d/%m/%Y', (time.localtime  (birth_timestamp)))

            pessoa['nascimento'] = birth_converted

        return data
    except:
        return print("There was a problem executing the 'atualiza_nascimento_timestamp_para_date_string' function from the atualizacoes file.  ")

def atualiza_altura_centimetro_para_metro(data: List[dict]) -> List[dict]:
    """
    This function receives a object dict type, unzips it, processes the height,
     which is in centimeters and transforms it into meters, for each person listed "
    """
    try:
        data = deepcopy(data)

        for pessoa in data:

            meter_in_centimetres = 100

            altura_converted = pessoa['altura'] / meter_in_centimetres

            pessoa['altura'] = altura_converted

        return data
    except:
        return print("There was a problem executing the 'atualiza_altura_centimetro_para_metro' function from the atualizacoes file.  ")


import imageio
from enum import Enum
from io import BytesIO
from fontTools.ttLib import TTFont
from typing import List, Tuple, Union
from typing_extensions import Literal
from PIL.Image import Image as IMG
from PIL.ImageFont import FreeTypeFont
from PIL import Image, ImageFont, ImageDraw
from emoji.unicode_codes import UNICODE_EMOJI

from .download import get_image, get_font, get_thumb

OVER_LENGTH_MSG = "文字长度过长，请适当缩减"
BREAK_LINE_MSG = "文字长度过长，请手动换行或适当缩减"
DEFAULT_FONT = "SourceHanSansSC-Regular.otf"
EMOJI_FONT = "NotoColorEmoji.ttf"

def save_jpg(frame: IMG) -> BytesIO:
    output = BytesIO()
    frame = frame.convert("RGB")
    frame.save(output, format="jpeg")
    return output

def save_png(frame: IMG) -> BytesIO:
    output = BytesIO()
    frame = frame.convert("RGBA")
    frame.save(output, format="png")
    return output

def save_gif(frames: List[IMG], duration: float) -> BytesIO:
    output = BytesIO()
    imageio.mimsave(output, frames, format="gif", duration=duration)
    return output

async def load_image(name: str) -> IMG:
    image = await get_image(name)
    return Image.open(BytesIO(image))

async def load_font(name: str, fontsize: int) -> FreeTypeFont:
    font = await get_font(name)
    return ImageFont.truetype(BytesIO(font), fontsize, encoding="utf-8")

async def load_thumb(name: str) -> IMG:
    image = await get_thumb(name)
    return Image.open(BytesIO(image))

def wrap_text(
    text: str, font: FreeTypeFont, max_width: float, stroke_width: int = 0
) -> List[str]:
    line = ""
    lines = []
    for t in text:
        if t == "\n":
            lines.append(line)
            line = ""
        elif font.getsize(line + t, stroke_width=stroke_width)[0] > max_width:
            lines.append(line)
            line = t
        else:
            line += t
    lines.append(line)
    return lines

async def fit_font_size(
    text: str,
    max_width: float,
    max_height: float,
    fontname: str,
    max_fontsize: int,
    min_fontsize: int,
    stroke_ratio: float = 0,
) -> int:
    fontsize = max_fontsize
    while True:
        font = await load_font(fontname, fontsize)
        width, height = font.getsize_multiline(
            text, stroke_width=int(fontsize * stroke_ratio)
        )
        if width > max_width or height > max_height:
            fontsize -= 1
        else:
            return fontsize
        if fontsize < min_fontsize:
            return 0

async def draw_text(
    img: IMG,
    pos: Tuple[float, float],
    text: str,
    font: FreeTypeFont,
    fill=None,
    spacing: int = 4,
    align: Literal["left", "right", "center"] = "left",
    stroke_width: int = 0,
    stroke_fill=None,
):
    if not text:
        return
    lines = text.strip().split("\n")
    draw = ImageDraw.Draw(img)
    max_w = font.getsize_multiline(text)[0]
    current_x, current_y = pos

    emoji_font_file = BytesIO(await get_font(EMOJI_FONT))
    emoji_font = ImageFont.truetype(emoji_font_file, 109, encoding="utf-8")
    emoji_ttf = TTFont(emoji_font_file)

    def has_emoji(emoji: str):
        for table in emoji_ttf["cmap"].tables:  # type: ignore
            if ord(emoji) in table.cmap.keys():
                return True
        return False

    for line in lines:
        line_w = font.getsize(line)[0]
        dw = max_w - line_w
        current_x = pos[0] + stroke_width
        if align == "center":
            current_x += dw / 2
        elif align == "right":
            current_x += dw

        for char in line:
            if char in UNICODE_EMOJI["en"] and has_emoji(char):
                emoji_img = Image.new("RGBA", (150, 150))
                emoji_draw = ImageDraw.Draw(emoji_img)
                emoji_draw.text((0, 0), char, font=emoji_font, embedded_color=True)
                emoji_img = emoji_img.crop(emoji_font.getbbox(char))
                emoji_x, emoji_y = font.getsize(char)
                emoji_img = fit_size(
                    emoji_img, (emoji_x, emoji_y), FitSizeMode.INSIDE, FitSizeDir.SOUTH
                )
                img.paste(emoji_img, (int(current_x), int(current_y)), emoji_img)
                current_x += emoji_x
            else:
                draw.text(
                    (current_x, current_y),
                    char,
                    font=font,
                    fill=fill,
                    stroke_width=stroke_width,
                    stroke_fill=stroke_fill,
                )
                current_x += font.getsize(char)[0]
        current_y += font.getsize("A", stroke_width=stroke_width)[1] + spacing

# 适应大小模式
class FitSizeMode(Enum):
    INSIDE = 0  # 图片必须在指定的大小范围内
    INCLUDE = 1  # 图片必须包括指定的大小范围

# 适应大小方向
class FitSizeDir(Enum):
    CENTER = 0  # 居中
    NORTH = 1
    SOUTH = 2
    WEST = 3
    EAST = 4
    NORTHWEST = 5
    NORTHEAST = 6
    SOUTHWEST = 7
    SOUTHEAST = 8

def limit_size(
    img: IMG, size: Tuple[int, int], mode: FitSizeMode = FitSizeMode.INCLUDE
) -> IMG:
    """
    调整图片到指定的大小，不改变长宽比（即返回的图片大小不一定是指定的size）
    :params
      * ``img``: 待调整的图片
      * ``size``: 期望图片大小
      * ``mode``: FitSizeMode.INSIDE 表示图片必须在指定的大小范围内；FitSizeMode.INCLUDE 表示图片必须包括指定的大小范围
    """
    w, h = size
    img_w, img_h = img.size
    if mode == FitSizeMode.INSIDE:
        ratio = min(w / img_w, h / img_h)
    elif mode == FitSizeMode.INCLUDE:
        ratio = max(w / img_w, h / img_h)
    img_w = int(img_w * ratio)
    img_h = int(img_h * ratio)
    return img.resize((img_w, img_h), Image.ANTIALIAS)

def cut_size(
    img: IMG,
    size: Tuple[int, int],
    direction: FitSizeDir = FitSizeDir.CENTER,
    bg_color: Union[str, float, Tuple[float, ...]] = (255, 255, 255, 0),
) -> IMG:
    """
    裁剪图片到指定的大小，超出部分裁剪，不足部分设为指定颜色
    :params
      * ``img``: 待调整的图片
      * ``size``: 期望图片大小
      * ``direction``: 调整图片大小时图片的方位；默认为居中 FitSizeDir.CENTER
      * ``bg_color``: FitSizeMode.INSIDE 时的背景颜色
    """
    w, h = size
    img_w, img_h = img.size
    x = int((w - img_w) / 2)
    y = int((h - img_h) / 2)
    if direction in [FitSizeDir.NORTH, FitSizeDir.NORTHWEST, FitSizeDir.NORTHEAST]:
        y = 0
    elif direction in [FitSizeDir.SOUTH, FitSizeDir.SOUTHWEST, FitSizeDir.SOUTHEAST]:
        y = h - img_h
    if direction in [FitSizeDir.WEST, FitSizeDir.NORTHWEST, FitSizeDir.SOUTHWEST]:
        x = 0
    elif direction in [FitSizeDir.EAST, FitSizeDir.NORTHEAST, FitSizeDir.SOUTHEAST]:
        x = w - img_w
    result = Image.new("RGBA", size, bg_color)
    result.paste(img, (x, y))
    return result

def fit_size(
    img: IMG,
    size: Tuple[int, int],
    mode: FitSizeMode = FitSizeMode.INCLUDE,
    direction: FitSizeDir = FitSizeDir.CENTER,
    bg_color: Union[str, float, Tuple[float, ...]] = (255, 255, 255, 0),
) -> IMG:
    """
    调整图片到指定的大小，超出部分裁剪，不足部分设为指定颜色
    :params
      * ``img``: 待调整的图片
      * ``size``: 期望图片大小
      * ``mode``: FitSizeMode.INSIDE 表示图片必须在指定的大小范围内，不足部分设为指定颜色；FitSizeMode.INCLUDE 表示图片必须包括指定的大小范围，超出部分裁剪
      * ``direction``: 调整图片大小时图片的方位；默认为居中 FitSizeDir.CENTER
      * ``bg_color``: FitSizeMode.INSIDE 时的背景颜色
    """
    return cut_size(limit_size(img, size, mode), size, direction, bg_color)


import pefile
import os
import sys

def calc(file_name, physical_addr):
    pe_load_file = pefile.PE(file_name)
    pe_load_file.parse_data_directories()
    #find physical_addr in which section
    for i in pe_load_file.sections:
        if (physical_addr >= i.PointerToRawData and
            physical_addr <= i.PointerToRawData +i.SizeOfRawData):
            return physical_addr - i.PointerToRawData + i.VirtualAddress
    raise Exception("Bad Para.");

def main():
    if len(sys.argv) != 3:
        print ("pe_physical covert pe file physical address to RVA\nusage:\n pe_physical exe_file physical_addr")
        return
    if sys.argv[2][:2] != "0x":
        print ("physical_addr must be a hex format, like \"0x4000\"")
        return
    target_addr = int(sys.argv[2], 0)
    try:
        target_file_stat = os.stat(sys.argv[1]);
        if (target_file_stat.st_size < target_addr):
            print ("physical_addr too big to file " + sys.argv[1] + ".")
            return
    except WindowsError:
        print ("can't open file "+sys.argv[1]+".")
        return
    print("RVA: " + str(hex(calc(sys.argv[1], target_addr))));
if __name__=="__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import twint
from urlextract import URLExtract
import newspaper
import validators
import hashlib
import base64
import gzip
import simplejson as json
import redis
import sys
import datetime
import configparser
import argparse
import requests
from urllib.parse import urlparse
import signal
from pymisp import MISPObject

def jsonclean(o):
    if isinstance(o, datetime.datetime):
        return o.__str__()

uuid = "aae656ec-d446-4a21-acf0-c88d4e09d506"
ailfeedertype = "ail_feeder_twitter"
ailurlextract = "ail_feeder_urlextract"
# config reader

config = configparser.ConfigParser()
config.read('../etc/ail-feeder-twitter.cfg')

if 'general' in config:
    uuid = config['general']['uuid']
    tweet_limit = config['general']['tweet_limit']
else:
    uuid = "aae656ec-ffff-4a21-acf0-c88d4e09d506"

if 'redis' in config:
    r = redis.Redis(
        host=config['redis']['host'],
        port=config['redis']['port'],
        db=config['redis']['db'],
    )
else:
    r = redis.Redis(host='localhost', port=6379, db=0)

if 'cache' in config:
    cache_expire = config['cache']['expire']
else:
    cache_expire = 86400

def ail_publish(url=config['ail']['url'], apikey=config['ail']['apikey'], data=None):
    response = requests.post(
        url,
        headers={'Content-Type': 'application/json', 'Authorization': apikey},
        data=data,
        verify=False,
    )
    return response

# arguments parsing

parser = argparse.ArgumentParser()
parser.add_argument("query", help="query to search on Twitter to feed AIL")
parser.add_argument("--verbose", help="verbose output", action="store_true")
parser.add_argument("--nocache", help="disable cache", action="store_true")
parser.add_argument(
    "--tweetlimit",
    help="maximum number of tweet to fetch",
    type=int,
    default=tweet_limit,
)
parser.add_argument("--disable-push", help="disable AIL API push", action="store_true")
parser.add_argument(
    "--export-misp",
    help="Export in MISP object format instead of AIL JSON",
    action="store_true",
    default=False,
)

args = parser.parse_args()

c = twint.Config()
c.Search = args.query
c.Limit = args.tweetlimit
c.Store_object = True
c.Hide_output = True
c.Filter_retweets = True

signal.alarm(10)
try:
    twint.run.Search(c)
except TimeoutException:
    print("Timeout reached for search: {}".format(c), file=sys.stderr)
    sys.exit(1)
else:
    signal.alarm(0)

tweets = twint.output.tweets_list

extractor = URLExtract()

for tweet in tweets:
    urls = extractor.find_urls(tweet.tweet)
    if r.exists("c:{}".format(tweet.id)):
        if args.verbose:
            print("Tweet {} already processed".format(tweet.id), file=sys.stderr)
        if not args.nocache:
            continue
    else:
        r.set("c:{}".format(tweet.id), tweet.tweet)
        r.expire("c:{}".format(tweet.id), cache_expire)

    output_tweet = {}
    output_tweet['source'] = ailfeedertype
    output_tweet['source-uuid'] = uuid
    output_tweet['default-encoding'] = 'UTF-8'
    output_tweet['meta'] = {}
    output_tweet['meta']['twitter:tweet_id'] = tweet.id
    output_tweet['meta']['twitter:user_id'] = tweet.user_id
    output_tweet['meta']['twitter:id'] = tweet.username
    output_tweet['meta']['twitter:name'] = tweet.name
    output_tweet['meta']['twitter:link'] = tweet.link
    output_tweet['meta']['twitter:place'] = tweet.place
    output_tweet['meta']['twitter:geo'] = tweet.geo
    output_tweet['meta']['twitter:near'] = tweet.near
    output_tweet['meta']['twitter:urls'] = tweet.urls
    output_tweet['meta']['twitter:replies_count'] = tweet.replies_count
    output_tweet['meta']['twitter:retweets_count'] = tweet.retweets_count
    output_tweet['meta']['twitter:likes_count'] = tweet.likes_count
    output_tweet['meta']['twitter:source'] = tweet.source
    output_tweet['meta']['twitter:datestamp'] = tweet.datestamp
    output_tweet['meta']['twitter:timestamp'] = tweet.timestamp
    output_tweet['meta']['twitter:timezone'] = tweet.timezone

    m = hashlib.sha256()
    m.update(tweet.tweet.encode('utf-8'))
    output_tweet['data-sha256'] = m.hexdigest()
    output_tweet['data'] = base64.b64encode(gzip.compress(tweet.tweet.encode()))
    if not args.export_misp:
        print(json.dumps(output_tweet, indent=4, sort_keys=True, default=jsonclean))
    elif args.export_misp:
        twitter_object = MISPObject('twitter-post')
        twitter_object.add_attribute('post-id', **{'type': 'text', 'value': tweet.id})
        twitter_object.add_attribute(
            'name', **{'type': 'text', 'value': tweet.username}
        )
        twitter_object.add_attribute(
            'post', **{'type': 'text', 'value': tweet.tweet.encode()}
        )
        twitter_object.add_attribute('link', **{'type': 'text', 'value': tweet.link})
        twitter_object.add_attribute('geo', **{'type': 'text', 'value': tweet.geo})
        for url in tweet.urls:
            twitter_object.add_attribute(
                'embedded-link', **{'type': 'text', 'value': url}
            )
        twitter_object.add_attribute('post-id', **{'type': 'text', 'value': tweet.id})
        setattr(twitter_object, 'first_seen', tweet.datestamp)
        t = json.loads(twitter_object.to_json())
        output = {}
        output['Object'] = []
        output['Object'].append(t)
        print(json.dumps(output))
    if not args.disable_push:
        ail_publish(
            data=json.dumps(output_tweet, indent=4, sort_keys=True, default=jsonclean)
        )

    for url in urls:
        output = {}
        output['source'] = ailurlextract
        output['source-uuid'] = uuid
        output['default-encoding'] = 'UTF-8'
        output['meta'] = {}
        output['meta']['parent:twitter:tweet_id'] = tweet.id
        surl = url.split()[0]
        output['meta']['twitter:url-extracted'] = surl
        u = urlparse(surl)
        if u.hostname is not None:
            if "twitter.com" in u.hostname:
                continue
        if not validators.url(surl):
            continue
        if args.verbose:
            print("Downloading and parsing {}".format(surl), file=sys.stderr)
        signal.alarm(10)
        try:
            article = newspaper.Article(surl)
        except TimeoutException:
            print("Timeout reached for {}".format(surl), file=sys.stderr)
            continue
        else:
            signal.alarm(0)

        if r.exists("cu:{}".format(base64.b64encode(surl.encode()))):
            if args.verbose:
                print("URL {} already processed".format(surl), file=sys.stderr)
            if not args.nocache:
                continue
        else:
            r.set("cu:{}".format(base64.b64encode(surl.encode())), tweet.tweet)
            r.expire("cu:{}".format(base64.b64encode(surl.encode())), cache_expire)

        try:
            article.download()
            article.parse()
        except:
            if args.verbose:
                print("Unable to download/parse {}".format(surl), file=sys.stderr)
            continue
        else:
            signal.alarm(0)
        m = hashlib.sha256()
        m.update(article.html.encode('utf-8'))
        output['data-sha256'] = m.hexdigest()
        output['data'] = base64.b64encode(gzip.compress(article.html.encode()))
        output['meta']['newspaper:text'] = article.text
        output['meta']['newspaper:authors'] = article.authors
        try:
            article.nlp()
        except:
            if args.verbose:
                print("Unable to nlp {}".format(surl), file=sys.stderr)
            continue
        output['meta']['newspaper:keywords'] = article.keywords
        output['meta']['newspaper:publish_date'] = article.publish_date
        output['meta']['newspaper:top_image'] = article.top_image
        output['meta']['newspaper:movies'] = article.movies
        # MISP export should be added for the crawled items
        if not args.export_misp:
            print(json.dumps(output, indent=4, sort_keys=True, default=jsonclean))
        if not args.disable_push:
            ail_ret = ail_publish(
                data=json.dumps(output, indent=4, sort_keys=True, default=jsonclean)
            )
            print(f"AIL publish HTTP return code {ail_ret.status_code}")


import csv
import matplotlib.pyplot as plt

#----- Define variables -----
model_name = "R_2agb_moreprofiles"
folder_output = "Models/" + model_name
#column_name = "surf_avg_omega_div_omega_crit"
#column_name = "star_mass"
column_name = "pp"
x_column_name = "model_number"

#----- Preferences -----
graph_type = "normal" # "normal", "semilogx", "semilogy", "loglog"
use_custom_ylabel = False
use_custom_xlabel = False
set_title = True
use_custom_xlim = False
use_custom_ylim = False
keep_figure_open = True
append_to_filename = False

ylabel = "conv mx2 bot"
xlabel = "model number"
title = r"Test Suite: conserve\_angular\_momentum"
custom_xlim = [ 3.7, 3.68 ]
custom_ylim = [ -0.75, -0.5 ]
append_to_filename_text = "One-At-Time"

#----- Don't change (constants, or to be populated by the code) -----
folder_csv = "Models/" + model_name + "/CSV"
output_filename = "History-" + column_name + "-vs-" + x_column_name
cols = []
x_data = []
history_data = []

if append_to_filename:
	output_filename += "-" + append_to_filename_text

#----- Read history.csv and extract values -----
with open( folder_csv + "/history.csv" ) as data_file:
	data = csv.reader( data_file )
	headers = next( data )

	#--- Check age and column are included -----
	for par in [ x_column_name, column_name ]:
		if not par in headers:
			raise ValueError( "%s not in history.csv." %par )
		else:
			cols.append( headers.index( par ) )

	#--- Populate lists ---
	for row in data:
		x_data.append( float( row[ cols[0] ] ) )
		history_data.append( float( row[ cols[1] ] ) )

#----- Plot graph -----
plt.rc( "text", usetex=True )
plt.rcParams.update({ "font.size":14 })

fig = plt.figure( figsize=(8,5) )
ax = fig.add_subplot( 111 )
ax.set_xlabel( xlabel if use_custom_xlabel else x_column_name.replace("_"," ") )
ax.set_ylabel( ylabel if use_custom_ylabel else column_name.replace("_"," ") )

if set_title:
	ax.set_title( title )
if use_custom_xlim:
	ax.set_xlim( custom_xlim[0], custom_xlim[1] )
if use_custom_ylim:
	ax.set_ylim( custom_ylim[0], custom_ylim[1] )

if graph_type == "normal":
	ax.plot( x_data, history_data, color="b", ls="-" )
elif graph_type == "semilogx":
	ax.semilogx( x_data, history_data, color="b", ls="-" )
elif graph_type == "semilogy":
	ax.semilogy( x_data, history_data, color="b", ls="-" )
elif graph_type == "loglog":
	ax.loglog( x_data, history_data, color="b", ls="-" )
else:
	raise ValueError( "graph_type '%s' not understood. Only 'normal', 'semilogx', 'semilogy', 'loglog' are accepted." %graph_type )

fig.tight_layout()
plt.savefig( folder_output + "/" + output_filename )

if not keep_figure_open:
	plt.close()


from django.shortcuts import render, get_object_or_404, reverse
from django.views import generic, View
from django.views.generic import ListView, CreateView, UpdateView, DeleteView
from django.views.generic import TemplateView
from django.views.generic.edit import FormView
from django.http import HttpResponseRedirect
from .models import Post
from django.contrib.auth.mixins import UserPassesTestMixin
from django.contrib.messages.views import SuccessMessageMixin
from django.contrib import messages
from .forms import CommentForm
from .forms import PostForm, AddressFilterForm

class PostList(generic.ListView):
    """
    This class inherits from generic view
    filter posts by status
    order them by creation time and date
    and paginates the post page
    """
    model = Post
    queryset = Post.objects.filter(status=1).order_by('-created_on')
    template_name = 'index.html'
    paginate_by = 6

class PostDetail(View):

    def get(self, request, slug, *args, **kwargs):
        """
        This function gets the food item posts by slug
        filters out the active ones
        post them and their associated comments on the
        postdetail page
        """
        queryset = Post.objects.filter(status=1)
        post = get_object_or_404(queryset, slug=slug)
        comments = post.comments.filter(approved=True).order_by('created_on')
        reserved = True
        while post.reserve.filter(id=request.user.id).exists():
            reserved = False
            break

        return render(
            request,
            "post_detail.html",
            {
                "post": post,
                "comments": comments,
                "reserved": reserved,
                "commented": False,
                "comment_form": CommentForm()
            },
        )

    def post(self, request, slug, *args, **kwargs):
        """
        this function gets data from the user form
        filters approved comments
        checks for user form validity
        asigns comment to its respective post
        and save the comment
        or return an empty field if invalid
        """
        queryset = Post.objects.filter(status=1)
        post = get_object_or_404(queryset, slug=slug)
        comments = post.comments.filter(approved=True).order_by("-created_on")
        comment_form = CommentForm(data=request.POST)
        if comment_form.is_valid():
            comment_form.instance.email = request.user.email
            comment_form.instance.name = request.user.username
            comment = comment_form.save(commit=False)
            comment.post = post
            comment.save()
        else:
            comment_form = CommentForm()

        return render(
            request,
            "post_detail.html",
            {
                "post": post,
                "comments": comments,
                "commented": True,
                "comment_form": comment_form,
            },
        )

class AddPost(CreateView):
    """user post view """
    model = Post
    form_class = PostForm
    template_name = 'add_post.html'
    success_url = '/'

    def form_valid(self, form):
        """
        form for post submission
        the function checks if the form is valid
        if it is, it saves the input data
        and alerts a success message
        """
        form.instance.author = self.request.user
        messages.success(
            self.request,
            f'Post submitted successfully'
        )

        return super(AddPost, self).form_valid(form)

class EditPost(UserPassesTestMixin, SuccessMessageMixin, UpdateView):
    """user edit post view """
    model = Post
    form_class = PostForm
    template_name = 'edit_post.html'
    success_url = '/'

    def form_valid(self, form):
        """
        form for post update
        the function checks if the form is valid
        if it is, it saves the updated data
        and alerts a success message
        """
        form.instance.author = self.request.user
        post = self.get_object()
        messages.success(
            self.request,
            f'{post.title} successfully updated'
        )

        return super(EditPost, self).form_valid(form)

    def test_func(self):
        """
        ensures user is author of the post in question
        if not user cannot edit the post
        and a 403 error is thrown
        """
        if self.request.user.is_staff:
            return True
        else:
            return self.request.user == self.get_object().author

class DeletePost(SuccessMessageMixin, UserPassesTestMixin, DeleteView):
    """user delete post view """
    model = Post
    template_name = 'delete_post.html'
    success_url = '/'

    def delete(self, request, *args, **kwargs):
        """
        references the post to be deleted
        and alerts a success message upon deletion
        """
        post = self.get_object()
        messages.success(
            self.request,
            f'{post.title} successfully deleted'
        )
        return super(DeletePost, self).delete(request, *args, **kwargs)

    def test_func(self):
        """
        ensures user is author of the post in question
        if not user cannot delete the post
        and a 403 error is thrown
        """
        if self.request.user.is_staff:
            return True
        else:
            return self.request.user == self.get_object().author

class Reserve_Food_Item(View):
    """food item reservation view"""

    def post(self, request, slug):
        """
        this function first checks if the user is the
        author of the respective post
        then, checks if a post item is reserved
        if it is, the reserved status is removed
        if not, then reserved it added
        and returns back to postdetail page
        """
        post = get_object_or_404(Post, slug=slug)
        if request.user.id == post.author.id:
            if post.reserve.filter(id=request.user.id).exists():
                post.reserve.remove(request.user)
            else:
                post.reserve.add(request.user)
                post.save()

        return HttpResponseRedirect(reverse('post_detail', args=[slug]))

class AboutView(TemplateView):
    """about view"""
    template_name = "about.html"

def filter_posts(request):
    """filter view"""
    query = request.GET.get('search')

    if query:
        posts = Post.objects.filter(address__icontains=query)
    else:
        posts = Post.objects.all()

    return render(
        request, 'filter_posts.html', {'posts': posts})


import tensorflow as tf

from parameterized import parameterized

from opennmt.layers import position

class _DummyPositionEncoder(position.PositionEncoder):
    """Encoder that simply forwards the position indices."""

    def _encode(self, positions, depth):
        positions = tf.expand_dims(positions, 2)
        positions = tf.tile(positions, [1, 1, depth])
        return tf.cast(positions, self.dtype)

class PositionTest(tf.test.TestCase):
    def testApplyOneEncoding(self):
        encoder = _DummyPositionEncoder()
        inputs = tf.zeros([2, 1, 3])
        outputs = encoder(inputs, 2)
        outputs = self.evaluate(outputs)
        self.assertAllEqual(outputs, [[[2, 2, 2]], [[2, 2, 2]]])

    def testApplyPositionEncoding(self):
        encoder = _DummyPositionEncoder()
        inputs = tf.zeros([2, 4, 3])
        outputs = encoder(inputs)
        outputs = self.evaluate(outputs)
        self.assertAllEqual(
            outputs,
            [
                [[1, 1, 1], [2, 2, 2], [3, 3, 3], [4, 4, 4]],
                [[1, 1, 1], [2, 2, 2], [3, 3, 3], [4, 4, 4]],
            ],
        )

    def _testSinusoidalPositionEncoder(self, depth, dtype=tf.float32):
        encoder = position.SinusoidalPositionEncoder(dtype=dtype)
        inputs = tf.zeros([2, 6, depth], dtype=dtype)
        outputs = encoder(inputs)
        self.assertEqual(dtype, outputs.dtype.base_dtype)
        outputs = self.evaluate(outputs)
        self.assertAllEqual([2, 6, depth], outputs.shape)

    def testSinusoidalPositionEncoder(self):
        self._testSinusoidalPositionEncoder(10)

    def testSinusoidalPositionEncoderFloat16(self):
        self._testSinusoidalPositionEncoder(10, dtype=tf.float16)

    def testSinusoidalPositionEncoderInvalidDepth(self):
        with self.assertRaises(ValueError):
            self._testSinusoidalPositionEncoder(5)

    @parameterized.expand([[tf.float32], [tf.float16]])
    def testPositionEmbedder(self, dtype):
        encoder = position.PositionEmbedder(dtype=dtype)
        inputs = tf.zeros([3, 5, 10], dtype=dtype)
        outputs = encoder(inputs)
        self.assertEqual(outputs.dtype, dtype)
        self.assertEqual(encoder.embedding.dtype.base_dtype, dtype)
        outputs = self.evaluate(outputs)
        self.assertAllEqual(outputs.shape, [3, 5, 10])

if __name__ == "__main__":
    tf.test.main()


from django import forms

from users.models import Newsletter, Profile

class NewsletterForm(forms.ModelForm):
    class Meta:
        model = Newsletter
        fields = "__all__"

    def __init__(self, *args, **kwargs):
        super(NewsletterForm, self).__init__(*args, **kwargs)

        self.fields['email'].widget.attrs.update({'class': 'form-control'})

class ProfileForm(forms.ModelForm):
    class Meta:
        model = Profile
        fields = "__all__"
        exclude = ["user"]

    def __init__(self, *args, **kwargs):
        super(ProfileForm, self).__init__(*args, **kwargs)

        for name, field in self.fields.items():
            field.widget.attrs.update({'class': 'form-control form-label'})


import scrapy
import pandas as pd
import sqlite3

i = 0

class BooksSpider(scrapy.Spider):
    name = "books"
    allowed_domains = ["books.toscrape.com"]
    start_urls = ["https://books.toscrape.com"]

    def parse(self, response):

        for book_link in response.css("ol.row article.product_pod a"):
            yield response.follow(book_link.attrib["href"] , callback=self.extract_book)

        for next_page_link in response.css("ul.pager li.next a"):
            yield response.follow(next_page_link.attrib["href"] , callback=self.parse)

    def extract_book(self, response):
        title = response.css("div.product_main h1::text").get()
        price = response.css("div.product_main p::text").get()
        description = response.css("#product_description + p::text").get()
        table_info = response.css("table.table").get()
        book_info = pd.read_html(table_info)[0].set_index(0).to_dict()[1]
        book_info["title"] = title
        book_info["categories"] = response.css("div.page_inner ul.breadcrumb li a ::text").getall()
        book_info['price'] = price
        book_info['description'] = description
        return book_info


from typing import Optional

import httpx

# -----------------------------------------------------------------------------
# Public Imports
# -----------------------------------------------------------------------------

from aioeapi import Device as DeviceEAPI
from aioeapi.config_session import SessionConfig

from netcad.device import Device
from netcam.dcfg import AsyncDeviceConfigurable
from netcad.logger import get_logger

# -----------------------------------------------------------------------------
# Privae Imports
# -----------------------------------------------------------------------------

from netcam_aioeos.eos_plugin_globals import g_eos

# -----------------------------------------------------------------------------
# Exports
# -----------------------------------------------------------------------------

_Caps = AsyncDeviceConfigurable.Capabilities

class EOSDeviceConfigurable(AsyncDeviceConfigurable):
    """
    This class provides the Arista EOS device-under-test plugin for directly
    communicating with the device via the EAPI interface.  The underpinning
    transport is using asyncio.  Refer to the `aioeapi` distribution for further
    details.

    Attributes
    ----------
    eapi: aioeapi.Device
        The asyncio driver instance used to communicate with the EOS eAPI.
    """

    DEFAULT_CAPABILITIES = _Caps.diff | _Caps.rollback | _Caps.check | _Caps.replace

    def __init__(self, *, device: Device, **_kwargs):
        """
        Initialize the instance with eAPI
        Parameters
        ----------
        device:
            The netcad device instance from the design.
        """
        super().__init__(device=device)
        self._scp_creds = g_eos.scp_creds
        self._dev_fs = "flash:"
        self.eapi = DeviceEAPI(
            host=device.name, auth=g_eos.basic_auth_rw, timeout=g_eos.config.timeout
        )
        self.sesson_config: SessionConfig | None = None
        self.capabilities = self.DEFAULT_CAPABILITIES

    def _set_config_id(self, name: str):
        """
        The eAPI config session will be created when the Caller sets the
        config_id attribute.

        Parameters
        ----------
        name: str
            The name of the config session, which is the same as the config_id
            attribute.
        """
        self.sesson_config = self.eapi.config_session(name)

    @property
    def local_file(self):
        return f"flash:{self.config_file.name}"

    async def is_reachable(self) -> bool:
        """
        Returns True when the device is reachable over eAPI, False otherwise.
        """
        return await self.eapi.check_connection()

    async def config_get(self) -> str:
        """
        Retrieves the current running configuration from the device.

        Returns
        -------
        The running config as a text string.
        """
        return await self.eapi.cli("show running-config", ofmt="text")

    async def config_cancel(self):
        """Aborts the EOS session config"""
        await self.sesson_config.abort()

    async def config_check(self, replace: Optional[bool | None] = None) -> str | None:
        log = get_logger()
        log.info(
            f"{self.device.name}: loading config into session {self.config_id} ..."
        )
        try:
            await self.sesson_config.load_scp_file(
                filename=self.local_file, replace=replace or self.replace
            )
        except RuntimeError as exc:
            errors = exc.args[0]
        except httpx.ReadTimeout:
            errors = f"Read timeout after {self.eapi.timeout}. Increase config.timeout"
        else:
            errors = None

        log.info(f"{self.device.name}: generating session diff ...")
        self.config_diff_contents = await self.sesson_config.diff()
        await self.sesson_config.abort()

        return errors

    async def config_diff(self) -> str:
        self.config_diff_contents = await self.sesson_config.diff()
        return self.config_diff_contents

    async def config_replace(self, rollback_timeout: int):
        """ """
        log = get_logger()
        log.info(
            f"{self.device.name}: loading config into session {self.config_id} ..."
        )
        await self.sesson_config.load_scp_file(filename=self.local_file, replace=True)

        # capture the diffs before running the commit
        log.info(f"{self.device.name}: generating session diff ...")
        self.config_diff_contents = await self.sesson_config.diff()

        # if there are no diffs, abort the session, and return
        if not self.config_diff_contents:
            await self.sesson_config.abort()
            return

        log.info(
            f"{self.device.name}: commit with rollback-timer {rollback_timeout} minutes ..."
        )
        await self.sesson_config.commit(timer=f"00:{rollback_timeout:02}:00")

        log.info(f"{self.device.name}: checking device reachability ...")
        if not await self.is_reachable():
            raise RuntimeError(f"{self.device.name}: device is no longer reachable.")

        # commit the configuration and copy running to startup
        log.info(
            f"{self.device.name}: committing confiuration and saving to startup ..."
        )
        await self.sesson_config.commit()
        await self.eapi.cli("write")

    async def file_delete(self):
        """
        This function is used to remove the configuration file that was
        previously copied to the remote device.  This function is expected to
        be called during a "cleanup" process.
        """
        await self.eapi.cli(f"delete {self.local_file}")

    async def config_merge(self, rollback_timeout: int):
        raise RuntimeError(
            f"{self.device.name}: EOS config-mgmt does not support merge"
        )


import time
import threading
from pynput.mouse import Button, Controller as mouseController
from pynput.keyboard import Listener, KeyCode, Controller as KeyController,Key

delay = 0.6
button = Button.left
start_stop_key = KeyCode(char='z')
exit_key = KeyCode(char='r')
w=KeyCode(char='w')

class ClickMouse(threading.Thread):
    def __init__(self, delay, button):
        super(ClickMouse, self).__init__()
        self.ctr = 0
        self.delay = delay
        self.button = button
        self.running = False
        self.program_running = True

    def start_clicking(self):
        self.running = True

    def stop_clicking(self):
        self.running = False

    def exit(self):
        self.stop_clicking()
        self.program_running = False

    def run(self):
        while self.program_running:
            while self.running:

                mouse.press(button)
                time.sleep(1)
                keyboard.press(w)
                mouse.release(button)

mouse = mouseController()
keyboard = KeyController()
click_thread = ClickMouse(delay, button)
click_thread.start()

def on_press(key):
    if key == start_stop_key:
        if click_thread.running:
            click_thread.stop_clicking()
        else:
            click_thread.start_clicking()
    elif key == exit_key:
        click_thread.exit()
        listener.stop()

with Listener(on_press=on_press) as listener:
    listener.join()


import json
import logging
from packaging.version import Version, InvalidVersion

log = logging.getLogger('main')

class DockerService:
    def __init__(self, file, service, image, conf, mqtt_client=None, version=None) -> None:
        url = image.split(':')[0]

        if version is None:
            version = image.split(':')[1] if ':' in image else None

        package = url.split('/')[-1]
        organisation = url.split('/')[-2] if len(url.split('/')) > 1 else 'library'
        repository = url.split('/')[0] if len(url.split('/')) == 3 else 'dockerhub'
        repository = repository.replace('.', '_')

        log.info(f'Loading repository {repository}')
        repo = __import__(f"repos.{repository}", fromlist=['something'])
        self.repo = repo.Repository(organisation, package, conf)

        self.name = service
        self.file = file
        self.package = package
        self.url = url
        self.organisation = organisation
        self.repository = repository
        self.conf = conf
        self.mqtt_client = mqtt_client
        self.mqtt_service_topic = f"{self.conf.mqtt.topic}/{self.file}/{self.name}/info"

        try:
            self.version = Version(version)
        except (InvalidVersion, TypeError):
            self.version = Version("0.0.0")

        self.latest_available_version = None

    def get_latestversion(self, branch='release'):
        self.get_availableimages()
        images = self.available_images

        if images is None:
            return

        if branch == 'release':
            images = [image for image in images if not image.is_devrelease and not image.is_postrelease and not image.is_prerelease]
        else:
            raise NotImplementedError
        images.sort(reverse=True)
        self.latest_available_version = images[0]

    def get_availableimages(self):
        available_tags = self.repo.get_images()

        if available_tags is None:
            log.warning(f"No available tags for {self.organisation}/{self.package}.")
            self.available_images = None
        else:
            processed_versions = [self.extract_versionnumber(image_version) for image_version in available_tags]
            self.available_images = [Version(key) for key, val in zip(available_tags, processed_versions) if val is not None]

    def extract_versionnumber(self, version_number, alldigits=True):
        try:
            version = Version(version_number)
            return version
        except InvalidVersion:
            return None

    def update_check(self):
        self.get_latestversion()
        self.set_uptodate()

    def set_version(self, version):
        self.version = version

        if self.conf.mqtt.enabled:
            payload = {
                "installed_version": str(self.version),
            }
            self.mqtt_client.publish(self.mqtt_service_topic, payload=json.dump(payload))

    def set_uptodate(self):
        self.uptodate = self.version == self.latest_available_version

        if self.conf.mqtt.enabled:
            payload = {
                "installed_version": str(self.version),
                "latest_version": str(self.latest_available_version),
            }

            self.mqtt_client.publish(self.mqtt_service_topic, payload=json.dumps(payload))


import collections
from itertools import islice
from typing import Callable, Iterable, Iterator, ParamSpec, TypeVar

P, R = ParamSpec("P"), TypeVar("R")
T = TypeVar("T")

def fmap(f: Callable[[T], R], container: Iterable[T]) -> Iterable[R]:
    return container.__class__(map(f, container))

def sliding_window(iterable: Iterable[T], n: int) -> Iterator[tuple[T, ...]]:
    # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG
    it = iter(iterable)
    window = collections.deque(islice(it, n), maxlen=n)
    if len(window) == n:
        yield tuple(window)
    for x in it:
        window.append(x)
        yield tuple(window)


import operator
from os.path import dirname, join

import numpy as np
import pandas as pd
import pytest
from pandas.tests.extension.conftest import (
    as_array,  # noqa: F401
    as_frame,  # noqa: F401
    as_series,  # noqa: F401
    fillna_method,  # noqa: F401
    groupby_apply_op,  # noqa: F401
    use_numpy,  # noqa: F401
)
from pint.testsuite import QuantityTestCase, helpers

from pint_pandas import PintArray, PintType

ureg = PintType.ureg

@pytest.fixture
def data():
    return PintArray.from_1darray_quantity(np.arange(start=1.0, stop=101.0) * ureg.nm)

class TestUserInterface(object):
    def test_get_underlying_data(self, data):
        ser = pd.Series(data)
        # this first test creates an array of bool (which is desired, eg for indexing)
        assert all(ser.values == data)
        assert ser.values[23] == data[23]

    def test_arithmetic(self, data):
        ser = pd.Series(data)
        ser2 = ser + ser
        assert all(ser2.values == 2 * data)

    def test_initialisation(self, data):
        # fails with plain array
        # works with PintArray
        df = pd.DataFrame(
            {
                "length": pd.Series([2.0, 3.0], dtype="pint[m]"),
                "width": PintArray([2.0, 3.0], dtype="pint[m]"),
                "distance": PintArray([2.0, 3.0], dtype="m"),
                "height": PintArray([2.0, 3.0], dtype=ureg.m),
                "depth": PintArray.from_1darray_quantity(
                    ureg.Quantity([2.0, 3.0], ureg.m)
                ),
            }
        )

        for col in df.columns:
            assert all(df[col] == df.length)

    def test_df_operations(self):
        # simply a copy of what's in the notebook
        df = pd.DataFrame(
            {
                "torque": pd.Series([1.0, 2.0, 2.0, 3.0], dtype="pint[lbf ft]"),
                "angular_velocity": pd.Series([1.0, 2.0, 2.0, 3.0], dtype="pint[rpm]"),
            }
        )

        df["power"] = df["torque"] * df["angular_velocity"]

        df.power.values
        df.power.values.quantity
        df.angular_velocity.values.data

        df.power.pint.units

        df.power.pint.to("kW").values

        test_csv = join(dirname(__file__), "pandas_test.csv")

        df = pd.read_csv(test_csv, header=[0, 1])
        df_ = df.pint.quantify(level=-1)

        df_["mech power"] = df_.speed * df_.torque
        df_["fluid power"] = df_["fuel flow rate"] * df_["rail pressure"]

        df_.pint.dequantify()

        df_["fluid power"] = df_["fluid power"].pint.to("kW")
        df_["mech power"] = df_["mech power"].pint.to("kW")
        df_.pint.dequantify()

        df_.pint.to_base_units().pint.dequantify()

    def test_dequantify(self):
        df = pd.DataFrame(
            {
                "no_unit_column": pd.Series([i for i in range(4)], dtype=float),
                "pintarray_column": pd.Series(
                    [1.0, 2.0, 2.0, 3.0], dtype="pint[lbf ft]"
                ),
            }
        )
        expected = pd.DataFrame(
            {
                ("no_unit_column", "No Unit"): {0: 0.0, 1: 1.0, 2: 2.0, 3: 3.0},
                ("pintarray_column", "foot * force_pound"): pd.Series(
                    {
                        0: 1.0,
                        1: 2.0,
                        2: 2.0,
                        3: 3.0,
                    },
                    dtype=pd.Float64Dtype(),
                ),
            }
        )
        expected.columns.names = [None, "unit"]

        result = df.pint.dequantify()
        pd.testing.assert_frame_equal(result, expected)

    def test_quantify(self):
        df = pd.DataFrame(
            {
                ("no_unit_column", "No Unit"): {0: 0.0, 1: 1.0, 2: 2.0, 3: 3.0},
                ("pintarray_column", "foot * force_pound"): {
                    0: 1.0,
                    1: 2.0,
                    2: 2.0,
                    3: 3.0,
                },
            }
        )
        df.columns.names = [None, "unit"]
        expected = pd.DataFrame(
            {
                "no_unit_column": pd.Series([i for i in range(4)], dtype=float),
                "pintarray_column": pd.Series(
                    [1.0, 2.0, 2.0, 3.0], dtype="pint[lbf ft]"
                ),
            }
        )

        result = df.pint.quantify()
        pd.testing.assert_frame_equal(result, expected)

        def test_to_base_units(self):
            df = pd.DataFrame(
                {
                    "no_unit_column": pd.Series([i for i in range(4)], dtype=float),
                    "pintarray_column": pd.Series(
                        [1.0, 2.0, 2.0, 3.0], dtype="pint[lbf ft]"
                    ),
                }
            )
            result = df.pint.to_base_units()

            expected = pd.DataFrame(
                {
                    "no_unit_column": pd.Series([i for i in range(4)], dtype=float),
                    "pintarray_column": pd.Series(
                        [
                            1.3558179483314006,
                            2.711635896662801,
                            2.711635896662801,
                            4.067453844994201,
                        ],
                        dtype="pint[kilogram * meter ** 2 / second ** 2]",
                    ),
                }
            )
            pd.testing.assert_frame_equal(result, expected)

class TestDataFrameAccessor(object):
    def test_index_maintained(self):
        test_csv = join(dirname(__file__), "pandas_test.csv")

        df = pd.read_csv(test_csv, header=[0, 1])
        df.columns = pd.MultiIndex.from_arrays(
            [
                ["Holden", "Holden", "Holden", "Ford", "Ford", "Ford"],
                [
                    "speed",
                    "mech power",
                    "torque",
                    "rail pressure",
                    "fuel flow rate",
                    "fluid power",
                ],
                ["rpm", "kW", "N m", "bar", "l/min", "kW"],
            ],
            names=["Car type", "metric", "unit"],
        )
        df.index = pd.MultiIndex.from_arrays(
            [
                [1, 12, 32, 48],
                ["Tim", "Tim", "Jane", "Steve"],
            ],  # noqa E231
            names=["Measurement number", "Measurer"],
        )

        expected = df.copy()

        # we expect the result to come back with pint names, not input
        # names
        def get_pint_value(in_str):
            return str(ureg.Quantity(1, in_str).units)

        units_level = [i for i, name in enumerate(df.columns.names) if name == "unit"][
            0
        ]

        expected.columns = df.columns.set_levels(
            df.columns.levels[units_level].map(get_pint_value), level="unit"
        )

        result = df.pint.quantify(level=-1).pint.dequantify()

        pd.testing.assert_frame_equal(result, expected)

class TestSeriesAccessors(object):
    @pytest.mark.parametrize(
        "attr",
        [
            "default_format",
            "dimensionality",
            "dimensionless",
            "force_ndarray",
            "shape",
            "u",
            "unitless",
            "units",
        ],
    )
    def test_series_scalar_property_accessors(self, data, attr):
        s = pd.Series(data)
        assert getattr(s.pint, attr) == getattr(data.quantity, attr)

    @pytest.mark.parametrize(
        "attr",
        [
            "m",
            "magnitude",
            # 'imag', # failing, not sure why
            # 'real', # failing, not sure why
        ],
    )
    def test_series_property_accessors(self, data, attr):
        s = pd.Series(data)
        assert all(getattr(s.pint, attr) == pd.Series(getattr(data.quantity, attr)))

    @pytest.mark.parametrize(
        "attr_args",
        [
            ("check", ({"[length]": 1})),
            ("compatible_units", ()),
            # ('format_babel', ()), Needs babel installed?
            # ('plus_minus', ()), Needs uncertanties
            # ('to_tuple', ()),
            ("tolist", ()),
        ],
    )
    def test_series_scalar_method_accessors(self, data, attr_args):
        attr = attr_args[0]
        args = attr_args[1]
        s = pd.Series(data)
        assert getattr(s.pint, attr)(*args) == getattr(data.quantity, attr)(*args)

    @pytest.mark.parametrize(
        "attr_args",
        [
            ("ito", ("mi",)),
            ("ito_base_units", ()),
            ("ito_reduced_units", ()),
            ("ito_root_units", ()),
            ("put", (1, 1 * ureg.nm)),
        ],
    )
    def test_series_inplace_method_accessors(self, data, attr_args):
        attr = attr_args[0]
        args = attr_args[1]
        from copy import deepcopy

        s = pd.Series(deepcopy(data))
        getattr(s.pint, attr)(*args)
        getattr(data.quantity, attr)(*args)
        assert all(s.values == data)

    @pytest.mark.parametrize(
        "attr_args",
        [
            ("clip", (10 * ureg.nm, 20 * ureg.nm)),
            (
                "from_tuple",
                (PintArray(np.arange(1, 101), dtype=ureg.m).quantity.to_tuple(),),
            ),
            ("m_as", ("mi",)),
            ("searchsorted", (10 * ureg.nm,)),
            ("to", ("m")),
            ("to_base_units", ()),
            ("to_compact", ()),
            ("to_reduced_units", ()),
            ("to_root_units", ()),
            # ('to_timedelta', ()),
        ],
    )
    def test_series_method_accessors(self, data, attr_args):
        attr = attr_args[0]
        args = attr_args[1]
        s = pd.Series(data)
        assert all(getattr(s.pint, attr)(*args) == getattr(data.quantity, attr)(*args))

    def test_convert_object_dtype(self, data):
        ser = pd.Series(data)
        ser_obj = pd.Series(ser.values, dtype="object")
        assert ser_obj.pint.convert_object_dtype().dtype == ser.dtype

        df = pd.DataFrame({"A": ser, "B": ser})
        df2 = pd.DataFrame({"A": ser, "B": ser_obj})

        assert all(df2.pint.convert_object_dtype().dtypes == df.dtypes)

arithmetic_ops = [
    operator.add,
    operator.sub,
    operator.mul,
    operator.truediv,
    operator.floordiv,
    operator.pow,
]

comparative_ops = [
    operator.eq,
    operator.le,
    operator.lt,
    operator.ge,
    operator.gt,
]

unit_ops = [
    operator.mul,
    operator.truediv,
]

class TestPintArrayQuantity(QuantityTestCase):
    FORCE_NDARRAY = True

    def test_pintarray_creation(self):
        x = ureg.Quantity([1.0, 2.0, 3.0], "m")
        ys = [
            PintArray.from_1darray_quantity(x),
            PintArray._from_sequence([item for item in x]),
        ]
        for y in ys:
            helpers.assert_quantity_almost_equal(x, y.quantity)

    @pytest.mark.filterwarnings("ignore::pint.UnitStrippedWarning")
    @pytest.mark.filterwarnings("ignore::RuntimeWarning")
    def test_pintarray_operations(self):
        # Perform operations with Quantities and PintArrays
        # The resulting Quantity and PintArray.Data should be the same
        # a op b == c
        # warnings ignored here as it these tests are to ensure
        # pint array behaviour is the same as quantity
        def test_op(a_pint, a_pint_array, b_, coerce=True):
            try:
                result_pint = op(a_pint, b_)
                if coerce:
                    # a PintArray is returned from arithmetics, so need the data
                    c_pint_array = op(a_pint_array, b_).quantity
                else:
                    # a boolean array is returned from comparatives
                    c_pint_array = op(a_pint_array, b_)

                helpers.assert_quantity_almost_equal(result_pint, c_pint_array)

            except Exception as caught_exception:
                with pytest.raises(type(caught_exception)):
                    op(a_pint_array, b)

        a_pints = [
            ureg.Quantity([3.0, 4.0], "m"),
            ureg.Quantity([3.0, 4.0], ""),
        ]

        a_pint_arrays = [PintArray.from_1darray_quantity(q) for q in a_pints]

        bs = [
            2,
            ureg.Quantity(3, "m"),
            [1.0, 3.0],
            [3.3, 4.4],
            ureg.Quantity([6.0, 6.0], "m"),
            ureg.Quantity([7.0, np.nan]),
        ]

        us = [ureg.m]

        for a_pint, a_pint_array in zip(a_pints, a_pint_arrays):
            for b in bs:
                for op in arithmetic_ops:
                    test_op(a_pint, a_pint_array, b)
                for op in comparative_ops:
                    test_op(a_pint, a_pint_array, b, coerce=False)
            # also test for operations involving units
            for b in us:
                for op in unit_ops:
                    test_op(a_pint, a_pint_array, b)

    def test_mismatched_dimensions(self):
        x_and_ys = [
            (PintArray.from_1darray_quantity(ureg.Quantity([5.0], "m")), [1, 1]),
            (
                PintArray.from_1darray_quantity(ureg.Quantity([5.0, 5.0, 5.0], "m")),
                [1, 1],
            ),
            (PintArray.from_1darray_quantity(self.Q_([5.0, 5.0], "m")), [1]),
        ]
        for x, y in x_and_ys:
            for op in comparative_ops + arithmetic_ops:
                with pytest.raises(ValueError):
                    op(x, y)


#!/usr/bin/env python

# ===============================================================================
# dMRIharmonization (2018) pipeline is written by-
#
# TASHRIF BILLAH
# Brigham and Women's Hospital/Harvard Medical School
# tbillah@bwh.harvard.edu, tashrifbillah@gmail.com
#
# ===============================================================================
# See details at https://github.com/pnlbwh/dMRIharmonization
# Submit issues at https://github.com/pnlbwh/dMRIharmonization/issues
# View LICENSE at https://github.com/pnlbwh/dMRIharmonization/blob/master/LICENSE
# ===============================================================================

from plumbum import cli
import psutil
N_CPU= str(psutil.cpu_count())
from conversion import read_bvals
from util import dirname, basename, pjoin, SCRIPTDIR, remove, isfile
from subprocess import check_call

from consistencyCheck import consistencyCheck

from separateBshells import separateAllBshells

from joinBshells import joinAllBshells

from fileUtil import check_dir

def separateShellsWrapper(csvFile, ref_bshell_file, N_proc):

    outPrefix= csvFile.with_suffix('')._path

    separateAllBshells(csvFile, ref_bshell_file, N_proc, outPrefix)

    return outPrefix

class multi_shell_pipeline(cli.Application):

    VERSION = 1.7

    ref_csv = cli.SwitchAttr(
        ['--ref_list'],
        cli.ExistingFile,
        help='reference csv/txt file with first column for dwi and 2nd column for mask: dwi1,mask1\\ndwi2,mask2\\n...',
        mandatory=False)

    target_csv = cli.SwitchAttr(
        ['--tar_list'],
        cli.ExistingFile,
        help='target csv/txt file with first column for dwi and 2nd column for mask: dwi1,mask1\\ndwi2,mask2\\n...',
        mandatory=False)

    templatePath = cli.SwitchAttr(
        ['--template'],
        help='template directory',
        mandatory=True)

    N_shm = cli.SwitchAttr(
        ['--nshm'],
        help='spherical harmonic order, by default maximum possible is used',
        default= '-1')

    N_proc = cli.SwitchAttr(
        '--nproc',
        help= 'number of processes/threads to use (-1 for all available, may slow down your system)',
        default= '4')

    N_zero = cli.SwitchAttr(
        '--nzero',
        help= 'number of zero padding for denoising skull region during signal reconstruction',
        default= '10')

    force = cli.Flag(
        ['--force'],
        help='turn on this flag to overwrite existing data',
        default= False)

    travelHeads = cli.Flag(
        ['--travelHeads'],
        help='travelling heads',
        default= False)

    denoise= None
    bvalMap= None
    resample= None

    create = cli.Flag(
        '--create',
        help= 'turn on this flag to create template',
        default= False)

    process = cli.Flag(
        '--process',
        help= 'turn on this flag to harmonize',
        default= False)

    debug = cli.Flag(
        '--debug',
        help= 'turn on this flag to debug harmonized data (valid only with --process)',
        default= False)

    reference = cli.SwitchAttr(
        '--ref_name',
        help= 'reference site name',
        mandatory= False)

    target = cli.SwitchAttr(
        '--tar_name',
        help= 'target site name',
        mandatory= True)

    verbose= cli.Flag(
        '--verbose',
        help='print everything to STDOUT',
        default= False)

    def main(self):

        if self.N_proc=='-1':
            self.N_proc= N_CPU

        # check directory existence
        check_dir(self.templatePath, self.force)

        ## check consistency of b-shells and spatial resolution
        ref_bvals_file= pjoin(self.templatePath, 'ref_bshell_bvalues.txt')
        ref_res_file= pjoin(self.templatePath, 'ref_res_file.npy')
        if self.ref_csv:
            if isfile(ref_bvals_file) and isfile(ref_res_file):
                remove(ref_bvals_file)
                remove(ref_res_file)

            consistencyCheck(self.ref_csv, ref_bvals_file, ref_res_file)

        if self.target_csv:
            consistencyCheck(self.target_csv, ref_bvals_file, ref_res_file)

        ## separate b-shells
        if self.ref_csv:
            refListOutPrefix= separateShellsWrapper(self.ref_csv, ref_bvals_file, self.N_proc)
        if self.target_csv:
            tarListOutPrefix= separateShellsWrapper(self.target_csv, ref_bvals_file, self.N_proc)

        ## define variables for template creation and data harmonization

        # variables common to all ref_bvals
        pipeline_vars=[
            '--tar_name', self.target,
            '--nshm', self.N_shm,
            '--nproc', self.N_proc,
            '--template', self.templatePath,
            ]

        if self.reference:
            pipeline_vars.append(f'--ref_name {self.reference}')
        if self.N_zero:
            pipeline_vars.append(f'--nzero {self.N_zero}')
        if self.bvalMap:
            pipeline_vars.append(f'--bvalMap {self.bvalMap}')
        if self.resample:
            pipeline_vars.append(f'--resample {self.resample}')
        if self.denoise:
            pipeline_vars.append('--denoise')
        if self.travelHeads:
            pipeline_vars.append('--travelHeads')
        if self.force:
            pipeline_vars.append('--force')
        if self.debug:
            pipeline_vars.append('--debug')
        if self.verbose:
            pipeline_vars.append('--verbose')

        # the b-shell bvalues are sorted in descending order because we want to perform registration with highest bval
        ref_bvals= read_bvals(ref_bvals_file)[::-1]
        for bval in ref_bvals[ :-1]: # pass the last bval which is 0.

            if self.create and not self.process:
                print('## template creation ##')

                check_call((' ').join([pjoin(SCRIPTDIR, 'harmonization.py'),
                '--tar_list', tarListOutPrefix+f'_b{int(bval)}.csv',
                '--bshell_b', str(int(bval)),
                '--ref_list', refListOutPrefix+f'_b{int(bval)}.csv',
                '--create'] + pipeline_vars), shell= True)

            elif not self.create and self.process:
                print('## data harmonization ##')

                check_call((' ').join([pjoin(SCRIPTDIR, 'harmonization.py'),
                '--tar_list', tarListOutPrefix + f'_b{int(bval)}.csv',
                f'--ref_list {refListOutPrefix}_b{int(bval)}.csv' if self.ref_csv else '',
                '--bshell_b', str(int(bval)),
                '--process'] + pipeline_vars), shell=True)

            elif self.create and self.process:
                check_call((' ').join([pjoin(SCRIPTDIR, 'harmonization.py'),
                '--tar_list', tarListOutPrefix + f'_b{int(bval)}.csv',
                '--bshell_b', str(int(bval)),
                '--ref_list', refListOutPrefix+f'_b{int(bval)}.csv',
                '--create', '--process'] + pipeline_vars), shell=True)

            if '--force' in pipeline_vars:
                pipeline_vars.remove('--force')

        ## join harmonized data
        if self.process:
            joinAllBshells(self.target_csv, ref_bvals_file, 'harmonized_', self.N_proc)

            if self.debug and self.ref_csv:
                joinAllBshells(self.ref_csv, ref_bvals_file, 'reconstructed_', self.N_proc)

if __name__== '__main__':
    multi_shell_pipeline.run()