from typing import Callable, List, Optional

import datetime
import re

from langchain.memory.chat_memory import BaseChatMemory
from langchain.schema import BaseMessage

from chatflock.backing_stores.in_memory import InMemoryChatDataBackingStore
from chatflock.base import ChatMessage, ChatParticipant

def base_message_to_chat_message(base_message: BaseMessage) -> ChatMessage:
    content = str(base_message.content)

    pattern = re.compile(r"(\d+)\.\s*(.+?):\s*(.*)", re.DOTALL)
    match = pattern.match(content)

    if not match:
        return ChatMessage(id=-1, sender_name="SYSTEM", content=content)

    id_number = int(match.group(1))
    sender_name = match.group(2)
    message_content = match.group(3)

    return ChatMessage(id=id_number, sender_name=sender_name, content=message_content)

class LangChainMemoryBasedChatDataBackingStore(InMemoryChatDataBackingStore):
    no_output_message: str = "##NO_OUTPUT##"

    def __init__(
        self,
        memory: BaseChatMemory,
        memory_key_getter: Optional[Callable[[BaseChatMemory], str]] = None,
        messages: Optional[List[ChatMessage]] = None,
        include_timestamp_in_messages: bool = False,
        participants: Optional[List[ChatParticipant]] = None,
    ):
        super().__init__(participants=participants)

        self.memory = memory
        self.include_timestamp_in_messages = include_timestamp_in_messages

        if memory_key_getter is None:

            def default_memory_key_getter(memory: BaseChatMemory) -> str:
                if hasattr(memory, "memory_key"):
                    return str(memory.memory_key)

                return self.memory.output_key or "history"

            self.memory_key_getter: Callable[[BaseChatMemory], str] = default_memory_key_getter
        else:
            self.memory_key_getter = memory_key_getter

    def get_messages(self) -> List[ChatMessage]:
        prev_return_messages = self.memory.return_messages

        self.memory.return_messages = True

        memory_key = self.memory_key_getter(self.memory)
        base_messages = self.memory.load_memory_variables({})[memory_key]
        chat_messages = [
            base_message_to_chat_message(base_message)
            for base_message in base_messages
            if base_message.content != self.no_output_message
        ]

        self.memory.return_messages = prev_return_messages

        return chat_messages

    def add_message(self, sender_name: str, content: str, timestamp: Optional[datetime.datetime] = None) -> ChatMessage:
        message = super().add_message(sender_name=sender_name, content=content)

        prefix = ""
        if self.include_timestamp_in_messages:
            pretty_datetime = message.timestamp.strftime("%m-%d-%Y %H:%M:%S")
            prefix = f"[{pretty_datetime}] "

        self.memory.save_context(
            {"input": f"{prefix}{message.id}. {message.sender_name}: {message.content}"},
            {"output": self.no_output_message},
        )

        return message

    def clear_messages(self):
        super().clear_messages()

        self.memory.clear()


import rlp
from rlp.sedes import (
    binary,
    CountableList,
)

from eth.rlp.sedes import (
    uint384,
)

from .attestation_data import (
    AttestationData,
)

from eth.beacon.typing import (
    Bitfield,
    BLSSignature,
)
from eth.beacon.constants import EMPTY_SIGNATURE

class Attestation(rlp.Serializable):
    """
    Note: using RLP until we have standardized serialization format.
    """
    fields = [
        ('data', AttestationData),
        # Attester participation bitfield
        ('participation_bitfield', binary),
        # Proof of custody bitfield
        ('custody_bitfield', binary),
        # BLS aggregate signature
        ('aggregate_signature', CountableList(uint384)),
    ]

    def __init__(self,
                 data: AttestationData,
                 participation_bitfield: Bitfield,
                 custody_bitfield: Bitfield,
                 aggregate_signature: BLSSignature=EMPTY_SIGNATURE) -> None:
        super().__init__(
            data,
            participation_bitfield,
            custody_bitfield,
            aggregate_signature,
        )


import django.forms
import models
from django.forms.formsets import formset_factory

class ServiceForm(django.forms.Form):
    id = django.forms.CharField(widget=django.forms.HiddenInput())
    display_name = django.forms.CharField(required=False)
    consumer_secret = django.forms.CharField()
    consumer_key = django.forms.CharField()
    request_token_url = django.forms.URLField(initial='http://')
    authorize_url = django.forms.URLField(initial='http://')
    access_token_url = django.forms.URLField(initial='http://')

def create_service_formset(data=None):

    ServiceFormset = formset_factory(ServiceForm, extra=0)

    if data is None:
        initial_values = []
        for service in models.OAuthService.all():
            service_initial_values = {}
            service_initial_values["id"]                = service.key().name()
            service_initial_values["display_name"]      = service.display_name
            service_initial_values["consumer_secret"]   = service.consumer_secret
            service_initial_values["consumer_key"]      = service.consumer_key
            service_initial_values["request_token_url"] = service.request_token_url
            service_initial_values["authorize_url"]     = service.authorize_url
            service_initial_values["access_token_url"]  = service.access_token_url
            initial_values.append(service_initial_values)

        return ServiceFormset(initial = initial_values)

    return ServiceFormset(data)

def save_formset(service_formset):
    for form in service_formset:
        if form.is_valid():
            service = models.OAuthService.get_by_key_name(form.cleaned_data["id"])
            service.consumer_secret   = form.cleaned_data["consumer_secret"]
            service.consumer_key      = form.cleaned_data["consumer_key"]
            service.request_token_url = form.cleaned_data["request_token_url"]
            service.authorize_url     = form.cleaned_data["authorize_url"]
            service.access_token_url  = form.cleaned_data["access_token_url"]
            service.save()
            return True
        return False


import sklearn
from sklearn.preprocessing import StandardScaler
import csv
import numpy as np
import math
import matplotlib.pyplot as plt
from sklearn import preprocessing
import numpy as np
import data
from utils import Config
from math import pi
from math import exp
from math import sqrt
import pandas as pd
import seaborn as sn
from sklearn import metrics

from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_classif

from sklearn.model_selection import train_test_split

from sklearn.metrics import accuracy_score
from sklearn.svm import SVC

from sklearn.neighbors import KNeighborsClassifier

from sklearn.linear_model import LogisticRegression

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

from sklearn.naive_bayes import GaussianNB

from sklearn.linear_model import Perceptron

from sklearn import tree
# Split the dataset by class values, returns a dictionary
from random import randrange

def separate_by_class(dataset):
    separated = dict()
    for i in range(len(dataset)):
        vector = dataset[i]
        class_value = vector[-1]
        if class_value not in separated:
            separated[class_value] = list()
        separated[class_value].append(vector)
    return separated

# Calculate the mean of a list of numbers
def mean(numbers):
    return sum(numbers) / float(len(numbers))

# Calculate the standard deviation of a list of numbers
def stdev(numbers):
    avg = mean(numbers)
    variance = sum([(x - avg)**2 for x in numbers]) / float(len(numbers) - 1)
    return sqrt(variance)

# Calculate the mean, stdev and count for each column in a dataset
def summarize_dataset(dataset):
    summaries = [(mean(column), stdev(column), len(column))
                 for column in zip(*dataset)]
    del (summaries[-1])
    return summaries

def summarize_by_class(dataset):
    separated = separate_by_class(dataset)
    summaries = dict()
    for class_value, rows in separated.items():
        summaries[class_value] = summarize_dataset(rows)
    return summaries

# Calculate the Gaussian probability distribution function for x
def calculate_probability(x, mean, stdev):
    exponent = exp(-((x - mean)**2 / (2 * stdev**2)))
    return (1 / (sqrt(2 * pi) * stdev)) * exponent

# Calculate the probabilities of predicting each class for a given row
def calculate_class_probabilities(summaries, row):
    total_rows = sum([summaries[label][0][2] for label in summaries])
    probabilities = dict()
    for class_value, class_summaries in summaries.items():
        probabilities[class_value] = summaries[class_value][0][2] / float(
            total_rows)
        for i in range(len(class_summaries)):
            mean, stdev, count = class_summaries[i]
            probabilities[class_value] *= calculate_probability(
                row[i], mean, stdev)
    return probabilities

# Predict the class for a given row
def predict(summaries, row):
    probabilities = calculate_class_probabilities(summaries, row)
    best_label, best_prob = None, -1
    for class_value, probability in probabilities.items():
        if best_label is None or probability > best_prob:
            best_prob = probability
            best_label = class_value
    return best_label

# Calculate accuracy percentage
def accuracy_metric(test_set, predicted):
    correct = 0
    for i in range(len(test_set)):
        if test_set[i][-1] == predicted[i]:
            correct += 1
    return correct / float(len(test_set)) * 100.0

def feature_selection(train_set, test_set):
    train_feature = [i[0:13] for i in train_set]
    train_label = [i[-1] for i in train_set]
    test_feature = [i[0:13] for i in test_set]
    test_label = [i[-1] for i in test_set]

    ##PCA
    #     pca = PCA(n_components=13)
    #     pca.fit(train_feature)
    #     train_feature = pca.transform(train_feature)
    #     test_feature = pca.transform(test_feature)
    ## selectkbest
    model = SelectKBest(k=8).fit(train_feature, train_label)
    train_feature = model.transform(train_feature)
    test_feature = model.transform(test_feature)

    return train_feature, train_label, test_feature, test_label

# Naive Bayes Algorithm
def naive_bayes(train, test):
    summarize = summarize_by_class(train)
    predictions = list()
    for row in test:
        output = predict(summarize, row)
        predictions.append(output)
    accuracy = accuracy_metric(test, predictions)
    # return predictions
    return accuracy

#SVC
def SVC_model(train_set, test_set):
    train_feature, train_label, test_feature, test_label = feature_selection(
        train_set, test_set)
    model = SVC(C=1, gamma='auto', kernel='rbf')
    # model.fit(train_feature, train_label)
    y_pred = model.fit(train_feature, train_label).predict(test_feature)
    #     gnb.fit(train_feature, train_label).
    accuracy = model.score(test_feature, test_label)
    return accuracy, y_pred, test_label

def KNN_modell(train_set, test_set):
    train_feature, train_label, test_feature, test_label = feature_selection(
        train_set, test_set)
    classifier = KNeighborsClassifier(n_neighbors=4)
    y_pred = classifier.fit(train_feature, train_label).predict(test_feature)
    #     gnb.fit(train_feature, train_label).
    accuracy = classifier.score(test_feature, test_label)
    return accuracy, y_pred, test_label

def log_regression(train_set, test_set):
    train_feature, train_label, test_feature, test_label = feature_selection(
        train_set, test_set)
    logreg = LogisticRegression()
    y_pred = logreg.fit(train_feature, train_label).predict(test_feature)
    #     gnb.fit(train_feature, train_label).
    accuracy = logreg.score(test_feature, test_label)
    return accuracy, y_pred, test_label

def linear_discriminant(train_set, test_set):
    train_feature, train_label, test_feature, test_label = feature_selection(
        train_set, test_set)
    lda = LinearDiscriminantAnalysis()
    y_pred = lda.fit(train_feature, train_label).predict(test_feature)
    #     gnb.fit(train_feature, train_label).
    accuracy = lda.score(test_feature, test_label)
    return accuracy, y_pred, test_label

def perceptron(train_set, test_set):
    train_feature, train_label, test_feature, test_label = feature_selection(
        train_set, test_set)
    clf = Perceptron(tol=1e-3, random_state=0)
    y_pred = clf.fit(train_feature, train_label).predict(test_feature)
    #     gnb.fit(train_feature, train_label).
    accuracy = clf.score(test_feature, test_label)
    return accuracy, y_pred, test_label

def decisionTree(train_set, test_set):
    train_feature, train_label, test_feature, test_label = feature_selection(
        train_set, test_set)
    clf = tree.DecisionTreeClassifier()
    y_pred = clf.fit(train_feature, train_label).predict(test_feature)
    #     gnb.fit(train_feature, train_label).
    accuracy = clf.score(test_feature, test_label)
    return accuracy, y_pred, test_label

#split into k
def cross_split(dataset, folds):
    dataset_split = list()
    dataset_copy = list(dataset)
    for i in range(len(folds)):
        fold = list()
        for index in folds[i]:
            fold.append(dataset_copy[index])
        dataset_split.append(fold)
    return dataset_split

def evaluate_algorithm(dataset, algorithm, folds, *args):
    scores = []
    datadset_split = cross_split(dataset, folds)
    for fold in datadset_split:
        train_set = list(datadset_split)
        test_set = fold
        train_set.remove(fold)
        train_set = sum(train_set, [])
        accuracy, y_pred, test_label = algorithm(train_set, test_set)
        scores.append(accuracy)
    return scores

def test(train, test, best_one):
    accuracy, y_pred, test_label = best_one(train, test)
    return accuracy, y_pred, test_label

dataset, folds = data.getResult(Config['train'])

# n_validation = Config['n_validation']
# epoches = Config['epoches']
# scores = evaluate_algorithm(dataset, naive_bayes, folds)
# # scores = evaluate_algorithm(dataset, naive_bayes, folds, n_validation, epoches)
# print('Scores: %s' % scores)
# print('Mean Accuracy: %.3f%%' % (np.mean(scores, axis=0) * 100))

algorithms = [
    KNN_modell, log_regression, SVC_model, linear_discriminant, naive_bayes,
    perceptron, decisionTree
]
accuracies = []
for algorithm in algorithms:
    scores = evaluate_algorithm(dataset, algorithm, folds)
    accuracy = np.mean(scores, axis=0) * 100
    accuracies.append(accuracy)
    #  print('Mean Accuracy: %.3f%%' % (np.mean(scores2, axis=0) * 100))
print(accuracies)

# graph
name_list = ['KNN', 'logic', 'SVM', 'linear', 'naive', 'perceptron', 'Tree']
plt.bar(range(len(accuracies)), accuracies, color='rgb', tick_label=name_list)
plt.show()

index = accuracies.index(max(accuracies))
print("the best index is " + str(index))

# scores2 = evaluate_algorithm(dataset, svm_model, folds)
# print('Mean Accuracy: %.3f%%' % (np.mean(scores2, axis=0) * 100))
best_one = algorithms[index]
t, e = data.getResult(Config['test'])
accuracy, y_pred, test_label = test(dataset, t, best_one)
print("the best test accuracy is " + str(accuracy))

# confusion matrix for test
# list1 = [2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0]
# list2 = [1, 1, 0, 1, 2, 1, 1, 0, 1, 0, 0, 0]
data = {'y_Actual': test_label, 'y_Predicted': y_pred}
df = pd.DataFrame(data, columns=['y_Actual', 'y_Predicted'])
confusion_matrix = pd.crosstab(df['y_Actual'],
                               df['y_Predicted'],
                               rownames=['Actual'],
                               colnames=['Predicted'])

sn.heatmap(confusion_matrix, annot=True)
plt.show()


# -*- coding: utf-8 -*-

# Standard imports
from functools import partial
from multiprocessing import Pool

# External imports
import numpy as np

# Script information
__author__ = "Marti Municoy"
__license__ = "GPL"
__version__ = "1.0.1"
__maintainer__ = "Marti Municoy"
__email__ = "marti.municoy@bsc.es"

def p_extract_metrics(cols, report_path):
    results = []

    if (report_path.is_file()):
        try:
            with open(str(report_path), 'r') as f:
                f.readline()
                for line in f:
                    line.strip()
                    fields = line.split()
                    metrics = []
                    for col in cols:
                        metrics.append(fields[col - 1])
                    results.append(metrics)

        except IndexError:
            print(' - p_extract_ligand_metric Warning: wrong index '
                  + 'supplied for trajectory: \'{}\''.format(report_path))
    else:
        print(' - p_extract_ligand_metric Warning: wrong path to report '
              + 'for trajectory: \'{}\''.format(report_path))

    return results

def extract_metrics(reports, cols, proc_number=None):
    if (proc_number is None and len(reports) == 1):
        return [p_extract_metrics(cols, reports[0]), ]

    parallel_function = partial(p_extract_metrics, cols)

    with Pool(proc_number) as pool:
        results = pool.map(parallel_function,
                           reports)

    return results

def extract_PELE_ids(reports):
    epochs = []
    trajectories = []
    models = []

    for repo in reports:
        with open(str(repo), 'r') as f:
            f.readline()
            for i, line in enumerate(f):
                epochs.append(int(repo.parent.name))
                trajectories.append(
                    int(''.join(filter(str.isdigit, repo.name))))
                models.append(i)

    return epochs, trajectories, models

def get_metric_by_PELE_id(PELE_ids, metrics):
    metric_by_PELE_id = {}
    for e, t, m, metric in zip(*PELE_ids, np.concatenate(metrics)):
        metric_by_PELE_id[(e, t, m)] = metric

    return metric_by_PELE_id


# -*- coding: utf-8 -*-
"""
Created on Thu Jun 11 22:34:20 2020
@author: Krish Naik
"""

from __future__ import division, print_function
# coding=utf-8
import sys
import os
import glob
import re
import numpy as np

# Keras
from backend.model import load_food_model, predict_class

# Flask utils
from flask import Flask, request, render_template
from werkzeug.utils import secure_filename
#from gevent.pywsgi import WSGIServer

# Define a flask app
app = Flask(__name__)

# # Model saved with Keras model.save()
# MODEL_PATH ='model_vgg19.h5'

# # Load your trained model
# model = load_model(MODEL_PATH)

food_model = load_food_model()

def model_predict(img_path):
    preds = predict_class(food_model, img_path)

    return preds

@app.route('/', methods=['GET'])
def index():
    # Main page
    return render_template('index.html')

@app.route('/predict', methods=['GET', 'POST'])
def upload():
    if request.method == 'POST':
        # Get the file from post request
        f = request.files['file']

        # Save the file to ./uploads
        basepath = os.path.dirname(__file__)
        file_path = os.path.join(
            basepath, 'uploads', secure_filename(f.filename))
        f.save(file_path)

        # Make prediction
        preds = model_predict(file_path)
        result=preds
        return result
    return None

if __name__ == '__main__':
    app.run(debug=True, port = 3001)


from datetime import datetime, date, timedelta
import calendar

def get_month_range(start_date=None):
    if start_date is None:
        start_date = date.today().replace(day=1)
    days_in_month = calendar.monthrange(start_date.year, start_date.month)[1]
    end_date = start_date + timedelta(days=days_in_month)
    return (start_date, end_date)

first_day, last_day = get_month_range()
a_day = timedelta(days=1)
while first_day < last_day:
    print(first_day)
    first_day += a_day

def daterange(start, stop, step):
    while start < stop:
        yield start
        start += step

for d in daterange(date(2012, 8, 1), date(2012, 8, 11), timedelta(days=1)):
    print(d)

for d in daterange(datetime(2012, 8, 1), datetime(2012, 8, 3), timedelta(minutes=30)):
    print(d)


import unittest
import json
import csv
from find_contacts_at_companies_fn import find_contacts_at_companies

class FindContactsTestCase(unittest.TestCase):
    """Tests find_contacts_at_companies_fn.py"""

    # Remember, test methods must begin with the word "test."
    def test_1(self):
        """When the input_filename is passed to the
        function, does the returned results_list match the
        corresponding output_list_answer.json?"""

        input_filename_companies = 'test_files/input_data_companies_test2.csv'
        input_filename_contacts = 'test_files/input_data_contacts_test2.csv'

        (results_contacts, results_companies) = find_contacts_at_companies(
            input_filename_companies, input_filename_contacts
            )

        right_answer_file1 = 'test_files/output_companies_answer2.json'
        with open(right_answer_file1) as f_obj:
            right_companies_answer1 = json.load(f_obj)

        right_answer_file2 = 'test_files/output_contacts_answer2.json'
        with open(right_answer_file2) as f_obj:
            right_contacts_answer1 = json.load(f_obj)

        self.assertEqual(
            (results_contacts, results_companies),
            (right_contacts_answer1, right_companies_answer1)
            )

unittest.main()


import torch
from torch.utils import data
from torch.utils.data import TensorDataset, DataLoader
from utils.buffer.buffer import Buffer
from agents.base import ContinualLearner
from continuum.data_utils import dataset_transform
from utils.setup_elements import input_size_match
from utils.utils import maybe_cuda, AverageMeter
from kornia.augmentation import RandomResizedCrop, RandomHorizontalFlip, ColorJitter, RandomGrayscale
import torch.nn as nn

class SupContrastReplay(ContinualLearner):
    def __init__(self, model, opt, params):
        super(SupContrastReplay, self).__init__(model, opt, params)
        self.buffer = Buffer(model, params)
        self.mem_size = params.mem_size
        self.eps_mem_batch = params.eps_mem_batch
        self.mem_iters = params.mem_iters

        im_size = input_size_match(model=params.model, reshape224=params.reshape224)[params.data]
        self.transform = nn.Sequential(
            RandomResizedCrop(size=(im_size[1], im_size[2]), scale=(0.2, 1.)),
            RandomHorizontalFlip(),
            ColorJitter(0.4, 0.4, 0.4, 0.1, p=0.8),
            RandomGrayscale(p=0.2)

        )

    def train(self, train_loader, epoch, mem_iters=1, mem=None):
        # setup tracker
        losses = AverageMeter()
        acc_batch = AverageMeter()

        for ep in range(epoch):
            for i, batch_data in enumerate(train_loader):
                # batch update
                batch_x, batch_y = batch_data
                batch_x = maybe_cuda(batch_x, self.cuda)
                batch_y = maybe_cuda(batch_y, self.cuda)

                for j in range(self.mem_iters):
                    mem_x, mem_y = self.buffer.retrieve(x=batch_x, y=batch_y)

                    if mem_x.size(0) > 0:
                        mem_x = maybe_cuda(mem_x, self.cuda)
                        mem_y = maybe_cuda(mem_y, self.cuda)
                        combined_batch = torch.cat((mem_x, batch_x))
                        combined_labels = torch.cat((mem_y, batch_y))
                        combined_batch_aug = self.transform(combined_batch)
                        features = torch.cat([self.model.forward(combined_batch).unsqueeze(1), self.model.forward(combined_batch_aug).unsqueeze(1)], dim=1)
                        loss = self.criterion(features, combined_labels)
                        losses.update(loss, batch_y.size(0))
                        self.opt.zero_grad()
                        loss.backward()
                        self.opt.step()

                # update mem
                self.buffer.update(batch_x, batch_y)
                if i % 100 == 1 and self.verbose:
                        print(
                            '==>>> it: {}, avg. loss: {:.6f}, '
                                .format(i, losses.avg(), acc_batch.avg())
                        )


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

# This scripts generates a report of open security bugs rated with sec-critical
# or sec-high, containing
# * number of open bugs by week.
# * number of opened bugs by week.
# * number of closed bugs by week.
# * resolutions of bugs closed.
#
# The 'open' time
# * starts once 'sec-critical' or 'sec-high' gets added to the bugs' keywords
# * ends once
#   - the bug gets set as resolved
#   - neither 'sec-critical' or 'sec-high' is anymore in the keywords of the bug
#   - the keyword 'stalled' is added (either information to proceed missing like
#     data which can only captured during a violation and which is very rare, or
#     lack of developers to investigate further)
# If multiple affected time ranges are created by those, the time from the first
# time one of the security ratings got added to either the resolution of the bug
# or - if occurred before - the setting of the keyword 'stalled' or removal of
# the security rating from the keywords is regarded as the affected time.

import argparse
import copy
import csv
import datetime
from dateutil.relativedelta import relativedelta
import json
from libmozdata.bugzilla import Bugzilla
from logger import logger
import pytz
import utils

PRODUCTS_TO_CHECK = [
    'Core',
    'DevTools',
    'External Software Affecting Firefox',
    'Firefox',
    'Firefox Build System',
    'Firefox for Android',
    'Firefox for iOS',
    'GeckoView',
    'NSPR',
    'NSS',
    'Testing',
    'Toolkit',
    'WebExtensions',
]

critical = 'sec-critical'
high = 'sec-high'

SEC_RATINGS = [critical, high]

STATUS_GLOBAL_OPEN = ['UNCONFIRMED', 'NEW', 'ASSIGNED', 'REOPENED']
STATUS_GLOBAL_RESOLUTIONS = [
                             '---',
                             'FIXED',
                             'VERIFIED',
                             'INVALID',
                             'WONTFIX',
                             'INACTIVE',
                             'DUPLICATE',
                             'WORKSFORME',
                             'INCOMPLETE',
                             'SUPPORT',
                             'EXPIRED',
                             'MOVED',
                            ]

WFMT = '{}-{:02d}'

# Bugzilla data can be loaded from file
bugzilla_data_loaded = None

# Bugzilla data can be saved froto file
bugzilla_data_to_save = {}

def add_bugzilla_data_to_save(node_path, data):
    # node_path is an array of strings representing the nodes in the JSON to
    # which the data shall be saved. The node has a child 'data' which holds the
    # data.
    node = bugzilla_data_to_save
    for path_step in node_path:
        if not path_step in node:
            node[path_step] = {'data': []}
        node = node[path_step]
    node['data'].append(data)

def get_weeks(start_date, end_date):
    res = []
    weeks_attrs = {}
    while start_date.strftime('%Y-%W') <= end_date.strftime('%Y-%W'):
        y, w, _ = start_date.isocalendar()
        week_start, week_end = utils.get_week_bounds(start_date)
        week_label = WFMT.format(y, w)
        res.append(week_label)
        weeks_attrs[week_label] = {
                                  'week_start' : week_start,
                                  'week_end' : week_end,
                                 }
        start_date += relativedelta(days=7)
    return res, weeks_attrs

def get_bugs():

    def bug_handler(bug_data, other_data):
        if bzdata_save_path:
            add_bugzilla_data_to_save(['sec-critical-high'], bug_data)

        bug_creation_time_str = bug_data['creation_time']
        bug_creation_time = datetime.datetime.strptime(bug_creation_time_str, '%Y-%m-%dT%H:%M:%SZ')
        bug_creation_time = pytz.utc.localize(bug_creation_time)

        sec_important_start = None
        sec_important_end = None
        sec_important_rating = None

        sec_important_times = []
        stalled_added_last = None
        # Look for changes to the 'keywords' field.
        for historyItem in bug_data['history']:
            for change in historyItem['changes']:
                if change['field_name'] == 'keywords':
                    change_time_str = historyItem['when']
                    change_time = datetime.datetime.strptime(change_time_str, '%Y-%m-%dT%H:%M:%SZ')
                    change_time = pytz.utc.localize(change_time)

                    keywords_removed = change['removed'].split(', ')
                    keywords_added = change['added'].split(', ')

                    sec_rating_removed = critical in keywords_removed or high in keywords_removed
                    sec_rating_added = critical in keywords_added or high in keywords_added
                    if sec_rating_removed or sec_rating_added:
                        # Find and set security rating for time range.
                        if sec_rating_removed:
                            if critical in keywords_removed:
                                sec_important_rating = critical
                            elif high in keywords_removed:
                                sec_important_rating = high
                            if not sec_important_start:
                                sec_important_start = max(date_start, bug_creation_time)
                            sec_important_end = change_time
                            sec_important_times.append({
                                                        'rating' : sec_important_rating,
                                                        'start' : sec_important_start,
                                                        'end' : sec_important_end,
                                                      })
                        if sec_rating_added:
                            if critical in keywords_added:
                                sec_important_rating = critical
                            elif high in keywords_added:
                                sec_important_rating = high
                            sec_important_start = change_time
                        # no relevant security added
                        else:
                            sec_important_start = None
                            sec_important_rating = None
                        sec_important_end = None

                    # Bugs which got the 'stalled' keyword added are inactive
                    # and not counted as open (active).
                    stalled_removed = 'stalled' in keywords_removed
                    stalled_added = 'stalled' in keywords_added
                    if stalled_added:
                        stalled_added_last = change_time
                    elif stalled_removed:
                        stalled_added_last = None
        if sec_important_start:
            # Open time range at the end of the investigated time range.
            if not sec_important_end:
                sec_important_end = date_end
            sec_important_times.append({
                                        'rating' : sec_important_rating,
                                        'start' : sec_important_start,
                                        'end' : sec_important_end,
                                      })
        # No keyword changes at all, keyword should have been set when bug got
        # created.
        if len(sec_important_times) == 0:
            keywords = bug_data['keywords']
            sec_important_rating = None
            if critical in keywords:
                sec_important_rating = critical
            elif high in keywords:
                sec_important_rating = high
            else:
                sys.exit('Found neither sec-critical nor sec-high in keywords but expected one of them.')
            sec_important_times.append({
                                        'rating' : sec_important_rating,
                                        'start' : max(date_start, bug_creation_time),
                                        'end' : date_end,
                                      })

        sec_open_end = None
        if critical in bug_data['keywords'] or high in bug_data['keywords']:
            if bug_data['is_open']:
                if 'stalled' in bug_data['keywords']:
                    # Open but stalled bug with a security rating
                    if stalled_added_last:
                        sec_open_end = max(stalled_added_last, date_start)
                    else:
                        # Can only get here if bug got created with 'stalled'
                        # keyword, e.g. if bug got cloned.
                        sec_open_end = max(date_start, bug_creation_time)
                else:
                    sec_open_end = date_end
            else:
                # Bug closed, last time it got resolved used as ended of
                # affected time range.
                last_resolved_str = bug_data['cf_last_resolved']
                last_resolved = datetime.datetime.strptime(last_resolved_str, '%Y-%m-%dT%H:%M:%SZ')
                last_resolved = pytz.utc.localize(last_resolved)
                sec_open_end = min(sec_important_times[-1]['end'], last_resolved)
        else:
            sec_open_end = sec_important_times[0]['start']
        stalled = True if 'stalled' in bug_data['keywords'] else False
        bug_data_to_export = {
                              'id' : bug_data['id'],
                              'rating' : sec_important_times[-1]['rating'],
                              'start' : sec_important_times[0]['start'],
                              'end' : sec_open_end,
                              'stalled' : stalled,
                              'status' : bug_data['status'],
                              'resolution' : bug_data['resolution'],
                            }
        bug_sec_open_ranges.append(bug_data_to_export)

    # Load Bugzilla data from file
    if bzdata_load_path:
        for bug_data in bugzilla_data_loaded['sec-critical-high']['data']:
            other_data = {}
            bug_handler(bug_data, other_data)
    # Load Bugzilla data from Bugzilla server
    else:
        queries = []
        fields = [
                  'id',
                  'creation_time',
                  'status',
                  'is_open',
                  'resolution',
                  'cf_last_resolved',
                  'keywords',
                  'history',
                 ]

        params = {
            'include_fields': fields,
            'product': PRODUCTS_TO_CHECK,
            'j_top' : 'OR',
            # Either the keywords 'sec-critical' or 'sec-high' got removed
            # after the given date.
            'f1' : 'OP',
            'j1' : 'AND_G',
            'f2' : 'keywords',
            'o2' : 'changedfrom',
            'v2' : 'sec-critical',
            'f3' : 'keywords',
            'o3' : 'changedafter',
            'v3' : '',
            'f4' : 'CP',
            'f5' : 'OP',
            'j5' : 'AND_G',
            'f6' : 'keywords',
            'o6' : 'changedfrom',
            'v6' : 'sec-high',
            'f7' : 'keywords',
            'o7' : 'changedafter',
            'v7' : '',
            'f8' : 'CP',
            # Or the bug still has either the keyword 'sec-critical' or
            # 'sec-high' and
            'f9' : 'OP',
            'f10' : 'keywords',
            'o10' : 'anywords',
            'v10' : 'sec-critical, sec-high',
            'f11' : 'OP',
            'j11' : 'OR',
            # ... got resolved after the given date ...
            'f12' : 'cf_last_resolved',
            'o12' : 'changedafter',
            'v12' : '',
            # or hasn't been resolved yet.
            'f13' : 'bug_status',
            'o13' : 'anywords',
            'v13' : 'UNCONFIRMED, NEW, ASSIGNED, REOPENED',
            'f14' : 'CP',
            'f15' : 'CP',
        }

        params['v3'] = date_start_str
        params['v7'] = date_start_str
        params['v12'] = date_start_str

        query = Bugzilla(params,
                         bughandler=bug_handler,
                         bugdata={},
                         timeout=960)
        query.get_data().wait()

def log(message):
    print(message)

def aggregate_to_weekly_reports():
    for bug_sec_data in bug_sec_open_ranges:
        weeks_affected, unused_week_attrs = get_weeks(bug_sec_data['start'], bug_sec_data['end'])
        for week in weeks_affected:
            if week < weeks[0]:
                continue
            if week == weeks_affected[-1] and (bug_sec_data['status'] not in STATUS_GLOBAL_OPEN or bug_sec_data['stalled']):
                continue
            bug_sec_data_dict[bug_sec_data['rating']][week][bug_sec_data['id']] = bug_sec_data

    for sec_rating in bug_sec_data_dict:
        for week_pos in range(len(weeks) - 1, -1, -1):
            week = weeks[week_pos]
            open_by_week[sec_rating][week] = len(bug_sec_data_dict[sec_rating][week])

    for sec_rating in bug_sec_data_dict:
        for week_pos in range(len(weeks) - 1, 0, -1):
            week_this = weeks[week_pos]
            week_prev = weeks[week_pos - 1]
            open_this = set(bug_sec_data_dict[sec_rating][week_this].keys())
            open_prev = set(bug_sec_data_dict[sec_rating][week_prev].keys())
            opened_bugs = open_this - open_prev
            closed_bugs = open_prev - open_this
            opened_by_week[sec_rating][week_this] = len(opened_bugs)
            closed_by_week[sec_rating][week_this] = len(closed_bugs)
            for bug_id in closed_bugs:
                bug_data = bug_sec_data_dict[sec_rating][week_prev][bug_id]
                if bug_data['status'] not in STATUS_GLOBAL_OPEN:
                    resolution = bug_data['resolution']
                    if resolution not in STATUS_GLOBAL_RESOLUTIONS:
                        resolution = 'unknown'
                elif bug_data['stalled']:
                    resolution = 'stalled'
                resolutions_by_week[resolution][week_this] += 1

def write_csv():
    with open('data/security_bugs_report.csv', 'w') as Out:
        writer = csv.writer(Out, delimiter=',')

        date_row = ['Monday date']
        for week_pos in range(len(weeks) - 1, -1, -1):
            w = weeks[week_pos]
            date_row.append(weeks_attrs[w]['week_start'].strftime('%Y-%m-%d'))

        writer.writerow(['Open security bugs by week'])
        writer.writerow(date_row)

        for sec_rating in SEC_RATINGS:
            sec_rating_row = [sec_rating]
            for week_pos in range(len(weeks) - 1, -1, -1):
                w = weeks[week_pos]
                sec_rating_row.append(open_by_week[sec_rating][w])
            writer.writerow(sec_rating_row)
        sec_rating_total_row = ['Total']
        for week_pos in range(len(weeks) - 1, -1, -1):
            w = weeks[week_pos]
            sec_rating_total = 0
            for sec_rating in SEC_RATINGS:
                sec_rating_total += open_by_week[sec_rating][w]
            sec_rating_total_row.append(sec_rating_total)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow([])

        writer.writerow(['Opened security bugs by week'])
        writer.writerow(date_row)

        for sec_rating in SEC_RATINGS:
            sec_rating_row = [sec_rating]
            for week_pos in range(len(weeks) - 1, 0, -1):
                w = weeks[week_pos]
                sec_rating_row.append(opened_by_week[sec_rating][w])
            writer.writerow(sec_rating_row)
        sec_rating_total_row = ['Total']
        for week_pos in range(len(weeks) - 1, 0, -1):
            w = weeks[week_pos]
            sec_rating_total = 0
            for sec_rating in SEC_RATINGS:
                sec_rating_total += opened_by_week[sec_rating][w]
            sec_rating_total_row.append(sec_rating_total)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow([])

        writer.writerow(['Closed security bugs by week'])
        writer.writerow(date_row)

        for sec_rating in SEC_RATINGS:
            sec_rating_row = [sec_rating]
            for week_pos in range(len(weeks) - 1, 0, -1):
                w = weeks[week_pos]
                sec_rating_row.append(closed_by_week[sec_rating][w])
            writer.writerow(sec_rating_row)
        sec_rating_total_row = ['Total']
        for week_pos in range(len(weeks) - 1, 0, -1):
            w = weeks[week_pos]
            sec_rating_total = 0
            for sec_rating in SEC_RATINGS:
                sec_rating_total += closed_by_week[sec_rating][w]
            sec_rating_total_row.append(sec_rating_total)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow([])

        writer.writerow(['Resolutions of security bugs by week'])
        writer.writerow(date_row)

        for resolution in STATUS_GLOBAL_RESOLUTIONS + ['stalled', 'unknown']:
            resolution_row = [resolution]
            for week_pos in range(len(weeks) - 1, 0, -1):
                w = weeks[week_pos]
                resolution_row.append(resolutions_by_week[resolution][w])
            writer.writerow(resolution_row)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow(['"stalled" is not a Resolution status but a keyword which is added if either ' + \
                         'information to proceed is missing like data which can only captured during a ' + \
                         'violation and which is very rare, or lack of developers to investigate further.'])
        writer.writerow(['"unknown" is catching resolutions unknown to this script, should be new to ' + \
                         'bugzilla.mozilla.org'])

parser = argparse.ArgumentParser(description='Count security bugs opened and closed by week')
parser.add_argument('--bzdata-load',
                    nargs='?',
                    default=argparse.SUPPRESS,
                    help='Load the Bugzilla data from a local JSON file. If no path is provided '
                         'the program will try to load "sec_bugs_bugzilla_data.json" from the "data" folder.')
parser.add_argument('--bzdata-save',
                    nargs='?',
                    default=argparse.SUPPRESS,
                    help='Save the Bugzilla data to a local JSON file. If no path is provided '
                         'the program will try to save as "sec_bugs_bugzilla_data.json" into the "data" folder.')
args = parser.parse_args()

# Start of time range used by report. Hardcoded default of 1 year.
date_start = datetime.datetime.now() - datetime.timedelta(days = 365)
date_start = pytz.utc.localize(date_start)
date_start_str = date_start.strftime('%Y-%m-%dT%H:%M:%SZ')

# End of time range used by report.
date_end = datetime.datetime.utcnow()
date_end = pytz.utc.localize(date_end)
date_end_str = date_end.strftime('%Y-%m-%dT%H:%M:%SZ')

weeks, weeks_attrs = get_weeks(date_start, date_end)

bzdata_load_path = None
if 'bzdata_load' in args:
    # Load Bugzilla data from file
    if args.bzdata_load:
        # File path provided as command line argument
        bzdata_load_path = args.bzdata_load
    else:
        # No file path provided, use default location
        bzdata_load_path = 'data/sec_bugs_bugzilla_data.json'
    with open(bzdata_load_path, 'r') as bugzilla_data_reader:
        bugzilla_data_loaded = json.load(bugzilla_data_reader)
    log('Loaded Bugzilla data from {}'.format(bzdata_load_path))
    date_start_str = bugzilla_data_loaded['date_start']['data'][0]
    date_start = datetime.datetime.strptime(date_start_str, '%Y-%m-%dT%H:%M:%SZ')
    date_start = pytz.utc.localize(date_start)
    log('Date start from loaded Bugzilla data: {}'.format(date_start_str))
    date_end_str = bugzilla_data_loaded['date_end']['data'][0]
    date_end = datetime.datetime.strptime(date_end_str, '%Y-%m-%dT%H:%M:%SZ')
    date_end = pytz.utc.localize(date_end)
    log('Date end from loaded Bugzilla data: {}'.format(date_end_str))

bzdata_save_path = None
if 'bzdata_save' in args:
    # File path to which Bugzilla data shall be saved
    if args.bzdata_save:
        # File path provided as command line argument
        bzdata_save_path = args.bzdata_save
    else:
        # No file path provided, use default location
        bzdata_save_path = 'data/sec_bugs_bugzilla_data.json'

# Holds the time range in which a bug was considered open (see top of file) and
# had security rating.
bug_sec_open_ranges = []
get_bugs()

bug_sec_data_dict = { sec_rating : { week : {} for week in weeks } for sec_rating in SEC_RATINGS }
report_data = { sec_rating : { week : set() for week in weeks } for sec_rating in SEC_RATINGS }
open_by_week = { sec_rating : { week : 0 for week in weeks } for sec_rating in SEC_RATINGS }
opened_by_week = { sec_rating : { week : 0 for week in weeks } for sec_rating in SEC_RATINGS }
closed_by_week = { sec_rating : { week : 0 for week in weeks } for sec_rating in SEC_RATINGS }
resolutions_by_week = { resolution : { week : 0 for week in weeks } for resolution in STATUS_GLOBAL_RESOLUTIONS }
# 'stalled' is not a Resolution status but a keyword which is added if either
# information to proceed is missing like data which can only captured during a
# violation and which is very rare, or lack of developers to investigate
# further.
resolutions_by_week['stalled'] = { week : 0 for week in weeks }
# 'unknown' is catching resolutions unknown to this script, should be new to
# bugzilla.mozilla.org
resolutions_by_week['unknown'] = { week : 0 for week in weeks }
aggregate_to_weekly_reports()

write_csv()

if bzdata_save_path:
    # Save Bugzilla data to file
    with open('data/sec_bugs_bugzilla_data.json', 'w') as bugzilla_data_writer:
        add_bugzilla_data_to_save(['date_start'], date_start_str)
        add_bugzilla_data_to_save(['date_end'], date_end_str)
        bugzilla_data_writer.write(json.dumps(bugzilla_data_to_save))
        log('Saved Bugzilla data to {}'.format(bzdata_save_path))


from flask import current_app, Flask, jsonify, request, abort
from swagger_parser import SwaggerParser

class FlaskSwagger(object):

    def __init__(self, app=None, swagger_file=None):
        self.app = app
        if app is not None:
            self.init_app(app, swagger_file=swagger_file)

    # Resolves in this order:
    # - full example on endpoint's first response
    # - individual fields on endpoint
    # - full example on model
    # = individual fields on model
    def resolve_example(self, method_config):

        def responder():
            # Prep
            if 'responses' in method_config:
                first_response_code = sorted([key for key in method_config['responses']])[0]

            # First option
            if 'responses' in method_config and 'examples' in method_config['responses'][first_response_code]:
                example = method_config['responses'][first_response_code]['examples']['application/json']

                return jsonify(example)

        return responder

    def before_request(self):

        print("BEFORE A REQUEST", request)

        # print('Request method', type(request.method), request.method)
        # print('Request path', type(request.path), request.path)
        # print('Request query string', type(request.query_string), request.query_string)

        # if request.is_json:
        #     print('Request json', type(request.get_json()), request.get_json())
        # else:
        #     print("Request is not JSON")

        valid = self.validate_werkzeug_request(request)

        if not valid:
            abort(400)

    def validate_werkzeug_request(self, req):

        json_methods = set(['post', 'put', 'patch'])
        method = req.method.lower()

        if method in json_methods:
            if not req.is_json:
                raise Exception('A ' + method + ' call is not JSON. Check it has a body of well-formed JSON and the content-type is set to application/json')
            body = req.get_json()
        else:
            body = None

        return self.parser.validate_request(req.path, method, body=body)

    def init_app(self, app, swagger_file=None):
        if not swagger_file:
            raise Exception('Swagger file not specified')

        with open(swagger_file) as f:
            swagger_yaml = f.read()

        if not swagger_yaml:
            raise Exception('No swagger found')

        self.parser = SwaggerParser(swagger_yaml=swagger_yaml)

        for path, path_config in self.parser.paths.items():
            for method, method_config in path_config.items():
                found = False

                # print(app.url_map)
                for rule in app.url_map.iter_rules():
                    if rule.rule == path and method.upper() in rule.methods:
                        found = True
                        # print(dir(rule.map.iter_rules))

                if not found:
                    print("Warning: spec route not implemented; using example from Swagger: " + method.upper() + " " + path)

                    app.add_url_rule(path, path+method, self.resolve_example(method_config), methods=[method])

        app.before_request(self.before_request)

if __name__== "__main__":

    from waitress import serve

    app = Flask(__name__)

    @app.route('/pet/<pet_id>')
    def something(pet_id):
        return "This is from code"

    @app.route('/pet', methods=['POST'])
    def something_else():
        return 'valid'

    FlaskSwagger(app, "swagger.yaml")
    # serve(app, host='0.0.0.0', port=5000)
    app.run(debug=True, port=5000)


import logging

class filelogger:

    def filelog(self):

        # create logger
        logger = logging.getLogger("file_loger")
        logger.setLevel(logging.DEBUG)

        # adding the logger to file.
        file = logging.FileHandler("file_log_handler")

        # create formatter
        formatter = logging.Formatter('%(asctime)s - %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%H:%S %p')

        # add formatter to file
        file.setFormatter(formatter)

        # add file handler to logger
        logger.addHandler(file)

        # log message
        logger.debug(" this is the debug message")
        logger.info(" this is the info message")
        logger.warning("this is the warning message")
        logger.error("this is the error message")
        logger.critical("this is the critical message")

lo = filelogger()
lo.filelog()


import requests,json,ast,shapefile
from RandomPanoDownloader.countryFilter import getRandomLatLing
from RandomPanoDownloader.PanoDownloader import download_pano
from random import choice as randomChoice

def getPanoFromCountryCode(countryCode=None,outdoors=False):
    failedtries = 0
    while True:
        lat, long = getRandomLatLing(countryCode)
        radius = 1000

        if failedtries > 100: #Extend radius
            radius = 10000

        fetchurl = 'https://maps.googleapis.com/maps/api/js/GeoPhotoService.SingleImageSearch?pb=!1m5!1sapiv3!5sUS!11m2!1m1!1b0!2m4!1m2!3d{0}!4d{1}!2d{2}!3m18!2m2!1sen!2sUS!9m1!1e2!11m12!1m3!1e2!2b1!3e2!1m3!1e3!2b1!3e2!1m3!1e10!2b1!3e2!4m6!1e1!1e2!1e3!1e4!1e8!1e6&callback=initialize'.format(lat,long,radius)

        if outdoors:
            fetchurl = 'https://maps.googleapis.com/maps/api/js/GeoPhotoService.SingleImageSearch?pb=!1m5!1sapiv3!5sUS!11m2!1m1!1b0!2m4!1m2!3d{0}!4d{1}!2d{2}!3m20!1m1!3b1!2m2!1sen!2sUS!9m1!1e2!11m12!1m3!1e2!2b1!3e2!1m3!1e3!2b1!3e2!1m3!1e10!2b1!3e2!4m6!1e1!1e2!1e3!1e4!1e8!1e6&callback=initialize'.format(lat,long,radius)
            res = requests.get(fetchurl)

            # response = res.text[:-1]
            # response = response[29:]
        res = requests.get(fetchurl)
        response = res.text[:-1]
        response = response[29:]

        jsonified = json.loads(response)

        if len(jsonified) > 1:
            panoid = jsonified[1][1][1]

            metadata = None
            if len(jsonified[1][3]):
                metadata = jsonified[1][3][2] #No metadata attached

            coords = jsonified[1][5][0][1][0]
            lat, long = coords[2],coords[3]

            loc_name = None
            if metadata is not None:
                loc_name = metadata[-1][0]
            break
        else:
            failedtries += 1
            continue
    return panoid,lat,long,loc_name

# panoid,lat,long,loc_name = getPanoFromCountryCode(countryCode=None,outdoors=True) #Truly Fully random, no filters
# print(panoid,lat,long,loc_name)
# download_pano(panoid,'downloaded.png')


from typing import List, Optional

from hestia.unknown import UNKNOWN

from lifecycles.jobs import JobLifeCycle
from lifecycles.statuses import BaseStatuses, StatusOptions

class ExperimentLifeCycle(BaseStatuses):
    CREATED = StatusOptions.CREATED
    RESUMING = StatusOptions.RESUMING
    WARNING = StatusOptions.WARNING
    UNSCHEDULABLE = StatusOptions.UNSCHEDULABLE
    BUILDING = StatusOptions.BUILDING
    SCHEDULED = StatusOptions.SCHEDULED
    STARTING = StatusOptions.STARTING
    RUNNING = StatusOptions.RUNNING
    SUCCEEDED = StatusOptions.SUCCEEDED
    FAILED = StatusOptions.FAILED
    UPSTREAM_FAILED = StatusOptions.UPSTREAM_FAILED
    STOPPED = StatusOptions.STOPPED
    SKIPPED = StatusOptions.SKIPPED
    UNKNOWN = UNKNOWN

    CHOICES = (
        (CREATED, CREATED),
        (RESUMING, RESUMING),
        (WARNING, WARNING),
        (BUILDING, BUILDING),
        (SCHEDULED, SCHEDULED),
        (STARTING, STARTING),
        (RUNNING, RUNNING),
        (SUCCEEDED, SUCCEEDED),
        (FAILED, FAILED),
        (UPSTREAM_FAILED, UPSTREAM_FAILED),
        (STOPPED, STOPPED),
        (SKIPPED, SKIPPED),
        (UNKNOWN, UNKNOWN),
    )

    VALUES = {
        CREATED,
        RESUMING,
        WARNING,
        BUILDING,
        SCHEDULED,
        STARTING,
        RUNNING,
        SUCCEEDED,
        FAILED,
        UPSTREAM_FAILED,
        STOPPED,
        SKIPPED,
        UNKNOWN
    }

    HEARTBEAT_STATUS = {RUNNING, }
    WARNING_STATUS = {WARNING, }
    PENDING_STATUS = {CREATED, RESUMING, }
    RUNNING_STATUS = {SCHEDULED, BUILDING, STARTING, RUNNING, }
    DONE_STATUS = {FAILED, UPSTREAM_FAILED, STOPPED, SKIPPED, SUCCEEDED, }
    FAILED_STATUS = {FAILED, UPSTREAM_FAILED, }

    TRANSITION_MATRIX = {
        CREATED: {None, },
        RESUMING: {CREATED, WARNING, SUCCEEDED, SKIPPED, STOPPED, },
        BUILDING: {CREATED, RESUMING, WARNING, UNKNOWN, },
        SCHEDULED: {CREATED, RESUMING, BUILDING, WARNING, UNKNOWN, },
        STARTING: {CREATED, RESUMING, BUILDING, SCHEDULED, WARNING, },
        RUNNING: {CREATED, RESUMING, BUILDING, SCHEDULED, STARTING, WARNING, UNKNOWN, },
        SKIPPED: VALUES - DONE_STATUS,
        SUCCEEDED: VALUES - DONE_STATUS,
        FAILED: VALUES - DONE_STATUS,
        UPSTREAM_FAILED: VALUES - DONE_STATUS,
        STOPPED: VALUES - {STOPPED, SKIPPED, },
        WARNING: VALUES - DONE_STATUS - {WARNING, },
        UNKNOWN: VALUES - DONE_STATUS - {UNKNOWN, },
    }

    @staticmethod
    def jobs_unschedulable(job_statuses: List[str]) -> bool:
        return any([True if JobLifeCycle.is_unschedulable(job_status) else False
                    for job_status in job_statuses])

    @staticmethod
    def jobs_warning(job_statuses: List[str]) -> bool:
        return any([True if JobLifeCycle.is_warning(job_status) else False
                    for job_status in job_statuses])

    @staticmethod
    def jobs_starting(job_statuses: List[str]) -> bool:
        return any([True if JobLifeCycle.is_starting(job_status) else False
                    for job_status in job_statuses])

    @staticmethod
    def jobs_running(job_statuses: List[str]) -> bool:
        return any([True if JobLifeCycle.is_running(job_status) else False
                    for job_status in job_statuses])

    @staticmethod
    def jobs_succeeded(job_statuses: List[str]) -> bool:
        return all([True if job_status == JobLifeCycle.SUCCEEDED else False
                    for job_status in job_statuses])

    @staticmethod
    def jobs_failed(job_statuses: List[str]) -> bool:
        return any([True if job_status == JobLifeCycle.FAILED else False
                    for job_status in job_statuses])

    @staticmethod
    def jobs_stopped(job_statuses: List[str]) -> bool:
        return any([True if job_status == JobLifeCycle.STOPPED else False
                    for job_status in job_statuses])

    @classmethod
    def jobs_unknown(cls, job_statuses: List[str]) -> bool:
        return any([True if job_status == JobLifeCycle.UNKNOWN else False
                    for job_status in job_statuses])

    @classmethod
    def jobs_status(cls, job_statuses: List[str]) -> Optional[str]:
        if not job_statuses:
            return None

        if cls.jobs_unknown(job_statuses):
            return cls.UNKNOWN

        if cls.jobs_unschedulable(job_statuses):
            return cls.UNSCHEDULABLE

        if cls.jobs_warning(job_statuses):
            return cls.WARNING

        if cls.jobs_stopped(job_statuses):
            return cls.STOPPED

        if cls.jobs_succeeded(job_statuses):
            return cls.SUCCEEDED

        if cls.jobs_failed(job_statuses):
            return cls.FAILED

        if cls.jobs_starting(job_statuses):
            return cls.STARTING

        if cls.jobs_running(job_statuses):
            return cls.RUNNING

        return cls.UNKNOWN


from typing import Dict, Tuple, NamedTuple, List

import os
import fcntl
import shutil
import subprocess
import time

from string import Template

import json
import numpy as np
import tensorflow as tf

from boa.models import GPARModel, FullyFactorizedGPModel, RandomModel
from boa.models.bnn.multi_output_bnn import BasicBNN
from boa.acquisition.smsego import SMSEGO
from boa.grid import SobolGrid
from boa.objective.gem5_aladdin.util import create_gem5_sweep_config, get_output_vector_from_dict
from boa.objective.gem5_aladdin.process import parse_file, output_regexps, parse_stats_txt

"""
TODO: handle string categorical grids
TODO: show Miguel graphs
"""

# Note: run_experiment.py should always be run in a directory that contains a valid MachSuite or SMAUG config.py file.
try:
    from config import config
except (ModuleNotFoundError, ImportError):
    print("You must run this script where a valid MachSuite or SMAUG config.py file is located!")
    raise

# Set CPU as available physical device
tf.config.experimental.set_visible_devices([], 'GPU')

machsuite_tasks = [
    "fft_transpose",
    "stencil_stencil3d",
    "gemm_blocked"
]

smaug_tasks = [
    "minerva"
]

def train_surrogate_model(surrogate_model, model_config, model):

    if model == "sgs":
        # Random model neds no fitting
        pass

    elif model == "bnn":
        surrogate_model.fit(num_samples=model_config["num_samples"],
                            burnin=model_config["burnin"],
                            keep_every=model_config["keep_every"])

    elif model in ["ff-gp", "gpar-targets", "gpar-full"]:
        surrogate_model.fit(optimizer=model_config["optimizer"],
                            optimizer_restarts=model_config["optimizer_restarts"],
                            iters=model_config["iters"],
                            fit_joint=model_config["fit_joint"],
                            length_scale_init_mode=model_config["length_scale_init"],
                            map_estimate=False,
                            denoising=False,
                            trace=True)

    else:
        raise NotImplementedError

def pareto_frontier(points):
    points = points.numpy()

    is_pareto = np.ones(points.shape[0], dtype=bool)

    for i, point in enumerate(points):

        if is_pareto[i]:
            is_pareto[is_pareto] = np.any(points[is_pareto, :] < point, axis=1)
            is_pareto[i] = True

    return tf.convert_to_tensor(is_pareto)

def create_smaug_sweep_config(template_file_path: str,
                              input_settings: Dict):

    # Creates the commands describing what inputs to try next
    eval_settings = [f'"{inp}": [{val}]' for inp, val in input_settings.items()]
    eval_settings = ',\n    '.join(eval_settings)

    template_file = open(template_file_path)

    template = Template(template_file.read())

    sweep_config = template.substitute({"evaluation_settings": eval_settings})

    return sweep_config

def run_machsuite_simulation_with_configuration(task: str,
                                                simulation_dir: str,
                                                run_sh_subdir: str,
                                                simulation_config: str,
                                                output_order: List[str],
                                                singularity_image: str,
                                                template_name: str = "template.xe",
                                                template_generation_out_file_name: str = "template_generation_output.txt",
                                                generate_design_sweeps_py_path: str = "/workspace/gem5-aladdin/sweeps/generate_design_sweeps.py",
                                                simulation_output_file_name: str = "simulation_output.txt",
                                                stdout_path: str = "outputs/stdout",
                                                stderr_path: str = "outputs/stderr",
                                                results_file_name: str = "results.json",
                                                ):
    os.makedirs(simulation_dir, exist_ok=True)

    run_script_dir = os.path.join(simulation_dir, run_sh_subdir)
    nonaccel_file_path = os.path.join(run_script_dir, f"{task}-gem5")
    accel_file_path = os.path.join(run_script_dir, f"{task}-gem5-accel")

    # Write template
    with open(os.path.join(simulation_dir, template_name), "w") as template_file:
        template_file.write(simulation_config)

    # Generate design sweep

    print(f"Generating design in {simulation_dir}!")
    with open(os.path.join(simulation_dir, template_generation_out_file_name), "w") as log_file:

        result = subprocess.run([
            "singularity",
            "exec",
            "--pwd",  # Changes the working directory to the appropriate one
            simulation_dir,
            singularity_image,
            generate_design_sweeps_py_path,
            template_name],
            stdout=log_file,
            stderr=log_file,
            check=True,
        )

    print(f"Running simulation in {run_script_dir}!")

    with open(os.path.join(simulation_dir, simulation_output_file_name), "w") as log_file:
        try:
            result = subprocess.run([
                "singularity",
                "exec",
                "--pwd",
                run_script_dir,
                singularity_image,
                "bash",
                "run.sh"
            ],
                stdout=log_file,
                stderr=log_file,
                check=True,
            )

        except subprocess.CalledProcessError as e:
            stderr_path = os.path.join(simulation_dir, run_sh_subdir, stderr_path)
            log_file.writelines(["========================\n",
                                 f"Copied from {stderr_path}\n"
                                 "========================\n", ])

            with open(stderr_path) as stderr_log_file:
                stderr_log = stderr_log_file.read()

                log_file.write(stderr_log)

            print(str(e))
            raise ValueError(stderr_log)

    results = parse_file(os.path.join(simulation_dir, run_sh_subdir, stdout_path),
                         output_regexps)

    os.remove(nonaccel_file_path)
    os.remove(accel_file_path)

    with open(os.path.join(simulation_dir, results_file_name), "w") as results_file:
        json.dump(results, results_file)

    result_vec = get_output_vector_from_dict(results, output_order=output_order)

    return result_vec

def run_smaug_simulation_with_configuration(task: str,
                                            simulation_dir: str,
                                            run_sh_subdir: str,
                                            simulation_config: str,
                                            output_order: List[str],
                                            singularity_image: str,
                                            template_name: str = "template.json",
                                            template_generation_out_file_name: str = "template_generation_output.txt",
                                            main_py_path: str = "/workspace/smaug/experiments/sweeps/main.py",
                                            simulation_output_file_name: str = "simulation_output.txt",
                                            stdout_path: str = "stdout",
                                            stderr_path: str = "stderr",
                                            results_file_name: str = "results.json",
                                            ):

    os.makedirs(simulation_dir, exist_ok=True)

    run_script_dir = os.path.join(simulation_dir, run_sh_subdir)

    # Write template
    with open(os.path.join(simulation_dir, template_name), "w") as template_file:
        template_file.write(simulation_config)

    # Generate design sweep

    print(f"Generating design file in {simulation_dir}!")
    with open(os.path.join(simulation_dir, template_generation_out_file_name), "w") as log_file:

        os.makedirs(os.path.join(simulation_dir, task), exist_ok=True)

        result = subprocess.run([
            "singularity",
            "exec",
            "--pwd",  # Changes the working directory to the appropriate one
            simulation_dir,
            singularity_image,
            main_py_path,
            "--model", task,
            "--params", template_name,
            # This is a bit silly, but this way it's compatible with the structure of MachSuite tasks
            "--output-dir", os.path.join(simulation_dir, task, task)
            ],
            stdout=log_file,
            stderr=log_file,
            check=True,
        )

    print(f"Running simulation in {run_script_dir}!")

    with open(os.path.join(simulation_dir, simulation_output_file_name), "w") as log_file:
        try:
            result = subprocess.run([
                "singularity",
                "exec",
                "--pwd",
                run_script_dir,
                singularity_image,
                "bash",
                "run.sh"
            ],
                stdout=log_file,
                stderr=log_file,
                check=True,
            )

        except subprocess.CalledProcessError as e:
            stderr_path = os.path.join(simulation_dir, run_sh_subdir, stderr_path)
            log_file.writelines(["========================\n",
                                 f"Copied from {stderr_path}\n"
                                 "========================\n", ])

            with open(stderr_path) as stderr_log_file:
                stderr_log = stderr_log_file.read()

                log_file.write(stderr_log)

            print(str(e))
            raise ValueError(stderr_log)

    print(os.path.join(simulation_dir, run_sh_subdir, stdout_path))
    results = parse_file(os.path.join(simulation_dir, run_sh_subdir, stdout_path),
                         output_regexps)

    if task in smaug_tasks:
        results["total_time"] = parse_stats_txt(os.path.join(simulation_dir, run_sh_subdir, "outputs", "stats.txt"))

    with open(os.path.join(simulation_dir, results_file_name), "w") as results_file:
        json.dump(results, results_file)

    result_vec = get_output_vector_from_dict(results, output_order=output_order)

    return result_vec

def input_length():

    input_length = 0

    for spec in config["inputs"]:

        if spec.input_type == str:

            if len(spec.domain) == 2:
                input_length += 1

            else:
                input_length += len(spec.domain)

        else:

            input_length += 1

    return input_length

def run_experiment():
    # -------------------------------------------------------------------------
    # Setup
    # -------------------------------------------------------------------------
    seed = config["run"]["seed"]
    experiment_dir = config["run"]["experiment_dir"]
    task = config["run"]["task"]
    model = config["run"]["model"]

    warmup_dir = config["run"]["warmup_base_dir"]
    num_warmup_points = config['run']['num_warmup_points']

    machsuite_template_path = os.path.join(experiment_dir, "machsuite_template.xe")
    smaug_template_path = os.path.join(experiment_dir, "smaug_template.json")

    tf.random.set_seed(seed)

    grid = SobolGrid(dim_specs=config["inputs"],
                     seed=config["run"]["seed"],
                     num_grid_points=config["run"]["num_grid_points"],
                     save_dependency_graph_path=os.path.join(experiment_dir, "dependency_graph.png"))

    # In an experiment folder, this is the subdirectory where run.sh is located
    run_sh_subdir = f"{task}/{task}/0/"

    # -------------------------------------------------------------------------
    # Warmup
    # -------------------------------------------------------------------------
    warmup_xs_path = os.path.join(warmup_dir, f"warmup_xs_{num_warmup_points}.npy")
    warmup_ys_path = os.path.join(warmup_dir, f"warmup_ys_{num_warmup_points}.npy")

    if not os.path.exists(warmup_xs_path):

        print("Necessary warmup points don't exist yet, creating them now...")

        warmup_xs = grid.sample(num_warmup_points)

        warmup_ys = tf.zeros((0, len(config["outputs"])), dtype=tf.float64)

        # Create warmup sweep configs
        warmup_inputs_settings = grid.input_settings_from_points(points=warmup_xs)

        # Generate warmups
        for warmup_index, input_settings in enumerate(warmup_inputs_settings):

            warmup_dir_name = f"warmup_{warmup_index}"
            current_warmup_dir = os.path.join(experiment_dir, warmup_dir_name)

            current_warmup_xs_path = os.path.join(warmup_dir, f"warmup_xs_{warmup_index + 1}.npy")
            current_warmup_ys_path = os.path.join(warmup_dir, f"warmup_ys_{warmup_index + 1}.npy")

            if os.path.exists(current_warmup_ys_path):
                print(f"{current_warmup_dir}, already exists, skipping!")
                warmup_ys = tf.convert_to_tensor(np.load(current_warmup_ys_path))
                continue

            if task in machsuite_tasks:
                warmup_config = create_gem5_sweep_config(task=task,
                                                         template_file_path=machsuite_template_path,
                                                         output_dir=os.path.join(current_warmup_dir, task),
                                                         input_settings=input_settings,
                                                         generation_commands=("configs", "trace"))

                result_vec = run_machsuite_simulation_with_configuration(
                    task=task,
                    simulation_dir=current_warmup_dir,
                    run_sh_subdir=run_sh_subdir,
                    simulation_config=warmup_config,
                    output_order=config["outputs"],
                    singularity_image=config["run"]["singularity_image"],
                )

            elif task in smaug_tasks:
                warmup_config = create_smaug_sweep_config(template_file_path=smaug_template_path,
                                                          input_settings=input_settings)

                result_vec = run_smaug_simulation_with_configuration(
                    task=task,
                    simulation_dir=current_warmup_dir,
                    run_sh_subdir=run_sh_subdir,
                    simulation_config=warmup_config,
                    output_order=config["outputs"],
                    singularity_image=config["run"]["singularity_image"],
                )

            else:
                raise NotImplementedError

            warmup_ys = tf.concat([warmup_ys, result_vec[None, :]], axis=0)

            np.save(current_warmup_xs_path, warmup_xs.numpy())
            np.save(current_warmup_ys_path, warmup_ys.numpy())

            # Remove the simulation files after all important information has been extracted.
            # This is very important, because the simulation directory is effectively useless after,
            # can be regenerated easily and takes up a lot of space.
            shutil.rmtree(os.path.join(current_warmup_dir, task))

    else:
        print("Using precomputed warmup points!")
        warmup_xs = tf.convert_to_tensor(np.load(warmup_xs_path))
        warmup_ys = tf.convert_to_tensor(np.load(warmup_ys_path))

    # -------------------------------------------------------------------------
    # Setup for BO
    # TODO: Potential bug: if the target is a categorical. Disallow this with a check.
    # -------------------------------------------------------------------------

    # Reset the random seed, because depending on whether warmup points were generated or not,
    # we might get different results here
    tf.random.set_seed(seed)

    # Truncate the datapoints if necessary
    if model in ["ff-gp", "gpar-targets"]:
        warmup_ys = warmup_ys[:, -config["num_targets"]:]

    model_config = config["models"][model]

    input_dim = input_length()

    if model == "sgs":
        surrogate_model = RandomModel(input_dim=input_dim,
                                      output_dim=len(config["outputs"]),
                                      seed=seed,
                                      num_samples=10,
                                      verbose=True)
    elif model == "gpar-targets":
        surrogate_model = GPARModel(kernel=model_config["kernel"],
                                    input_dim=input_dim,
                                    output_dim=config["num_targets"],
                                    verbose=True)

    elif model == "gpar-full":
        surrogate_model = GPARModel(kernel=model_config["kernel"],
                                    input_dim=input_dim,
                                    output_dim=len(config["outputs"]),
                                    verbose=True)

    elif model == "ff-gp":
        surrogate_model = FullyFactorizedGPModel(kernel=model_config["kernel"],
                                                 input_dim=input_dim,
                                                 output_dim=config["num_targets"],
                                                 verbose=True)

    elif model == "bnn":
        surrogate_model = BasicBNN(input_dim=input_dim,
                                   output_dim=len(config["outputs"]))

    else:
        raise NotImplementedError

    acq_fun = SMSEGO(**config["run"]["acq_config"])

    surrogate_model = surrogate_model.condition_on(xs=warmup_xs,
                                                   ys=warmup_ys)

    if not os.path.exists(os.path.join(experiment_dir, "evaluation_0", "xs.npy")):
        train_surrogate_model(surrogate_model, model_config, config["run"]["model"])

    # -------------------------------------------------------------------------
    # BO loop
    # -------------------------------------------------------------------------

    for eval_index in range(config["run"]["max_num_evaluations"]):

        # Make sure every evaluation has its own unique seed
        tf.random.set_seed(seed + eval_index + 1)

        eval_dir_name = f"evaluation_{eval_index}"
        eval_dir = os.path.join(experiment_dir, eval_dir_name)

        grid_save_path = os.path.join(eval_dir, "grid.npy")

        xs_save_path = os.path.join(eval_dir, "xs.npy")
        ys_save_path = os.path.join(eval_dir, "ys.npy")

        if os.path.exists(ys_save_path):

            print(f"Evaluation {eval_index} at: {ys_save_path} already exists, skipping!")

            # If the grid is saved in this evaluation directory, it is the latest one.
            if os.path.exists(grid_save_path):
                print("Last saved model found")

                grid.points = tf.convert_to_tensor(np.load(grid_save_path))
                loaded_xs = tf.convert_to_tensor(np.load(xs_save_path))
                loaded_ys = tf.convert_to_tensor(np.load(ys_save_path))

                surrogate_model = surrogate_model.condition_on(loaded_xs, loaded_ys, keep_previous=False)

                tf.random.set_seed(seed + eval_index + 1)
                train_surrogate_model(surrogate_model, model_config, config["run"]["model"])

            continue

        print(f"{grid.points.shape[0]} points in the grid at iteration {eval_index + 1}")
        # ---------------------------------------------------------------------
        # Step 1: Evaluate acquisition function on every grid point
        # ---------------------------------------------------------------------
        grid_points = grid.index_to_one_hot(grid.points)

        print("1", grid.points.shape)

        acquisition_values, y_preds = acq_fun.evaluate(model=surrogate_model,
                                                       xs=surrogate_model.xs.numpy(),
                                                       ys=surrogate_model.ys.numpy(),
                                                       candidate_xs=grid_points.numpy())
        print("2", grid.points.shape)
        # ---------------------------------------------------------------------
        # Step 2: Find maximum of acquisition function
        # ---------------------------------------------------------------------
        pf_indices = pareto_frontier(tf.convert_to_tensor(y_preds[:, -config["num_targets"]:]))

        num_pareto_points = tf.reduce_sum(tf.cast(pf_indices, tf.int32))

        # Make sure we don't get too many gridpoints
        if grid.num_points() < 10**5:
            num_extra_samples_per_point = 1000 // num_pareto_points + 1

            pareto_xs = grid_points[pf_indices]

            # Try some extra points around the best solutions
            for pareto_x in pareto_xs:
                grid.sample_grid_around_point(pareto_x,
                                              num_samples=num_extra_samples_per_point,
                                              add_to_grid=True)

        else:
            num_extra_samples_per_point = 0

        print("3", grid.points.shape)

        print(
            f"{num_pareto_points} Pareto optimal points in grid, adding {num_extra_samples_per_point} samples around each.")

        acquisition_values_, y_preds_ = acq_fun.evaluate(model=surrogate_model,
                                                         xs=surrogate_model.xs.numpy(),
                                                         ys=surrogate_model.ys.numpy(),
                                                         candidate_xs=grid_points.numpy())

        max_acquisition_index_ = np.argmax(acquisition_values_)
        eval_x = grid_points[max_acquisition_index_]
        print("4", grid.points.shape)
        # ---------------------------------------------------------------------
        # Step 3: Run simulation at selected location
        # ---------------------------------------------------------------------
        eval_input_settings = grid.input_settings_from_points(eval_x)[0]

        if task in machsuite_tasks:

            eval_config = create_gem5_sweep_config(task=task,
                                                   template_file_path=machsuite_template_path,
                                                   output_dir=task,
                                                   input_settings=eval_input_settings,
                                                   generation_commands=("configs", "trace"))

            eval_y = run_machsuite_simulation_with_configuration(
                task=task,
                simulation_dir=eval_dir,
                run_sh_subdir=run_sh_subdir,
                simulation_config=eval_config,
                output_order=config["outputs"],
                singularity_image=config["run"]["singularity_image"],
            )

        elif task in smaug_tasks:
            eval_config = create_smaug_sweep_config(template_file_path=smaug_template_path,
                                                    input_settings=eval_input_settings)

            eval_y = run_smaug_simulation_with_configuration(
                task=task,
                simulation_dir=eval_dir,
                run_sh_subdir=run_sh_subdir,
                simulation_config=eval_config,
                output_order=config["outputs"],
                singularity_image=config["run"]["singularity_image"],
            )

        else:
            raise NotImplementedError

        # Chop off the first couple of dimensions for targets-only models
        if model in ["ff-gp", "gpar-targets"]:
            eval_y = eval_y[-config["num_targets"]:]

        # ---------------------------------------------------------------------
        # Step 4: Fit surrogate model to dataset augmented by the new datapoint
        # ---------------------------------------------------------------------
        surrogate_model = surrogate_model.condition_on(eval_x, eval_y)

        tf.random.set_seed(seed + eval_index + 1)
        train_surrogate_model(surrogate_model, model_config, config["run"]["model"])

        print("5", grid.points.shape)

        np.save(grid_save_path, grid.points.numpy())
        np.save(xs_save_path, surrogate_model.xs.numpy())
        np.save(ys_save_path, surrogate_model.ys.numpy())

        # Remove the simulation files AND the grid from the previous folder,
        # after all important information has been extracted.
        # This is very important, because the simulation directory is effectively useless after,
        # can be regenerated easily and takes up a lot of space.

        # Simulation directory
        shutil.rmtree(os.path.join(eval_dir, task))

        # Previous grid
        prev_grid_save_path = os.path.join(experiment_dir, f"evaluation_{eval_index - 1}", "grid.npy")

        if os.path.exists(prev_grid_save_path):
            os.remove(prev_grid_save_path)

    print("Experiment finised!")

if __name__ == "__main__":
    run_experiment()


# coding=utf-8
"""ABL Conditions and initialConditions class."""
from .foamfile import Condition, foamFileFromFile
from collections import OrderedDict
import math

class ABLConditions(Condition):
    """ABL Conditions."""

    # set default valus for this class
    __defaultValues = OrderedDict()
    __defaultValues['Uref'] = '0'   # wind velocity
    __defaultValues['Zref'] = '10'  # reference z value - usually 10 meters
    __defaultValues['z0'] = 'uniform 1'  # roughness - default is set to 1 for urban environment
    __defaultValues['flowDir'] = '(0 1 0)'  # direction of flow
    __defaultValues['zDir'] = '(0 0 1)'  # z direction (0 0 1) always for our cases
    __defaultValues['zGround'] = 'uniform 0'  # min z value of the bounding box
    __defaultValues['value'] = '$internalField'

    def __init__(self, values=None):
        """Init class."""
        super(ABLConditions, self).__init__(
            name='ABLConditions', cls='dictionary', location='0',
            defaultValues=self.__defaultValues, values=values
        )

    @classmethod
    def fromFile(cls, filepath):
        """Create a FoamFile from a file.

        Args:
            filepath: Full file path to dictionary.
        """
        return cls(values=foamFileFromFile(filepath, cls.__name__))

    @classmethod
    def fromInputValues(cls, flowSpeed, z0, flowDir, zGround):
        """Get ABLCondition."""
        _ABLCDict = {}
        _ABLCDict['Uref'] = str(flowSpeed)
        _ABLCDict['z0'] = 'uniform {}'.format(z0)
        _ABLCDict['flowDir'] = flowDir if isinstance(flowDir, str) \
            else '({} {} {})'.format(*flowDir)

        _ABLCDict['zGround'] = 'uniform {}'.format(zGround)
        return cls(_ABLCDict)

    @classmethod
    def fromWindTunnel(cls, windTunnel):
        """Init class from wind tunnel."""
        return cls(values=windTunnel.ABLConditionsDict)

    @property
    def flowDir(self):
        """Get flow dir as tuple (x, y, z)."""
        return eval(self.values['flowDir'])

    @property
    def flowSpeed(self):
        """Get flow speed as a float."""
        return float(self.values['Uref'])

class InitialConditions(Condition):
    """Initial conditions."""

    # set default valus for this class
    __defaultValues = OrderedDict()
    __defaultValues['flowVelocity'] = '(0 0 0)'
    __defaultValues['pressure'] = '0'
    __defaultValues['turbulentKE'] = None  # will be calculated based on input values
    __defaultValues['turbulentEpsilon'] = None  # will be calculated based on input values
    __defaultValues['#inputMode'] = 'merge'

    def __init__(self, values=None, Uref=0, Zref=10, z0=1, Cm=0.09, k=0.41):
        """Init class.

        Args:
            Uref: Reference wind velocity in m/s.
            Zref: Reference height for wind velocity. Normally 10 m.
            z0: Roughness (default: 1).
        """
        self.__Uref = float(Uref)
        self.__Zref = float(Zref)
        self.__z0 = float(z0)
        self.__Cm = Cm
        self.__k = k
        self.calculateKEpsilon(init=True)
        super(InitialConditions, self).__init__(
            name='initialConditions', cls='dictionary', location='0',
            defaultValues=self.__defaultValues, values=values
        )

    @classmethod
    def fromFile(cls, filepath):
        """Create a FoamFile from a file.

        Args:
            filepath: Full file path to dictionary.
        """
        return cls(values=foamFileFromFile(filepath, cls.__name__))

    def calculateKEpsilon(self, init=False):
        """Calculate turbulentKE and turbulentEpsilon.

        Args:
            init: True if the method is called when the class is initiated
                (default: False).
        """
        _Uabl = self.Uref * self.k / math.log((self.Zref + self.z0) / self.z0)
        epsilon = _Uabl ** 3 / (self.k * (self.Zref + self.z0))
        k = _Uabl ** 2 / math.sqrt(self.Cm)

        if init:
            self.__defaultValues['turbulentKE'] = '%.5f' % k
            self.__defaultValues['turbulentEpsilon'] = '%.5f' % epsilon
        else:
            self.values['turbulentKE'] = str(k)
            self.values['turbulentEpsilon'] = str(epsilon)

    @property
    def Uref(self):
        """Input velocity in m/s."""
        return self.__Uref

    @Uref.setter
    def Uref(self, value):
        """Input velocity in m/s."""
        self.__Uref = float(value)
        self.calculateKEpsilon()

    @property
    def Zref(self):
        """Input height reference for input velocity in meters."""
        return self.__Zref

    @Zref.setter
    def Zref(self, value):
        self.__Zref = float(value)
        self.calculateKEpsilon()

    @property
    def z0(self):
        """Roughness."""
        return self.__z0

    @z0.setter
    def z0(self, value):
        self.__z0 = float(value)
        self.calculateKEpsilon()

    @property
    def Cm(self):
        """Cm.

        default: 0.09
        """
        return self.__Cm

    @Cm.setter
    def Cm(self, value):
        self.__Cm = float(value)
        self.calculateKEpsilon()

    @property
    def k(self):
        """k.

        default: 0.41
        """
        return self.__k

    @k.setter
    def k(self, value):
        self.__k = float(value)
        self.calculateKEpsilon()


import os
import hmac
import json
import secrets
import asks
from contextlib import contextmanager

# Partial duplicate of gidgethub.sansio.validate_event
def sign_webhook(body: bytes, secret: str):
    hmaccer = hmac.new(secret.encode("ascii"), msg=body, digestmod="sha1")
    sig = "sha1=" + hmaccer.hexdigest()
    return sig

def fake_webhook(event_type, payload, secret):
    body = json.dumps(payload).encode("ascii")
    headers = {
        "x-github-event": event_type,
        "content-type": "application/json",
        "x-github-delivery": secrets.token_hex(16),
    }
    if secret is not None:
        headers["x-hub-signature"] = sign_webhook(body, secret)
    return headers, body

@contextmanager
def save_environ():
    saved_env = dict(os.environ)
    try:
        yield
    finally:
        os.environ.clear()
        os.environ.update(saved_env)


from collections import deque

with open("puzzle.txt", "r") as puzzle:
    lines = puzzle.read()

def part_1() -> int:
    deck = deque(maxlen=4)
    input = iter(lines)
    letter_no = 0
    while True:
        if len(deck) == 4 and len(set(deck)) == 4:
            return letter_no
        else:
            deck.append(next(input))
            letter_no += 1

def part_2() -> int:
    input = iter(lines)
    letter_no = 14
    marker = [next(input) for _ in range(14)]
    while True:
        if len(set(marker)) == 14:
            return letter_no
        else:
            del marker[0]
            marker.append(next(input))
            letter_no += 1

print("Part 1 =", part_1())
print("Part 2 =", part_2())


#!/usr/bin/python3
# -*- coding:utf-8 -*-

import json
import time

from kubernetes import client, config

config.load_kube_config()

def create_namespaced_cron_job(namespace='default', body=None):
    cronjob_json = body
    if body is None:
        print('body is required!')
        exit(0)
    name = body['metadata']['name']
    if judge_crontab_exists(namespace, name):
        print(f'{name} exists, please do not repeat!')
    else:
        v1 = client.BatchV1Api()
        ret = v1.create_namespaced_cron_job(namespace=namespace, body=cronjob_json, pretty=True,
                                            _preload_content=False, async_req=False)
        ret_dict = json.loads(ret.data)
        print(f'create succeed\n{json.dumps(ret_dict)}')

def delete_namespaced_cron_job(namespace='default', name=None):
    if name is None:
        print('name is required!')
        exit(0)
    if not judge_crontab_exists(namespace, name):
        print(f"{name} doesn't exists, please enter a new one!")
    else:
        v1 = client.BatchV1Api()
        ret = v1.delete_namespaced_cron_job(name=name, namespace=namespace, _preload_content=False, async_req=False)
        ret_dict = json.loads(ret.data)
        print(f'delete succeed\n{json.dumps(ret_dict)}')

def patch_namespaced_cron_job(namespace='default', body=None):
    cronjob_json = body
    if body is None:
        print('body is required!')
        exit(0)
    name = body['metadata']['name']
    if judge_crontab_exists(namespace, name):
        v1 = client.BatchV1Api()
        ret = v1.patch_namespaced_cron_job(name=name, namespace=namespace, body=cronjob_json,
                                           _preload_content=False, async_req=False)
        ret_dict = json.loads(ret.data)
        print(f'patch succeed\n{json.dumps(ret_dict)}')
    else:
        print(f"{name} doesn't exists, please enter a new one!")

def get_cronjob_list(namespace='default'):
    v1 = client.BatchV1Api()
    ret = v1.list_namespaced_cron_job(namespace=namespace, pretty=True, _preload_content=False)
    cron_job_list = json.loads(ret.data)
    print(f'cronjob number={len(cron_job_list["items"])}')
    return cron_job_list["items"]

def judge_crontab_exists(namespace, name):
    cron_job_list = get_cronjob_list(namespace)
    for cron_job in cron_job_list:
        if name == cron_job['metadata']['name']:
            return True
    return False

def get_cronjob_body(namespace, name, command):
    body = {
        "apiVersion": "batch/v1",
        "kind": "CronJob",
        "metadata": {
            "name": name,
            "namespace": namespace
        },
        "spec": {
            "schedule": "*/1 * * * *",
            "concurrencyPolicy": "Allow",
            "suspend": False,
            "jobTemplate": {
                "spec": {
                    "template": {
                        "spec": {
                            "containers": [
                                {
                                    "name": name,
                                    "image": "busybox:1.35",
                                    "command": command
                                }
                            ],
                            "restartPolicy": "Never"
                        }
                    }
                }
            },
            "successfulJobsHistoryLimit": 3,
            "failedJobsHistoryLimit": 1
        }
    }
    return body

if __name__ == '__main__':
    # get
    cronjob_list = get_cronjob_list()

    # delete
    delete_namespaced_cron_job('default', 'hostname')
    time.sleep(2)

    # create
    container_command = [
        "/bin/sh",
        "-c",
        "date; echo Hello from the Kubernetes cluster; hostname"
    ]
    hostname_json = get_cronjob_body('default', 'hostname', container_command)
    create_namespaced_cron_job('default', hostname_json)

    # update
    container_command[2] = "date; echo this is patch; hostname"
    hostname_json = get_cronjob_body('default', 'hostname', container_command)
    patch_namespaced_cron_job('default', hostname_json)


"""
Global constants
"""
import pygame
from pygame import freetype

pygame.font.init()
pygame.freetype.init()
pygame.mixer.pre_init(44100, -16, 1, 512)
pygame.mixer.init()

# Colors
BLUE = (0, 0, 255)

WHITE = (255, 255, 255)
BLACK = (0, 0, 0)

GREY = (200, 200, 200)

RED = (200, 0, 0)
LIGHT_RED = (255, 0, 0)

YELLOW = (200, 200, 0)
LIGHT_YELLOW = (255, 255, 0)

GREEN = (0, 155, 0)
LIGHT_GREEN = (0, 255, 0)

with open("config/graphics.ini", "r") as f:
    lines = f.read().splitlines()
    lines = list(map(lambda s: s.split("#")[0], lines))
    TITLE = lines[0]
    WIDTH, HEIGHT = tuple(map(int, lines[1].split()))
    FPS = int(lines[2])

# Tile size
TILE_SIZE = 32

GRID_WIDTH = WIDTH / TILE_SIZE
GRID_HEIGHT = HEIGHT / TILE_SIZE

GRAVITY = 5

all_sprites = pygame.sprite.Group()
entities = pygame.sprite.Group()
tiles = pygame.sprite.Group()

# Different text sizes for in menus
small_font = pygame.freetype.SysFont(pygame.font.get_default_font(), 25)
medium_font = pygame.freetype.SysFont(pygame.font.get_default_font(), 50)
large_font = pygame.freetype.SysFont(pygame.font.get_default_font(), 80)

small_font_old = pygame.font.SysFont("arial", 25)
medium_font_old = pygame.font.SysFont("arial", 50)
large_font_old = pygame.font.SysFont("arial", 80)


import re

import openpyxl
from playwright.sync_api import Page
from auction_scraper_base import browser_context, get_tenure
from auction_scraper_base import get_property_type, prepare_price, get_bedroom

start_url = "https://bidx1.com/en/ireland?division=80&region=1&maxprice=&page=1"

def property_scraper(detail_url_page, auction_house):
    propertyLink = f'https://bidx1.com{auction_house}'
    detail_url_page.goto(propertyLink)
    print(propertyLink)

    image_link = detail_url_page.locator("(//img[contains(@class,'d-block w-100')])[2]")
    print(image_link)

    guidePrice = detail_url_page.locator("xpath=//p[contains(text(),'Guide Price')]").text_content()
    price, currency = prepare_price(guidePrice)

    description = detail_url_page.locator("xpath=(//div[@class='content container'])[1]").text_content()
    print(description)

    address = detail_url_page.locator("xpath=//h2[@class='m-0 order-1 order-lg-0 ']").text_content()
    postal_code = address.split(',')[-1]

    numbers_of_bedrooms = detail_url_page.locator("xpath=//div[@id='elevator-pitch']/ul").text_content()
    number_of_bedrooms = get_bedroom(numbers_of_bedrooms)
    print(number_of_bedrooms)

    tenure = detail_url_page.locator("xpath=(//div[@id='elevator-pitch']//strong)[1]").text_content()

    if number_of_bedrooms is None:
        number_of_bedrooms = get_bedroom(description)

    property_type = get_property_type(description)

    if property_type == "other" and "land" in address.lower():
        property_type = "land"
    if "land" in address.lower():
        address = re.search(r"(?<= to ).*|(?<= of ).*|(?<= at ).*", address, re.IGNORECASE)
        if address:
            address = address.group().strip()

    if not tenure:
        tenure = get_tenure(description)

    data_hash = {
        "price": price,
        "currency_type": currency,
        "picture_link": image_link,
        "property_description": description,
        "property_link": propertyLink,
        "address": address,
        "postal_code": postal_code,
        "auction_venue": "online",
        "source": "auctionhouselondon",
        "property_type": property_type,
        "number_of_bedrooms": number_of_bedrooms,
        "tenure": tenure,
    }

def properties_scraper(page, browser):
    auction_divs = []
    anchor_locator = page.locator("//a[@class='disable-decoration d-flex flex-column flex-fill']")

    for i in range(anchor_locator.count()):
        property_href = anchor_locator.nth(i)
        property_card = property_href.get_attribute("href")
        auction_divs.append(property_card)

    detail_url_page: Page = browser.new_page()

    for auction_house in auction_divs:
        property_scraper(detail_url_page, auction_house)

def run():
    with browser_context(headless=False) as (page, browser):
        page.set_default_navigation_timeout(120000)
        page.goto(start_url)
        properties_scraper(page, browser)

if __name__ == "__main__":
    run()