import pandas as pd
import matplotlib.pyplot as plt

df_tesla = pd.read_csv('TESLA Search Trend vs Price.csv')

df_btc_search = pd.read_csv('Bitcoin Search Trend.csv')
df_btc_price = pd.read_csv('Daily Bitcoin Price.csv')

df_unemployment = pd.read_csv('UE Benefits Search vs UE Rate 2004-19.csv')

# Challenge:

# 1.What are the shapes of the dataframes?
# 2. How many rows and columns?
# 3. What are the column names?
# 4. Complete the f-string to show the largest/smallest number in the search data column
# 5. Try the .describe() function to see some useful descriptive statistics
# 6. What is the periodicity of the time series data (daily, weekly, monthly)?
# 7. What does a value of 100 in the Google Trend search popularity actually mean?

# Tesla
# 1, 2, 3
print(df_tesla.shape)
df_tesla.head()

# 4
print(f'Largest value for Tesla in Web Search: {df_tesla["TSLA_WEB_SEARCH"].max()}')
print(f'Smallest value for Tesla in Web Search: {df_tesla["TSLA_WEB_SEARCH"].min()}')

# 5
df_tesla.describe()

# Unemployment

df_unemployment.shape
df_unemployment.head()

print('Largest value for "Unemployemnt Benefits" '
      f'in Web Search: {df_unemployment["UE_BENEFITS_WEB_SEARCH"].max()}')

#Bitcoin

df_btc_price.shape
df_btc_price.head()

df_btc_search.shape
df_btc_search.head()

print(f'largest BTC News Search: {df_btc_search["BTC_NEWS_SEARCH"].max()}')

# Challenge: Are there any missing values in any of the dataframes? If so,
# which row/rows have missing values? How many missing values are there?

print(f'Missing values for Tesla?: {df_tesla.isna().values.any()}')
print(f'Missing values for U/E?: {df_unemployment.isna().values.any()}')
print(f'Missing values for BTC Search?: {df_btc_search.isna().values.any()}')

# This one has some
print(f'Missing values for BTC price?: {df_btc_price.isna().values.any()}')
# Two to be exact
print(f'Number of missing values: {df_btc_price.isna().values.sum()}')

# Challenge: Remove any missing values that you found.
df_btc_price = df_btc_price.dropna()

# Challenge
# Our DataFrames contain time-series data.
# Do you remember how to check the data type of the entries in the DataFrame?
# Have a look at the data types of the MONTH or DATE columns.
# Convert any strings you find into Datetime objects.
# Do this for all 4 DataFrames. Double-check if your type conversion was successful.

df_tesla.dtypes

df_tesla.MONTH = pd.to_datetime(df_tesla.MONTH)
df_btc_search.MONTH = pd.to_datetime(df_btc_search.MONTH)
df_unemployment.MONTH = pd.to_datetime(df_unemployment.MONTH)
df_btc_price.DATE = pd.to_datetime(df_btc_price.DATE)

# Challenge
# Plot the Tesla stock price against the Tesla search volume using a line chart and
# two different axes.

ax1 = plt.gca()
ax2 = ax1.twinx()

ax1.set_ylabel('TSLA Stock Price')
ax2.set_ylabel('Search Trend')

ax1.plot(df_tesla.MONTH, df_tesla.TSLA_USD_CLOSE)
ax2.plot(df_tesla.MONTH, df_tesla.TSLA_WEB_SEARCH)

# Challenge
# Try using one of the blue colour names for the search volume and
#  a HEX code for a red colour for the stock price.

ax1 = plt.gca()
ax2 = ax1.twinx()

ax1.set_ylabel('TSLA Stock Price', color='#E6232E') # can use a HEX code
ax2.set_ylabel('Search Trend', color='skyblue') # or a named colour

ax1.plot(df_tesla.MONTH, df_tesla.TSLA_USD_CLOSE, color='#E6232E')
ax2.plot(df_tesla.MONTH, df_tesla.TSLA_WEB_SEARCH, color='skyblue')

# Challenge
# There are still some ways to improve the look of this chart. First off, let's make it larger. Can you make the following changes:

# Increase the figure size (e.g., to 14 by 8).

# Increase the font sizes for the labels and the ticks on the x-axis to 14.

# Rotate the text on the x-axis by 45 degrees.

# Add a title that reads 'Tesla Web Search vs Price'

# Make the lines on the chart thicker.

# Keep the chart looking sharp by changing the dots-per-inch or DPI value.

# Set minimum and maximum values for the y and x-axis. Hint: check out methods like set_xlim().

# Finally use plt.show() to display the chart below the cell instead of relying on the automatic notebook output.

plt.figure(figsize=(14,8), dpi=120)
plt.title('Tesla Web Search vs Price', fontsize=18)

# Increase the size and rotate the labels on the x-axis
plt.xticks(fontsize=14, rotation=45)

ax1 = plt.gca()
ax2 = ax1.twinx()

ax1.set_ylabel('TSLA Stock Price', color='#E6232E', fontsize=14)
ax2.set_ylabel('Search Trend', color='skyblue', fontsize=14)

# Set the minimum and maximum values on the axes
ax1.set_ylim([0, 600])
ax1.set_xlim([df_tesla.MONTH.min(), df_tesla.MONTH.max()])

ax1.plot(df_tesla.MONTH, df_tesla.TSLA_USD_CLOSE, color='#E6232E', linewidth=3)
ax2.plot(df_tesla.MONTH, df_tesla.TSLA_WEB_SEARCH, color='skyblue', linewidth=3)

plt.show()

# Challenge
# Plot the search for "unemployment benefits" against the official unemployment rate.

# Change the title to: Monthly Search of "Unemployment Benefits" in the U.S. vs the U/E Rate

# Change the y-axis label to: FRED U/E Rate

# Change the axis limits

# Add a grey grid to the chart to better see the years and the U/E rate values. Use dashed lines for the line style.

# Can you discern any seasonality in the searches? Is there a pattern?

plt.figure(figsize=(14,8), dpi=120)
plt.title('Monthly Search of "Unemployment Benefits" in the U.S. vs the U/E Rate', fontsize=18)
plt.yticks(fontsize=14)
plt.xticks(fontsize=14, rotation=45)

ax1 = plt.gca()
ax2 = ax1.twinx()

ax1.set_ylabel('FRED U/E Rate', color='purple', fontsize=14)
ax2.set_ylabel('Search Trend', color='skyblue', fontsize=14)

# ax1.xaxis.set_major_locator(years)
# ax1.xaxis.set_major_formatter(years_fmt)
# ax1.xaxis.set_minor_locator(months)

ax1.set_ylim(bottom=3, top=10.5)
ax1.set_xlim([df_unemployment.MONTH.min(), df_unemployment.MONTH.max()])

# Show the grid lines as dark grey lines
ax1.grid(color='grey', linestyle='--')

# Change the dataset used
ax1.plot(df_unemployment.MONTH, df_unemployment.UNRATE,
         color='purple', linewidth=3, linestyle='--')
ax2.plot(df_unemployment.MONTH, df_unemployment.UE_BENEFITS_WEB_SEARCH,
         color='skyblue', linewidth=3)

plt.show()

Advanced/Day75/Google Trends Data Viz (start)/main.py


from django.db import router
from django.urls import path
from rest_framework.routers import DefaultRouter

from .views import *

app_name = "workspaces"

urlpatterns = [
    path("my-list", MyWorkspaceList.as_view(), name="my_workspace_list"),
    path("create", MyWorkspaceCreate.as_view(), name="my_workspace_list"),
    path("join", JoinWorkspaceView.as_view(), name="join_workspace"),
    path("shared", SharedWorkspaceListView.as_view(), name="shared_workspace"),
    path("members/<int:pk>", MemberListView.as_view(), name="member_List"),
    path("<int:pk>", MyWorkspaceRetrieve.as_view(), name="my_workspace_retrieve"),
    # folder
    path("folder-list/<int:pk>", WorkspaceFolderListView.as_view(), name="folder_list"),
    path("folder/<int:pk>", WorkspaceFolderModifyView.as_view(), name="folder_modify"),
    path("folder", WorkspaceFolderCreate.as_view(), name="folder_create"),
    # uploaded Files
    path("upload-file", WorkspaceUploadFileCreate.as_view(), name="workspace_upload_file"),
    path("upload-file/<int:pk>", WorkspaceUploadFileModify.as_view(), name="workspace_upload_modify"),
    path("upload-file-list/<int:pk>", WorkspaceUploadFileList.as_view(), name="workspace_upload_file_list"),
    # quill files
    path("quill-file", WorkspaceQuillFileCreate.as_view(), name="workspace_quill_file"),
    path("quill-file/<int:pk>", WorkspaceQuillFileModify.as_view(), name="workspace_quill_modify"),
    path("quill-file-list/<int:pk>", WorkspaceQuillFileList.as_view(), name="workspace_quill_file_list"),
]


from nbformat import read
from rest_framework import serializers
from .models import Student

# Type3- validators
def start_with_r(value):
    if value[0].lower()!='r':
        raise serializers.ValidationError("Name Should start with R")

# #only this can handle serializers
# class StudentSerializer(serializers.ModelSerializer):
#     class Meta:
#         model = Student
#         fields = ['name','roll','city']

class StudentSerializer(serializers.ModelSerializer):
    # name = serializers.CharField(read_only=True) # to make this field read only -->method 1
    name = serializers.CharField(validators=[start_with_r]) # to assign a validator
    class Meta:
        model = Student
        fields = ['name','roll','city']
        #read_only_fields = ['name','roll'] # to make this field read only -->method 2
       # extra_kwargs = {'name':{'read_only':True}} # to make this field read only -->method 3

    # Type1 validation - field level validation
    # def validate_roll(self,value):
    #     if value>=200:
    #         raise serializers.ValidationError("Seats full")
    #     return value

    #Type2 - object validation
    def validate(self, data):
        name = data.get('name')
        roll = data.get('roll')
        city = data.get('city')
        if name.lower()=="dinesh" and city.lower()!='bhimsar':
            raise serializers.ValidationError("Dinesh is from bhimsar.pls fill it correctly")
        return data

    #     if name[0:2]!="19" or name[2:5]!="BCE" or len(name)!=9:
    #         raise serializers.ValidationError("name should be of type 19BCEXXXX")
    #     return data


from typing import Type, Mapping, Dict, Collection
from hardsync.interfaces import Encoding, Exchange
from hardsync.types import FieldNotFoundError, ResponseValues, DecodedExchange, Stringable
from dataclasses import Field, fields

class AsciiEncoding(Encoding):
    argument_delimiter = ','
    argument_beginner = '('
    argument_ender = ')'
    argument_assigner = '='
    exchange_terminator = "\n"

    @staticmethod
    def encode(exchange: Type[Exchange], values: Mapping[str, Stringable | str], is_request=True) -> bytes:
        encoded_args = AsciiEncoding._encode_args(exchange=exchange, values=values)
        if is_request:
            r = 'Request'
        else:
            r = 'Response'
        encoded_function = exchange.identifier() + r
        encoded_string = encoded_function + '(' + encoded_args + ')' + AsciiEncoding.exchange_terminator
        return encoded_string.encode('ascii')

    @staticmethod
    def decode(exchange: Type[Exchange], contents: bytes) -> DecodedExchange:
        decoded_str = contents.decode('ascii')
        decoded_args = AsciiEncoding._decode_args(contents=decoded_str, exchange=exchange)
        decoded_name = AsciiEncoding._decode_name(contents=decoded_str)
        return DecodedExchange(name=decoded_name, values=decoded_args)

    @staticmethod
    def _encode_args(exchange: Type[Exchange], values: Mapping[str, Stringable | str]) -> str:
        encoded_args = ''
        for key, val in values.items():
            encoded_args += f'{key}{AsciiEncoding.argument_assigner}{val}{AsciiEncoding.argument_delimiter}'
        encoded_args = encoded_args.removesuffix(AsciiEncoding.argument_delimiter)
        return encoded_args

    @staticmethod
    def _arg_string(contents: str) -> str:
        start = contents.index(AsciiEncoding.argument_beginner)
        stop = contents.index(AsciiEncoding.argument_ender)
        return contents[start+1:stop]

    @staticmethod
    def _decode_args(exchange: Type[Exchange], contents: str, request=True) -> Dict[str, Stringable | str]:
        inner_string = AsciiEncoding._arg_string(contents=contents)
        args = inner_string.split(AsciiEncoding.argument_delimiter) if inner_string else []
        values = {}
        if request:
            decode_fields = fields(exchange.Request)
        else:
            decode_fields = fields(exchange.Response)

        for arg in args:
            key, value = arg.split(AsciiEncoding.argument_assigner)
            target_type = AsciiEncoding._lookup_field_type(name=key, available_fields=decode_fields)
            values[key] = target_type(value)

        return values

    @staticmethod
    def _lookup_field_type(name: str, available_fields: Collection[Field]):
        for field in available_fields:
            if field.name == name:
                return field.type
        raise FieldNotFoundError('Could not find field {name} in exchange')

    @staticmethod
    def _decode_name(contents: str) -> str:
        end = contents.index(AsciiEncoding.argument_beginner)
        return contents[:end]


import random
import matplotlib.pyplot as plt
import numpy as np

def TCPW(q, p, c, h, week):
    # initialization
    x_inv = q;  # inventory level
    x_dis = 0;  # remaining failure days
    T = 0;  # total cost
    PURCHASEPERMITTED = True;

    # update process
    for iter in range(week):
        if x_dis > 0:
            x_dis -= 1;
            PURCHASEPERMITTED = False;
        else:
            PURCHASEPERMITTED = True;

        if x_inv > 0:
            x_inv -= 1;
        else:
            T += p;

        if PURCHASEPERMITTED == True:
            u = random.uniform(0, 1);
            if u < 0.95:
                if x_inv <= q - 2:
                    x_inv += 2;
                    T += 2 * c;
                elif x_inv == q - 1:
                    x_inv += 1;
                    T += c;
            else:
                x_dis = 19;
        T += h * x_inv
    return T/week;
# factory produce equipment Model
# aim: determine the optimal inventory quantity threshold q

#-------------------------------------------------------------------------
#-------------------------------------------------------------------------

# total weeks
week = 1000000;
# threshold inventory quantity
Q = [i+1 for i in range(100)];
# parameter
p = 20;
c = 1;
h = 0.1;
# record for average cost
record_aver = [0] * len(Q);

count = 0;

for q in Q:
    aver_T = TCPW(q, p, c, h, week);
    record_aver[count] = aver_T;
    count += 1;

idx_opt = record_aver.index(min(record_aver));
quantity_opt = Q[idx_opt];
print('the optimal quantity is: ', quantity_opt);
print('the minimum average cost (per week) is: ', min(record_aver));

fig = plt.figure();
ax = fig.add_subplot(1,1,1);
ax.plot(Q, record_aver);
plt.title('Total Cost Per Week of each estimator q');
plt.xlabel('quantity');
plt.ylabel('Total Cost Per Week')
plt.show()


#!/usr/local/bin/python3
# -*- coding: utf-8 -*-

import os

import numpy as np
import pandas as pd

from sklearn import metrics
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression

import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

"""
paper figure 6.
"""

def generate_dataset_from_csv(target_name='Ea', fea_names='all'):
    """
    Description:
        Return a dictionary with mechanism types, target(activation energy),
        features(adsorption energies and geometrical descriptors) and their names.
    """
    # from csv file to dataframe
    if os.path.exists(DS_CSV):
        df = pd.read_csv(DS_CSV, index_col=0)
    else:
        raise ValueError('DataSet CSV does not exist.')

    # choose different targets: preferred or specific activation energy (three kinds)
    if target_name == 'Ea':
        df = df
        E = 'mE'
    elif target_name == 'Edelta':
        df = df.loc[df.loc[:,'E_ts']!='np.nan', :]
        df = df.loc[df.loc[:,'E_tsra']!='np.nan', :]
        df.loc[:,'E_ts'] = df.loc[:,'E_ts'].values.astype(np.float64) - \
                df.loc[:,'E_tsra'].values.astype(np.float64)
        E = 'E_ts'
    elif target_name == 'Ets':
        df = df.loc[df.loc[:,'E_ts']!='np.nan', :]
        E = 'E_ts'
    elif target_name == 'Etsra':
        df = df.loc[df.loc[:,'E_tsra']!='np.nan', :]
        E = 'E_tsra'

    # 'name', 'mtype', 'E_ts', 'E_tsra', Ea: min(E_ts, E_tsra)
    nodescriptors = ['name', 'mtype', 'E_ts', 'E_tsra', 'Ea']
    indexs_cols = df.iloc[:, 0:len(nodescriptors)]

    # choose number of descriptors (E_Hab3, E_CH3ab, Geos, ...)
    if fea_names == 'all':
        vals_cols = df.iloc[:,range(len(nodescriptors),len(df.columns))]
    else:
        vals_cols = df.loc[:, fea_names]

    # Get DataSet
    DS = {}

    DS['Etype'] = indexs_cols.loc[:, 'mtype'].values
    DS['target'] = indexs_cols.loc[:,E].values.reshape(-1,1).astype(np.float64)
    DS['fea_names'] = vals_cols.columns.values
    DS['features'] = vals_cols.values.astype(np.float64)

    return DS

def stardard_scaling(y, X):
    """
    Description:
        StandardScaler Data.
    In:
        unnormalized data y and X.
    Out:
        normalized data: y_s and X_s.
        scaler function: s_y and s_X
    """
    # scale y
    scaler4y = StandardScaler().fit(y)
    y_scalered = scaler4y.transform(y)

    # scale X
    scaler4X = StandardScaler()
    X_scalered = scaler4X.fit_transform(X)

    return y_scalered, X_scalered, scaler4y, scaler4X

def randomize_folds(y, splits):
    """"""
    kf = KFold(n_splits=splits, shuffle=True, random_state=0)
    folds_index = []
    for train_index, test_index in kf.split(y):
        folds_index.append([train_index, test_index])

    return folds_index

def linear_regression_with_cv(y, X):
    # generate k-folds
    n = 5
    folds_index = randomize_folds(y, n)
    MSEs, r2s = [], []
    ols = LinearRegression(fit_intercept=True, normalize=False, n_jobs=-1)
    for i in range(n):
        # get train and test set
        fold = folds_index[i]
        train_index, test_index = fold[0], fold[1]
        y_train, X_train = y[train_index], X[train_index]
        y_test, X_test = y[test_index], X[test_index]

        # scale data
        y_s, X_s, s4y, s4X = stardard_scaling(y_train, X_train)

        # train and test
        ols.fit(X_s, y_s)

        y_test_s, X_test_s = s4y.transform(y_test), s4X.transform(X_test)
        y_p = ols.predict(X_test_s)

        MSE = metrics.mean_squared_error(y_test_s, y_p)
        MSEs.append(MSE)

        r2 = metrics.r2_score(y_test_s, y_p)
        r2s.append(r2)

    # mean MSE and standard MSE
    MSEs = np.array(MSEs)

    return MSEs, r2s

def feature_selection(Etype, features):
    """"""
    # prepare data
    DS = generate_dataset_from_csv(Etype, features.keys())
    y, X = DS['target'], DS['features']

    # feature selection
    mean_MSEs, std_MSEs, mean_r2s, std_r2s = [], [], [], []
    feas_number = len(X[0])

    for i in range(feas_number):
        X_slice = []
        for j in range(i+1):
            X_slice.append(X.T[j])
        X_slice = np.array(X_slice).T
        MSEs, r2s = linear_regression_with_cv(y, X_slice)

        mean_MSEs.append(np.mean(MSEs))
        std_MSEs.append(np.var(MSEs))

        mean_r2s.append(np.mean(r2s))
        std_r2s.append(np.var(r2s))

    return mean_MSEs, std_MSEs, mean_r2s, std_r2s

def plot_selected_features(ax, features, feanames):
    """"""
    # figure general setting
    nfeatures = len(features.keys())

    ax.set_title('Descriptors Selected by Lasso', fontsize=16)

    ax.set_yticks(np.arange(nfeatures))
    ax.set_yticklabels(feanames, fontsize=10, fontweight='black')

    ax.set_xlabel('Coefficient', fontsize=16)
    ax.set_xlim(0, 1.0)

    # horizental bar
    coefs = np.array(list(features.values()))
    pos1 = ax.barh(np.arange(nfeatures), np.fabs(coefs), 0.5, \
            color='b', label='Positive', alpha=0.8)

    neg_features = {}
    for i, coef in enumerate(coefs):
        if coef <= 0e-6:
            neg_features[i] = np.fabs(coef)

    neg1 = ax.barh([int(i) for i in neg_features.keys()], \
            list(neg_features.values()), 0.5, \
            color='r', label='Negative', alpha=0.8)

    for i, coef in enumerate(features.values()):
        ax.text(np.fabs(coef), i, '%.2f' %coef, \
                fontsize=12, fontweight='bold')

    ax.legend(handles=[pos1, neg1], loc=1)

def plot_feature_selection(ax, Etype, features):
    """"""
    ax.set_title('Feature Selection by Linear Regression', fontsize=16)

    ax.set_xlabel('First N Features', fontsize=16)

    ax.set_ylabel('Statistics', fontsize=16)
    ax.set_ylim(-0.2, 1.0)

    mean_MSEs, std_MSEs, mean_r2s, std_r2s = feature_selection(Etype, features)

    mr2 = ax.scatter(range(1, len(mean_r2s)+1), mean_r2s, \
            color='g', marker='o', label='Mean $R^2$')
    mmse = ax.scatter(range(1, len(mean_MSEs)+1), mean_MSEs, \
            color='r', marker='o', label='Mean MSE')

    '''
    sr2 = ax.scatter(range(1, len(std_r2s)+1), std_r2s, \
            color='b', marker='x', label='Variance $R^2$')
    smse = ax.scatter(range(1, len(std_MSEs)+1), std_MSEs, \
            color='r', marker='x', label='Variance MSE')
    '''

    ax.legend(handles=[mr2, mmse], loc=5)

def plot_best_regression(ax, Etype, features, first_n):
    """"""
    DS = generate_dataset_from_csv(Etype, list(features.keys())[:first_n])
    y, X = DS['target'], DS['features']
    y_s, X_s, s4y, s4X = stardard_scaling(y, X)

    ols = LinearRegression(fit_intercept=True, normalize=False, n_jobs=-1)
    ols.fit(X_s, y_s)

    y_p = s4y.inverse_transform(ols.predict(X_s))
    r2 = metrics.r2_score(y, y_p)
    mse = metrics.mean_squared_error(y, y_p)

    ax.set_title('First %d Descriptors Linear Regression' %first_n, fontsize=16)

    ax.set_xlabel('True Values / eV', fontsize=16)
    ax.set_xlim(-0.75, 1.75)

    ax.set_ylabel('Predicted Values / eV', fontsize=16)
    ax.set_ylim(-0.75, 1.75)

    ax.scatter(y, y_p)
    ax.plot([-0.5, 1.5], [-0.5, 1.5], 'k--', lw=4)
    ax.text(-0.5, 1.0, '${R^2}$=%0.2f, MSE=%0.2f' %(r2, mse), \
            fontsize=12)

def plot_main():
    """"""
    # figure general setting
    fig, ax = plt.subplots(2, 3, figsize=(16, 9))
    plt.tight_layout(pad=2.0, w_pad=2.0, h_pad=2.0)
    plt.subplots_adjust(left=0.1, bottom=None, right=0.90, top=0.85, \
            wspace=0.3, hspace=0.5)

    plt.suptitle('Feature Selection of Geometrical Descriptors', \
            fontsize=24, fontweight='bold')

    # ts-ss part
    ''' old
    features = {'E_CH3ab': 0.84, 'E_Hab3': 0.50, \
            'a_O2-M4-C_CH3ab': 0.41, 'd_H1-C_CH3ab': -0.32, \
            'h_O2-C-M4-H1_CH3ab': 0.30, 'h_O2-H1-M4-C_CH3ab': -0.29, \
            'a_O2-M4-H1_Hab3': -0.24, 'a_O2-C-H1_CH3ab': -0.13, \
            'a_M4-C-H1_CH3ab': 0.11, 'a_O2-H1-M4_Hab3': 0.05, \
            'd_O2-C_CH3ab': 0.02}

    feanames = ['$E_{{CH}_3^*}$', '$E_{H^*}$', \
            '$A_{O_{br}-M_{cus}-C}^{CH_3^*}$', '$D_{H-C}^{CH_3^*}$', \
            '$H_{O_{br}-C-M_{cus}-H}^{CH_3^*}$', \
            '$H_{O_{br}-H-M_{cus}-C}^{CH_3^*}$', \
            '$A_{O_{br}-M_{cus}-H}^{H^*}$', '$A_{O_{br}-C-H}^{CH_3^*}$', \
            '$A_{M_{cus}-C-H}^{CH_3^*}$', '$A_{O_{br}-H-M_{cus}}^{H^*}$', \
            '$D_{O_{br}-C}^{{CH}_3^*}$']
    '''
    features = {'E_CH3ab': 0.84, 'E_Hab3': 0.50, \
            'a_O2-M4-C_CH3ab': 0.40, 'd_H1-C_CH3ab': -0.31, \
            'h_O2-C-M4-H1_CH3ab': 0.30, 'h_O2-H1-M4-C_CH3ab': -0.29, \
            'a_O2-M4-H1_Hab3': -0.24, 'a_O2-C-H1_CH3ab': -0.12, \
            'a_M4-C-H1_CH3ab': 0.11, 'a_O2-H1-M4_Hab3': 0.05, \
            'd_O2-C_CH3ab': 0.01}

    feanames = ['$E_{{CH}_3^*}$', '$E_{H^*}$', \
            '$A_{O_{br}-M_{cus}-C}^{CH_3^*}$', '$D_{H-C}^{CH_3^*}$', \
            '$H_{O_{br}-C-M_{cus}-H}^{CH_3^*}$', \
            '$H_{O_{br}-H-M_{cus}-C}^{CH_3^*}$', \
            '$A_{O_{br}-M_{cus}-H}^{H^*}$', '$A_{O_{br}-C-H}^{CH_3^*}$', \
            '$A_{M_{cus}-C-H}^{CH_3^*}$', '$A_{O_{br}-H-M_{cus}}^{H^*}$', \
            '$D_{O_{br}-C}^{{CH}_3^*}$']

    # ts-ss ax1 features selected by lasso
    plot_selected_features(ax[0][0], features, feanames)

    # ts-ss ax2 feature selection
    plot_feature_selection(ax[0][1], 'Ets', features)
    ax[0][1].text(-2, 1.2, '(a) Surface-stabilized Mechanism', \
            fontsize=20, fontweight='normal')

    # ts-ss ax3 best linear regression
    plot_best_regression(ax[0][2], 'Ets', features, 7)

    # ts-ra part
    ''' old
    features = {'E_Hab3': 0.92, \
            'd_O2-C_CH3ab': 0.04, 'd_O2-H1_Hab3': -0.02, 'd_O2-M4_suf': 0.01, \
            'h_O2-M4-H1-C_CH3ab': 0.01}

    feanames = ['$E_{H^*}$', \
            '$D_{O_{br}-C}^{CH_3^*}$', '$D_{O_{br}-H}^{H^*}$', \
            '$D_{O_{br}-M_{cus}}^{surface}$', '$H_{O_{br}-M_{cus}-H-C}^{CH_3^*}$']
    '''
    features = {'E_Hab3': 0.86, \
            'd_O2-H1_Hab3': -0.02, 'd_O2-M4_suf': 0.01}

    feanames = ['$E_{H^*}$', \
            '$D_{O_{br}-H}^{H^*}$', \
            '$D_{O_{br}-M_{cus}}^{surface}$']

    # ts-ra ax4 features selected by lasso
    plot_selected_features(ax[1][0], features, feanames)

    # ts-ra ax5 feature selection
    plot_feature_selection(ax[1][1], 'Etsra', features)
    ax[1][1].text(0.5, 1.2, '(b) Radical-like Mechanism', \
            fontsize=20, fontweight='normal')

    # ts-ra ax6 best linear regression
    plot_best_regression(ax[1][2], 'Etsra', features, 1)

    #plt.show()
    plt.savefig('./figures/fig6.png')

if __name__ == '__main__':
    DS_CSV = '../CH4_10.csv'
    plot_main()


from collections import deque
# import sys

# input=sys.stdin.readline

def water(water_area, data):
    new = []
    for i in water_area:
        for j in range(4):
            nx = i[0] + dx[j]
            ny = i[1] + dy[j]

            if nx < 0 or nx >= r or ny < 0 or ny >= c:
                continue

            if data[nx][ny] != 'X' and data[nx][ny] != 'D':
                data[nx][ny] = '*'
                new.append((nx, ny))
    return new

r, c = map(int, input().split())
data = []
water_area = []
x, y = 0, 0
for i in range(r):
    line = list(input())
    data.append(line)
    for j in range(len(line)):
        if line[j] == 'S':
            x = i
            y = j
        if line[j] == '*':
            water_area.append((i, j))

dx = [-1, 0, 1, 0]
dy = [0, -1, 0, 1]
result = [[0] * c for _ in range(r)]
ans = 0

q = deque()
q.append((x, y))
while q:
    now_x, now_y = q.popleft()
    if data[now_x][now_y] == 'D':
        ans = result[now_x][now_y]
        break
    water_area = water(water_area, data)
    for i in range(4):
        nx = now_x + dx[i]
        ny = now_y + dy[i]

        if nx < 0 or nx >= r or ny < 0 or ny >= c:
            continue
        if data[nx][ny] != '*' and data[nx][ny] != 'X':
            result[nx][ny] = result[now_x][now_y] + 1
            q.append((nx, ny))
            print(result)

print(result)
if ans == 0:
    print('KAKTUS')
else:
    print(ans)


import os
from collections import defaultdict
from arelle import (UrlUtil, XbrlConst)
from arelle.ModelObject import ModelObject
from arelle.ModelDtsObject import ModelConcept
from arelle.PrototypeInstanceObject import ContextPrototype, DimValuePrototype

def loadDimensionDefaults(val):
    # load dimension defaults when required without performing validations
    val.modelXbrl.dimensionDefaultConcepts = {}
    val.modelXbrl.qnameDimensionDefaults = {}
    val.modelXbrl.qnameDimensionContextElement = {}
    for baseSetKey in val.modelXbrl.baseSets.keys():
        arcrole, ELR, linkqname, arcqname = baseSetKey
        if ELR and linkqname and arcqname and arcrole in (XbrlConst.all, XbrlConst.dimensionDefault):
            checkBaseSet(val, arcrole, ELR, val.modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
    val.modelXbrl.isDimensionsValidated = True

def checkBaseSet(val, arcrole, ELR, relsSet):
    # check hypercube-dimension relationships
    if arcrole == XbrlConst.hypercubeDimension:
        for modelRel in relsSet.modelRelationships:
            fromConcept = modelRel.fromModelObject
            toConcept = modelRel.toModelObject
            if fromConcept is not None and toConcept is not None:
                if not fromConcept.isHypercubeItem:
                    val.modelXbrl.error("xbrldte:HypercubeDimensionSourceError",
                        _("Hypercube-dimension relationship from %(source)s to %(target)s in link role %(linkrole)s must have a hypercube declaration source"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                if not toConcept.isDimensionItem:
                    val.modelXbrl.error("xbrldte:HypercubeDimensionTargetError",
                        _("Hypercube-dimension relationship from %(source)s to %(target)s in link role %(linkrole)s must have a dimension declaration target"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
    # check all, notAll relationships
    elif arcrole in (XbrlConst.all, XbrlConst.notAll):
        fromRelationships = relsSet.fromModelObjects()
        for priItemConcept, hcRels in fromRelationships.items():
            for hasHcRel in hcRels:
                hcConcept = hasHcRel.toModelObject
                if priItemConcept is not None and hcConcept is not None:
                    if not priItemConcept.isPrimaryItem:
                        val.modelXbrl.error("xbrldte:HasHypercubeSourceError",
                            _("HasHypercube %(arcroleType)s relationship from %(source)s to %(target)s in link role %(linkrole)s must have a primary item source"),
                            modelObject=hasHcRel, arcroleType=os.path.basename(arcrole),
                            source=priItemConcept.qname, target=hcConcept.qname, linkrole=ELR)
                    if not hcConcept.isHypercubeItem:
                        val.modelXbrl.error("xbrldte:HasHypercubeTargetError",
                            _("HasHypercube %(arcroleType)s relationship from %(source)s to %(target)s in link role %(linkrole)s must have a hypercube declaration target"),
                            modelObject=hasHcRel, arcroleType=os.path.basename(arcrole),
                            source=priItemConcept.qname, target=hcConcept.qname, linkrole=ELR)
                    hcContextElement = hasHcRel.contextElement
                    if hcContextElement not in ("segment","scenario"):
                        val.modelXbrl.error("xbrldte:HasHypercubeMissingContextElementAttributeError",
                            _("HasHypercube %(arcroleType)s relationship from %(source)s to %(target)s in link role %(linkrole)s must have a context element"),
                            modelObject=hasHcRel, arcroleType=os.path.basename(arcrole),
                            source=priItemConcept.qname, target=hcConcept.qname, linkrole=ELR)

                    # must check the cycles starting from hypercube ELR (primary item consec relationship
                    dimELR = hasHcRel.targetRole
                    if not dimELR:
                        dimELR = ELR
                    hcDimRels = val.modelXbrl.relationshipSet(
                         XbrlConst.hypercubeDimension, dimELR).fromModelObject(hcConcept)
                    for hcDimRel in hcDimRels:
                        dimConcept = hcDimRel.toModelObject
                        if dimConcept is not None:
                            if arcrole == XbrlConst.all:
                                cntxElt = val.modelXbrl.qnameDimensionContextElement.setdefault(dimConcept.qname, hcContextElement)
                                if cntxElt != hcContextElement:
                                    val.modelXbrl.qnameDimensionContextElement[dimConcept.qname] = "ambiguous"
                            domELR = hcDimRel.targetRole
                            if not domELR:
                                domELR = dimELR
                            dimDomRels = val.modelXbrl.relationshipSet(
                                 XbrlConst.dimensionDomain, domELR).fromModelObject(dimConcept)
                            cycle = xdtCycle(val, domainTargetRoles(val, domELR,dimDomRels), dimDomRels, {hcConcept,dimConcept})
                            if cycle is not None:
                                if cycle is not None:
                                    cycle.append(hcDimRel)
                                    path = str(hcConcept.qname) + " " + " - ".join(
                                        "{0}:{1} {2}".format(rel.modelDocument.basename, rel.sourceline, rel.toModelObject.qname)
                                        for rel in reversed(cycle))
                                val.modelXbrl.error("xbrldte:DRSDirectedCycleError",
                                    _("Dimension relationships have a directed cycle in DRS role %(linkrole)s \nstarting from hypercube %(hypercube)s, \ndimension %(dimension)s, \npath %(path)s"),
                                    modelObject=[hcConcept] + cycle, hypercube=hcConcept.qname, dimension=dimConcept.qname, linkrole=ELR, path=path)
                            cycle = drsPolymorphism(val, domELR, dimDomRels, drsPriItems(val, ELR, priItemConcept))
                            if cycle is not None:
                                if cycle is not None:
                                    cycle.append(hcDimRel)
                                    path = str(priItemConcept.qname) + " " + " - ".join(
                                        "{0}:{1} {2}".format(rel.modelDocument.basename, rel.sourceline, rel.toModelObject.qname)
                                        for rel in reversed(cycle))
                                val.modelXbrl.error("xbrldte:PrimaryItemPolymorphismError",
                                    _("Dimension relationships have a polymorphism cycle in DRS role %(linkrole)s \nstarting from hypercube %(hypercube)s, \ndimension %(dimension)s, \npath %(path)s"),
                                    modelObject=[hcConcept] + cycle, hypercube=hcConcept.qname, dimension=dimConcept.qname, linkrole=ELR, path=path)
    # check dimension-domain relationships
    elif arcrole == XbrlConst.dimensionDomain:
        for modelRel in relsSet.modelRelationships:
            fromConcept = modelRel.fromModelObject
            toConcept = modelRel.toModelObject
            if fromConcept is not None and toConcept is not None:   # none if failed to load
                if not fromConcept.isDimensionItem:
                    val.modelXbrl.error("xbrldte:DimensionDomainSourceError",
                        _("Dimension-domain relationship from %(source)s to %(target)s in link role %(linkrole)s must have a dimension declaration source"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                elif fromConcept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef") is not None:
                    val.modelXbrl.error("xbrldte:DimensionDomainSourceError",
                        _("Dimension-domain relationship from %(source)s to %(target)s in link role %(linkrole)s has a typed dimension source"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                if not toConcept.isDomainMember:
                    val.modelXbrl.error("xbrldte:DimensionDomainTargetError",
                        _("Dimension-domain relationship from %(source)s to %(target)s in link role %(linkrole)s must have a domain member target"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
    # check dimension-default relationships
    elif arcrole == XbrlConst.dimensionDefault:
        for modelRel in relsSet.modelRelationships:
            fromConcept = modelRel.fromModelObject
            toConcept = modelRel.toModelObject
            if fromConcept is not None and toConcept is not None:
                if not fromConcept.isDimensionItem:
                    val.modelXbrl.error("xbrldte:DimensionDefaultSourceError",
                        _("Dimension-default relationship from %(source)s to %(target)s in link role %(linkrole)s must have a dimension declaration source"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                elif fromConcept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef"):
                    val.modelXbrl.error("xbrldte:DimensionDefaultSourceError",
                        _("Dimension-default relationship from %(source)s to %(target)s in link role %(linkrole)s has a typed dimension source"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                if not toConcept.isDomainMember:
                    val.modelXbrl.error("xbrldte:DimensionDefaultTargetError",
                        _("Dimension-default relationship from %(source)s to %(target)s in link role %(linkrole)s must have a domain member target"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                if fromConcept in val.modelXbrl.dimensionDefaultConcepts and toConcept != val.modelXbrl.dimensionDefaultConcepts[fromConcept]:
                    val.modelXbrl.error("xbrldte:TooManyDefaultMembersError",
                        _("Dimension %(source)s has multiple defaults %(target)s and %(target2)s"),
                        modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname,
                        target2=val.modelXbrl.dimensionDefaultConcepts[fromConcept].qname)
                else:
                    val.modelXbrl.dimensionDefaultConcepts[fromConcept] = toConcept
                    val.modelXbrl.qnameDimensionDefaults[fromConcept.qname] = toConcept.qname

    # check for primary item cycles
    elif arcrole == XbrlConst.domainMember:
        fromRelationships = relsSet.fromModelObjects()
        for priItemConcept, rels in fromRelationships.items():
                for domMbrRel in rels:
                    toConcept = domMbrRel.toModelObject
                    if toConcept is not None:
                        if not priItemConcept.isDomainMember:
                            val.modelXbrl.error("xbrldte:DomainMemberSourceError",
                                _("Domain-Member relationship from %(source)s to %(target)s in link role %(linkrole)s must have a domain primary item or domain member source"),
                                modelObject=domMbrRel, source=priItemConcept.qname, target=toConcept.qname, linkrole=ELR)
                        if not toConcept.isDomainMember:
                            val.modelXbrl.error("xbrldte:DomainMemberTargetError",
                                _("Domain-Member relationship from %(source)s to %(target)s in link role %(linkrole)s must have a domain primary item or domain member target"),
                                modelObject=domMbrRel, source=priItemConcept.qname, target=toConcept.qname, linkrole=ELR)

def domainTargetRoles(val, fromELR, rels, fromConcepts=None, ELRs=None):
    if fromConcepts is None:
        fromConcepts = set()
    if not ELRs:
        ELRs = {fromELR}
    for rel in rels:
        relTo = rel.toModelObject
        if relTo not in fromConcepts:
            fromConcepts.add(relTo)
            toELR = rel.targetRole
            if toELR:
                ELRs.add(toELR)
            else:
                toELR = fromELR
            domMbrRels = val.modelXbrl.relationshipSet(XbrlConst.domainMember, toELR).fromModelObject(relTo)
            domainTargetRoles(val, toELR, domMbrRels, fromConcepts, ELRs)
            fromConcepts.discard(relTo)
    return ELRs

def xdtCycle(val, ELRs, rels, fromConcepts):
    for rel in rels:
        relTo = rel.toModelObject
        if rel.isUsable and relTo in fromConcepts: # don't think we want this?? and toELR == drsELR: #forms a directed cycle
            return [rel,]
        fromConcepts.add(relTo)
        for ELR in ELRs:
            domMbrRels = val.modelXbrl.relationshipSet(XbrlConst.domainMember, ELR).fromModelObject(relTo)
            foundCycle = xdtCycle(val, ELRs, domMbrRels, fromConcepts)
            if foundCycle is not None:
                foundCycle.append(rel)
                return foundCycle
        fromConcepts.discard(relTo)
    return None

def drsPriItems(val, fromELR, fromPriItem, priItems=None):
    if priItems is None:
        priItems = {fromPriItem}
    for rel in  val.modelXbrl.relationshipSet(XbrlConst.domainMember, fromELR).fromModelObject(fromPriItem):
        toPriItem = rel.toModelObject
        if toPriItem not in priItems:
            if rel.isUsable:
                priItems.add(toPriItem)
            toELR = rel.targetRole
            drsPriItems(val, toELR if toELR else fromELR, toPriItem, priItems)
    return priItems

def drsPolymorphism(val, fromELR, rels, priItems, visitedMbrs=None):
    if visitedMbrs is None:
        visitedMbrs = set()
    for rel in rels:
        relTo = rel.toModelObject
        toELR = rel.targetRole
        if not toELR:
            toELR = fromELR
        if rel.isUsable and relTo in priItems: # don't think we want this?? and toELR == drsELR: #forms a directed cycle
            return [rel,]
        if relTo not in visitedMbrs:
            visitedMbrs.add(relTo)
            domMbrRels = val.modelXbrl.relationshipSet(XbrlConst.domainMember, toELR).fromModelObject(relTo)
            foundCycle = drsPolymorphism(val, toELR, domMbrRels, priItems, visitedMbrs)
            if foundCycle is not None:
                foundCycle.append(rel)
                return foundCycle
            visitedMbrs.discard(relTo)
    return None

def checkConcept(val, concept):
    if concept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef"):
        if concept.isDimensionItem:
            typedDomainElement = concept.typedDomainElement
            if typedDomainElement is None:
                url, id = UrlUtil.splitDecodeFragment(concept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef"))
                if len(id) == 0:
                    val.modelXbrl.error("xbrldte:TypedDimensionURIError",
                        _("Concept %(concept)s typedDomainRef has no fragment identifier"),
                        modelObject=concept, concept=concept.qname)
                else:
                    val.modelXbrl.error("xbrldte:OutOfDTSSchemaError",
                        _("Concept %(concept)s typedDomainRef is not resolved"),
                        modelObject=concept, concept=concept.qname)
            elif not isinstance(typedDomainElement, ModelConcept) or \
                        not typedDomainElement.isGlobalDeclaration or \
                        typedDomainElement.abstract == "true":
                val.modelXbrl.error("xbrldte:TypedDimensionError",
                    _("Concept %(concept)s typedDomainRef must identify a non-abstract element"),
                        modelObject=concept, concept=concept.qname)
        else:
            val.modelXbrl.error("xbrldte:TypedDomainRefError",
                _("Concept %(concept)s is not a dimension item but has a typedDomainRef"),
                modelObject=concept, concept=concept.qname)

def checkContext(val, cntx):
    # check errorDimensions of context
    for modelDimValues in (cntx.segDimValues.values(), cntx.scenDimValues.values(), cntx.errorDimValues):
        for modelDimValue in modelDimValues:
            dimensionConcept = modelDimValue.dimension
            if dimensionConcept is None or \
                not dimensionConcept.isDimensionItem or \
                modelDimValue.isTyped != (dimensionConcept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef") is not None):
                val.modelXbrl.error("xbrldie:TypedMemberNotTypedDimensionError" if modelDimValue.isTyped else "xbrldie:ExplicitMemberNotExplicitDimensionError",
                    _("Context %(contextID)s %(dimension)s %(value)s is not an appropriate dimension item"),
                    modelObject=modelDimValue, contextID=cntx.id,
                    dimension=modelDimValue.prefixedName, value=modelDimValue.dimensionQname)
            elif modelDimValue.isExplicit:
                memberConcept = modelDimValue.member
                if memberConcept is None or not memberConcept.isGlobalDeclaration:
                    val.modelXbrl.error("xbrldie:ExplicitMemberUndefinedQNameError",
                        _("Context %(contextID)s explicit dimension %(dimension)s member %(value)s is not a global member item"),
                        modelObject=modelDimValue, contextID=cntx.id,
                        dimension=modelDimValue.dimensionQname, value=modelDimValue.memberQname)
                if val.modelXbrl.dimensionDefaultConcepts.get(dimensionConcept) == memberConcept:
                    val.modelXbrl.error("xbrldie:DefaultValueUsedInInstanceError",
                        _("Context %(contextID)s explicit dimension %(dimension)s member {2} is a default member item"),
                        modelObject=modelDimValue, contextID=cntx.id,
                        dimension=modelDimValue.dimensionQname, value=modelDimValue.memberQname)
            elif modelDimValue.isTyped:
                typedDomainConcept = dimensionConcept.typedDomainElement
                problem = _("missing content")
                for element in modelDimValue:
                    if isinstance(element,ModelObject):
                        if problem is None:
                            problem = _("multiple contents")
                        elif element.localName != typedDomainConcept.name or \
                            element.namespaceURI != typedDomainConcept.qname.namespaceURI:
                            problem = _("wrong content {0}").format(element.prefixedName)
                        else:
                            problem = None
                if problem:
                    val.modelXbrl.error("xbrldie:IllegalTypedDimensionContentError",
                        _("Context %(contextID)s typed dimension %(dimension)s has %(error)s"),
                        modelObject=modelDimValue, contextID=cntx.id,
                        dimension=modelDimValue.dimensionQname, error=problem)

    for modelDimValue in cntx.errorDimValues:
        dimensionConcept = modelDimValue.dimension
        if dimensionConcept is not None \
           and (dimensionConcept in cntx.segDimValues or dimensionConcept in cntx.scenDimValues):
            val.modelXbrl.error("xbrldie:RepeatedDimensionInInstanceError",
                _("Context %(contextID)s dimension %(dimension)s is a repeated dimension value"),
                modelObject=modelDimValue, contextID=cntx.id, dimension=modelDimValue.dimensionQname)
    # decision by WG that dimensions in both seg & scen is also a duplication
    for modelDimValue in cntx.segDimValues.values():
        dimensionConcept = modelDimValue.dimension
        if dimensionConcept is not None and dimensionConcept in cntx.scenDimValues:
            val.modelXbrl.error("xbrldie:RepeatedDimensionInInstanceError",
                _("Context %(contextID)s dimension %(dimension)s is a repeated dimension value"),
                modelObject=modelDimValue, contextID=cntx.id, dimension=modelDimValue.dimensionQname)

def checkFact(val, f):
    if not isFactDimensionallyValid(val, f):
        val.modelXbrl.error("xbrldie:PrimaryItemDimensionallyInvalidError",
            _("Fact %(fact)s context %(contextID)s dimensionally not valid"),
            modelObject=f, fact=f.qname, contextID=f.context.id)

def isFactDimensionallyValid(val, f, setPrototypeContextElements=False):
    hasElrHc = False
    for ELR, hcRels in priItemElrHcRels(val, f.concept).items():
        hasElrHc = True
        if checkFactElrHcs(val, f, ELR, hcRels, setPrototypeContextElements):
            return True # meets hypercubes in this ELR

    if hasElrHc:
        # no ELR hypercubes fully met
        return False
    return True

def priItemElrHcRels(val, priItem, ELR=None, elrHcRels=None):
    if elrHcRels is None:
        elrHcRels = defaultdict(list)
    # add has hypercube relationships for ELR
    for arcrole in (XbrlConst.all, XbrlConst.notAll):
        for hasHcRel in val.modelXbrl.relationshipSet(arcrole,ELR).fromModelObject(priItem):
            elrHcRels[hasHcRel.linkrole].append(hasHcRel)
    # check inherited ELRs
    for domMbrRel in val.modelXbrl.relationshipSet(XbrlConst.domainMember).toModelObject(priItem):
        relLinkrole = domMbrRel.linkrole
        toELR = (domMbrRel.targetRole or relLinkrole)
        if ELR is None or ELR == toELR:
            priItemElrHcRels(val, domMbrRel.fromModelObject, relLinkrole, elrHcRels)
    return elrHcRels

NOT_FOUND = 0
MEMBER_USABLE = 1
MEMBER_NOT_USABLE = 2

def checkFactElrHcs(val, f, ELR, hcRels, setPrototypeContextElements=False):
    context = f.context
    elrValid = True # start assuming ELR is valid

    for hasHcRel in hcRels:
        hcConcept = hasHcRel.toModelObject
        hcIsClosed = hasHcRel.isClosed
        hcContextElement = hasHcRel.contextElement
        hcNegating = hasHcRel.arcrole == XbrlConst.notAll
        modelDimValues = context.dimValues(hcContextElement)
        if setPrototypeContextElements and isinstance(context,ContextPrototype):
            oppositeContextDimValues = context.dimValues(hcContextElement, oppositeContextElement=True)
        contextElementDimSet = set(modelDimValues.keys())
        modelNonDimValues = context.nonDimValues(hcContextElement)
        hcValid = True

        # if closed and any nonDim values, hc invalid
        if hcIsClosed and len(modelNonDimValues) > 0:
            hcValid = False
        else:
            dimELR = (hasHcRel.targetRole or ELR)
            for hcDimRel in val.modelXbrl.relationshipSet(
                                XbrlConst.hypercubeDimension, dimELR).fromModelObject(hcConcept):
                dimConcept = hcDimRel.toModelObject
                domELR = (hcDimRel.targetRole or dimELR)
                if dimConcept in modelDimValues:
                    memModelDimension = modelDimValues[dimConcept]
                    contextElementDimSet.discard(dimConcept)
                    memConcept = memModelDimension.member
                elif dimConcept in val.modelXbrl.dimensionDefaultConcepts:
                    memConcept = val.modelXbrl.dimensionDefaultConcepts[dimConcept]
                    memModelDimension = None
                elif setPrototypeContextElements and isinstance(context,ContextPrototype) and dimConcept in oppositeContextDimValues:
                    memModelDimension = oppositeContextDimValues[dimConcept]
                    memConcept = memModelDimension.member
                else:
                    hcValid = False
                    continue
                if not dimConcept.isTypedDimension:
                    if dimensionMemberState(val, dimConcept, memConcept, domELR) != MEMBER_USABLE:
                        hcValid = False
                if hcValid and setPrototypeContextElements and isinstance(memModelDimension,DimValuePrototype) and not hcNegating:
                    memModelDimension.contextElement = hcContextElement
        if hcIsClosed:
            if len(contextElementDimSet) > 0:
                hcValid = False # has extra stuff in the context element
        elif setPrototypeContextElements and isinstance(context,ContextPrototype) and hcValid and not hcNegating:
            for memModelDimension in modelDimValues.values(): # be sure no ambiguous items are left, this cube is open
                if memModelDimension.contextElement != hcContextElement: # if not moved by being explicit cube member
                    memModelDimension.contextElement = hcContextElement # move it
            if len(oppositeContextDimValues) > 0: # move any opposite dim values into open hypercube
                for memModelDimension in oppositeContextDimValues.values():
                    if memModelDimension.contextElement != hcContextElement: # if not moved by being explicit cube member
                        memModelDimension.contextElement = hcContextElement # move it
        if hcNegating:
            hcValid = not hcValid
        if not hcValid:
            elrValid = False
    return elrValid

def dimensionMemberState(val, dimConcept, memConcept, domELR):
    try:
        dimensionMemberStates = val.dimensionMemberStates
    except AttributeError:
        dimensionMemberStates = val.dimensionMemberStates = {}
    key = (dimConcept, memConcept, domELR)
    try:
        return dimensionMemberStates[key]
    except KeyError:
        dimDomRels = val.modelXbrl.relationshipSet(
                        XbrlConst.dimensionDomain, domELR).fromModelObject(dimConcept)
        state = memberStateInDomain(val, memConcept, dimDomRels, domELR)
        dimensionMemberStates[key] = state
        return state

def memberStateInDomain(val, memConcept, rels, ELR, toConceptELRs=None):
    foundState = NOT_FOUND
    if toConceptELRs is None:
        toConceptELRs = defaultdict(set)
    for rel in rels:
        toConcept = rel.toModelObject
        if toConcept == memConcept:
            foundState = max(foundState,
                             MEMBER_USABLE if rel.isUsable else MEMBER_NOT_USABLE)
        toELR = (rel.targetRole or ELR)
        toELRs = toConceptELRs[toConcept]
        if toELR not in toELRs:  # looping if it's already there in a visited ELR
            toELRs.add(toELR)
            domMbrRels = val.modelXbrl.relationshipSet(XbrlConst.domainMember, toELR).fromModelObject(toConcept)
            foundState = max(foundState,
                             memberStateInDomain(val, memConcept, domMbrRels, toELR, toConceptELRs))
            toELRs.discard(toELR)
    return foundState

''' removed because no valid way to check one isolated dimension for validity without full set of others
# check a single dimension value for primary item (not the complete set of dimension values)
# returnn the contextElement of the hypercube where valid
def checkPriItemDimValueValidity(val, priItemConcept, dimConcept, memConcept, srcCntxEltName=None):
    if priItemConcept is not None and dimConcept is not None:
        _priItemElrHcRels = priItemElrHcRels(val, priItemConcept)
        for ELR, hcRels in _priItemElrHcRels.items():
            hcCntxElt = checkPriItemDimValueElrHcs(val, priItemConcept, dimConcept, memConcept, srcCntxEltName, hcRels)
            if hcCntxElt:
                return hcCntxElt
        # look for open hypercubes in other ELRs
    for hasHcRel in val.modelXbrl.relationshipSet(XbrlConst.all).modelRelationships:
        if hasHcRel.linkrole not in _priItemElrHcRels:
            hcCntxElt = checkPriItemDimValueElrHcs(val, priItemConcept, dimConcept, memConcept, srcCntxEltName, hcRels, openOnly=True)
            if hcCntxElt:
                return hcCntxElt
    return None

def checkPriItemDimValueElrHcs(val, priItemConcept, matchDim, matchMem, srcCntxEltName, hcRels, openOnly=False):
    hcCntxElt = None
    for hasHcRel in hcRels:
        if hasHcRel.arcrole == XbrlConst.all:
            hcConcept = hasHcRel.toModelObject
            hcIsClosed = hasHcRel.isClosed
            cntxElt = hasHcRel.contextElement

            dimELR = (hasHcRel.targetRole or hasHcRel.linkrole)
            for hcDimRel in val.modelXbrl.relationshipSet(
                                XbrlConst.hypercubeDimension, dimELR).fromModelObject(hcConcept):
                dimConcept = hcDimRel.toModelObject
                if dimConcept != matchDim:
                    continue
                if matchMem is None and not openOnly:
                    if cntxElt == srcCntxEltName:
                        return cntxElt
                    else:
                        hcCntxElt = cntxElt
                        continue
                domELR = (hcDimRel.targetRole or dimELR)
                state = dimensionMemberState(val, dimConcept, matchMem, domELR)
                if state == MEMBER_USABLE and not openOnly:
                    if cntxElt == srcCntxEltName:
                        return cntxElt
                    else:
                        hcCntxElt = cntxElt
                        continue
            if not hcIsClosed:
                hcCntxElt = cntxElt
    return hcCntxElt
'''


import requests
import json
import re

# MITRE Groups https://attack.mitre.org/groups/
mitre_actors = requests.get('https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json')
mitre_actors = mitre_actors.json()

mitre_actor_list = []
for adversary in mitre_actors['objects']:
    mitre_actor_dict = {}
    if adversary['type'] == 'intrusion-set':
        mitre_actor_dict['id'] = adversary['id']
        mitre_actor_dict['name'] = adversary['name']
        created = adversary['created']
        created = created.split('T')
        mitre_actor_dict['created'] = created[0]
        last_modified = adversary['modified']
        last_modified = last_modified.split('T')
        mitre_actor_dict['last_modified'] = last_modified[0]
        mitre_actor_dict['variations'] = []
        mitre_actor_dict['variations_custom'] = []
        external_references = adversary['external_references']
        for reference in external_references:
            if 'external_id' in reference.keys():
                mitre_actor_dict['url'] = reference['url']
        try:
            variations = adversary['aliases']
            for variation in variations:
                mitre_actor_dict['variations'].append(variation)
        except KeyError:
            mitre_actor_dict['variations'].append(adversary['name'])

        variations_custom = []
        for item in mitre_actor_dict['variations']:
            variation_upper = item.upper()
            variations_custom.append(variation_upper)
            if ' ' in variation_upper:
                variation_noSpace = variation_upper.replace(' ', '')
                variations_custom.append(variation_noSpace)
            if '-' in variation_upper:
                variation_noDash = variation_upper.replace('-', '')
                variations_custom.append(variation_noDash)
            try:
                if '-' in variation_noSpace:
                    variation_noSpace_noDash = variation_noSpace.replace('-', '')
                    variations_custom.append(variation_noSpace_noDash)
            except NameError:
                continue
        for i in variations_custom:
            mitre_actor_dict['variations_custom'].append(i)

        mitre_actor_list.append(mitre_actor_dict)

# Populate MITRE Group TTPs
for mitre_actor in mitre_actor_list:
    technique_list = []
    actorID = mitre_actor['id']
    ttp_patternIDs = []
    for object_relationships in mitre_actors['objects']:
        try:
            if object_relationships['source_ref'] == actorID:
                if 'attack-pattern' in object_relationships['target_ref']:
                    ttp_patternIDs.append(object_relationships['target_ref'])
        except KeyError:
            continue
    for patternID in ttp_patternIDs:
        for object_ttp in mitre_actors['objects']:
            if object_ttp['id'] == patternID:
                try:
                    for external_reference in object_ttp['external_references']:
                        try:
                            if 'CAPEC' not in external_reference['external_id']:
                                technique_list.append(external_reference['external_id'])
                        except KeyError:
                            continue
                except KeyError:
                    continue

    mitre_actor['TTPs'] = technique_list

# ETDA Actors https://apt.etda.or.th/cgi-bin/listgroups.cgi
etda_actors = requests.get('https://apt.etda.or.th/cgi-bin/getmisp.cgi?o=g')
etda_actors = etda_actors.json()

etda_actor_list = []
for adversary in etda_actors['values']:
    etda_actor_dict = {}
    etda_actor_dict['id'] = adversary['uuid']
    name = adversary['value']
    etda_actor_dict['name'] = name
    name = name.replace('[', '')
    name = name.replace(']', '')
    name_list = re.split(', |,', name)
    etda_actor_dict['variations'] = []
    etda_actor_dict['variations_custom'] = []
    metadata = adversary['meta']
    etda_actor_dict['url'] = 'https://apt.etda.or.th/cgi-bin/showcard.cgi?u=' + adversary['uuid']
    try:
        etda_actor_dict['created'] = metadata['date']
    except KeyError:
        etda_actor_dict['created'] = 'None Provided'
    try:
        for variation in metadata['synonyms']:
            variation = variation.replace('[', '')
            variation = variation.replace(']', '')
            etda_actor_dict['variations'].append(variation)
        for name_variation in name_list:
            if name_variation not in etda_actor_dict['variations']:
                etda_actor_dict['variations'].append(name_variation)
    except KeyError:
        for name_variation in name_list:
            if name_variation not in etda_actor_dict['variations']:
                etda_actor_dict['variations'].append(name_variation)
    if 'country' in metadata.keys():
        etda_actor_dict['country'] = metadata['country']
    if 'motivation' in metadata.keys():
        etda_actor_dict['motivation'] = metadata['motivation']
    if 'cfr-target-category' in metadata.keys():
        etda_actor_dict['targeted_industries'] = metadata['cfr-target-category']
    if 'cfr-suspected-victims' in metadata.keys():
        etda_actor_dict['targeted_countries'] = metadata['cfr-suspected-victims']

    variations_custom = []
    for item in etda_actor_dict['variations']:
        variation_upper = item.upper()
        variations_custom.append(variation_upper)
        if ' ' in variation_upper:
            variation_noSpace = variation_upper.replace(' ', '')
            variations_custom.append(variation_noSpace)
        if '-' in variation_upper:
            variation_noDash = variation_upper.replace('-', '')
            variations_custom.append(variation_noDash)
        try:
            if '-' in variation_noSpace:
                variation_noSpace_noDash = variation_noSpace.replace('-', '')
                variations_custom.append(variation_noSpace_noDash)
        except NameError:
            continue
    for i in variations_custom:
        etda_actor_dict['variations_custom'].append(i)

    etda_actor_list.append(etda_actor_dict)

# Comparison
merge_list = []
id_check = []
for mitre_actor in mitre_actor_list:
    for mitre_variation in mitre_actor['variations_custom']:
        for etda_actor in etda_actor_list:
            if mitre_variation in etda_actor['variations_custom']:
                # Start compiling final data
                merge_dict = {}
                if mitre_actor['id'] in id_check:
                    continue
                else:
                    id_check.append(mitre_actor['id'])

                    merge_dict['mitre_attack_id'] = mitre_actor['id']
                    merge_dict['mitre_attack_name'] = mitre_actor['name']
                    merge_dict['mitre_attack_aliases'] = mitre_actor['variations']
                    merge_dict['mitre_attack_created'] = mitre_actor['created']
                    merge_dict['mitre_attack_last_modified'] = mitre_actor['last_modified']
                    merge_dict['mitre_url'] = mitre_actor['url']
                    merge_dict['etda_id'] = etda_actor['id']
                    merge_dict['etda_name'] = etda_actor['name']
                    merge_dict['etda_aliases'] = etda_actor['variations']
                    merge_dict['etda_first_seen'] = etda_actor['created']
                    merge_dict['etda_url'] = etda_actor['url']
                    if 'country' in etda_actor.keys():
                        merge_dict['country'] = etda_actor['country']
                    else:
                        merge_dict['country'] = 'None Provided'
                    if 'motivation' in etda_actor.keys():
                        merge_dict['motivation'] = etda_actor['motivation']
                    else:
                        merge_dict['motivation'] = 'None Provided'
                    if 'targeted_industries' in etda_actor.keys():
                        merge_dict['victim_industries'] = etda_actor['targeted_industries']
                    else:
                        merge_dict['victim_industries'] = 'None Provided'
                    if 'targeted_countries' in etda_actor.keys():
                        merge_dict['victim_countries'] = etda_actor['targeted_countries']
                    else:
                        merge_dict['victim_countries'] = 'None Provided'
                    merge_dict['mitre_attack_ttps'] = mitre_actor['TTPs']

                    merge_list.append(merge_dict)

outfile = open('Categorized_Adversary_TTPs.json', 'w', encoding='utf-8')
json.dump(merge_list, outfile, indent=2, ensure_ascii=False)


"""
Use the saleae module to interface with the Saleae Logic analyzer to capture i2c traffic

Notes:
    - Make sure the Logic GUI has scripting enabled:
      options -> preferences -> developer -> Enable scripting socket server

"""
import time
import logging
from saleae import Saleae
import pandas as pd
import defs

log = logging.getLogger(__name__)

CAPTURE_OUTPUT_CSV = ".\\i2c_capture.csv"

def init_saleae():
    sal = Saleae()
    analyzers = sal.get_analyzers()
    log.info(f"Saleae analyzers: {analyzers}")
    return sal

def run(sal, capture_seconds):
    """

    Args:
        sal (Saleae): Saleae object
        capture_seconds:

    Returns:

    """

    sal.set_capture_seconds(capture_seconds)
    sal.capture_start_and_wait_until_finished()
    sal.export_analyzer(analyzer_index=1, wait_for_processing=True, save_path=CAPTURE_OUTPUT_CSV)
    time.sleep(0.1)  # Make sure export is finished so that CSV file contains latest capture

def process_capture(path=CAPTURE_OUTPUT_CSV):
    """Process capture of 1-byte i2c traffic and convert into one transmission per line

    Args:
        path: path to exported i2c analyzer (CSV)

    Returns:

    """

    df = pd.read_csv(path)
    log.info(f"Imported df:\n {df.head(5)}")

    # First byte (always write) is register value
    df['Reg'] = df['Data']

    # Second byte (shifted up one) is R/W data
    df['R/W'] = df['Read/Write'].shift(-1)
    df['Data'] = df['Data'].shift(-1)

    print(df.head(5))

    # Combine SMB write/read transactions into single rows
    i2c_df: pd.DataFrame = df[df['ACK/NAK'] != "NAK"].copy()
    i2c_df.drop(columns=["Read/Write"], inplace=True)
    print(i2c_df.head(5))

    log.info(f"Address dtypes: {i2c_df['Address'].dtypes}")
    i2c_df['IC'] = i2c_df['Address'].apply(int, base=16).map(defs.I2C_ADDR_MAP)

    i2c_df.to_csv("i2c_analyzed.csv")

def process_2byte_capture(path):
    """Process capture of 2-byte i2c traffic and convert into one transmission per line

    Args:
        path: path to exported i2c analyzer (CSV)

    Returns:

    """
    df = pd.read_csv(path)
    log.info(f"Imported df from {path}:")
    log.info(df.head(5))

    # Filter out non-responsive traffic
    df.dropna(subset=['Packet ID'], inplace=True)
    log.debug("Dropped empty packet IDs")
    log.debug(df.head(5))

    # todo - make sure data is aligned so assignments are correct even if captured in the middle of traffic
    # First 2 bytes (always writes) are register values
    df['Reg0'] = df['Data']
    df['Reg1'] = df['Data'].shift(-1)

    # 3rd byte is R/W data
    df['R/W'] = df['Read/Write'].shift(-2)
    df['Data'] = df['Data'].shift(-2)

    # Use Data NAK for alignment and filtration
    df['Data NAK'] = df['ACK/NAK'].shift(-2)

    log.debug("Reassigned reg/data column names")
    log.debug(df.head(5))

    # Filter down to rows containing properly aligned transactions
    i2c_df = df[df['Data NAK'] == 'NAK'].copy()

    # Combine SMB write/read transactions into single rows
    # i2c_df: pd.DataFrame = df[df['ACK/NAK'] != "NAK"].copy()
    # i2c_df.drop(columns=["Read/Write"], inplace=True)

    log.debug("Filtered down to aligned transactions")
    log.debug(i2c_df.head(5))

    log.info(f"Address dtypes: {i2c_df['Address'].dtypes}")
    i2c_df['IC'] = i2c_df['Address'].apply(int, base=16).map(defs.I2C_ADDR_MAP)

    return i2c_df
    # i2c_df.to_csv("i2c_analyzed.csv")

if __name__ == '__main__':
    import sys
    logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
    # s = init_saleae()
    # run(sal=s, capture_seconds=1)
    # process_capture()
    i2c_2byte_df = process_2byte_capture(
        path=r"C:\Users\Robert King\Google Drive\LiliPad\MCU\Firmware\p9221_full_multi_reg_reads.csv"
    )
    i2c_2byte_df.to_csv("./i2c_2byte.csv")


import base64
from reliabilly.tests import MagicMock, TestCase, logging, warnings, json, skipUnless
from reliabilly.components.factories.factory import Factory, Components
from reliabilly.components.services.queuing import SqsReceiver, QueueMessage
from reliabilly.settings import Settings, Constants

TEST = 'test'
DUMMY = 'dummy'
TEST_CONTENT = {'content': TEST}
TEST_MESSAGE = QueueMessage(**TEST_CONTENT)
ENCODED_TEST_MESSAGE = 'eyJtZXNzYWdlIjogeyJjb250ZW50IjogInRlc3QifSwgImlkIjogInRlc3QifQ=='
MOCK_RESPONSE = {Constants.MESSAGES_KEY: [{Constants.BODY_KEY: ENCODED_TEST_MESSAGE,
                                           Constants.RECEIPT_HANDLE_KEY: '123'}]}
DUMMY_MESSAGE = {'content': DUMMY}
ENCODED_DUMMY_MESSAGE = base64.b64encode(json.dumps(DUMMY_MESSAGE).encode())
LOCAL_SQS = 'http://localhost:9324'

class QueueingTests(TestCase):
    def setUp(self):
        logging.disable(logging.CRITICAL)
        warnings.filterwarnings(Constants.IGNORE, category=ResourceWarning)

    def test_full_queue(self):
        mock = MagicMock()
        mock.create_queue.return_value = mock
        mock.get_queue_by_name.return_value = mock
        mock.receive_message.return_value = MOCK_RESPONSE
        client = Factory.create(Components.ServiceClients.SQS, queue_name=DUMMY, client=mock)
        self.assertIsNotNone(client)
        self.assertTrue(client.send_message(TEST_MESSAGE))
        mock.send_message.assert_called()
        message = client.receive_message()
        self.assertTrue(message)
        self.assertTrue(client.complete_message(message))
        mock.send_message.side_effect = Exception
        self.assertFalse(client.send_message(TEST_MESSAGE))
        mock.receive_message.side_effect = Exception
        self.assertFalse(client.receive_message())
        mock.delete_message.side_effect = Exception
        self.assertFalse(client.complete_message(message))

    def test_purge(self):
        mock = MagicMock()
        client = Factory.create(Components.ServiceClients.SQS, queue_name=DUMMY, client=mock)
        self.assertTrue(client.purge())
        mock.purge_queue.side_effect = Exception
        self.assertFalse(client.purge())

    def test_create_queue(self):
        mock = MagicMock()
        mock.get_queue_by_name.side_effect = Exception
        client = Factory.create(Components.ServiceClients.SQS, queue_name=DUMMY, client=mock)
        self.assertTrue(client.create_queue(DUMMY))
        mock.create_queue.side_effect = Exception
        self.assertFalse(client.create_queue(DUMMY))

class QueueReceiverTests(TestCase):
    def setUp(self):
        logging.disable(logging.CRITICAL)
        warnings.filterwarnings(Constants.IGNORE, category=ResourceWarning)

    def test_setup(self):
        mock = MagicMock()
        queue = SqsReceiver(DUMMY, client=mock)
        queue.setup()
        self.assertIsNotNone(queue.client)
        mock.create_queue.assert_called()

    def test_run(self):
        mock = MagicMock()
        queue = SqsReceiver(DUMMY, client=mock)
        queue.exit = True
        self.assertTrue(queue.run(mock))
        self.assertTrue(queue.run())
        queue.container = mock
        queue.start()

class TestQueueMessage(TestCase):
    def test_message_encoding(self):
        message = TEST_MESSAGE
        self.assertIsNotNone(message.body)
        self.assertEqual(len(message.body), 100)  # the encoded length in bytes will always include the changing id

    def test_message_decoding(self):
        message = QueueMessage.create_from_encoded(ENCODED_TEST_MESSAGE)
        self.assertIsNotNone(TEST_MESSAGE.message_id, message.message_id)
        self.assertEqual(TEST_CONTENT, message.message)

    def test_message_construction(self):
        message = QueueMessage(test=2, test2='test', message_id='123')
        self.assertEqual(str(message), "{'test': 2, 'test2': 'test', 'message_id': '123'}")

@skipUnless(Settings.RUN_SKIPPED, Constants.RUN_SKIPPED_MSG)
class TestRealQueue(TestCase):
    def setUp(self):
        logging.disable(logging.CRITICAL)
        warnings.filterwarnings(Constants.IGNORE, category=ResourceWarning)

    def test_queue_lost_message(self):
        client = Factory.create(Components.ServiceClients.SQS, queue_name='test', endpoint_url=LOCAL_SQS)
        message = QueueMessage()
        message.product = "site.reliability"
        message.account_name = "reliabilly_ps"
        message.portfolio = "Platform Services"
        message.threshold = "5"
        message.slo_type = "error_rate"
        message.component = "reliabillyecs_cluster"
        message.query = "SELECT sum(`provider.httpCodeTarget5XXCount.Sum`) from LoadBalancerSample where " \
                        "`label.serverclass` like 'lighthouse' and `provider.awsRegion` = 'us-west-2'"
        message.query_executor = "newrelic"
        message.id = "0000000-ae7a-4896-a957-9b736128eaf3"
        message.name = "lighthouse-error_rate-uat"
        client.send_message(message)
        received_message = client.receive_message()
        self.assertEqual(message.message_id, received_message.message_id)
        client.complete_message(received_message)

    def test_actual_sqs(self):
        client = Factory.create(Components.ServiceClients.SQS, queue_name='test')
        model = QueueMessage()
        model.message = 'This is a test ___'
        model.name = 'ODB'
        model.value = 420
        self.assertTrue(client.send_message(model))
        message = client.receive_message()
        self.assertEqual(message.name, model.name)
        client.complete_message(message)

    def test_actual_queue_message(self):
        client = Factory.create(Components.ServiceClients.SQS, queue_name='qm-queue')
        self.assertTrue(client.send_message(QueueMessage(**DUMMY_MESSAGE)))
        message = client.receive_message()
        self.assertIsNotNone(message)
        client.complete_message(message)


import itertools

def getInput(name):
    with open(name+'.txt', 'r') as file:
        return file.read().strip().split('\n')

def sumSizes(lines):
    sumSizes = [] # part 1
    accumulateSizes = [] # part 2
    for line in lines:
        command = line.split(' ')

        if command[0] == '$':
            if command[1] == 'cd':
                if command[2] == '..':
                    sumSizes.append(accumulateSizes.pop())
                    accumulateSizes[-1] += sumSizes[-1]
                else: # $cd newDir
                    accumulateSizes.append(0)
            else: pass # $ ls
        elif command[0] != 'dir': # output of LS for files
            accumulateSizes[-1] += int(command[0])

    return sumSizes + list(itertools.accumulate(accumulateSizes[::-1]))

def part1(lines):
    lowestDirSizes = filter(lambda x: x <= 100_000, sumSizes(lines))
    return sum(lowestDirSizes)

def part2(lines):
    sumDirSizes = sumSizes(lines)
    available = 70_000_000 - sumDirSizes[-1]
    return min(filter(lambda x: x + available >= 30_000_000, sumDirSizes))

input = getInput('sample')
# input = getInput('input')

print('>>> part 1:', part1(input)) # sample: 95437
print('>>> part 2:', part2(input)) # sample: 24933642


#!/usr/bin/python
# -*- coding: UTF-8 -*-

#import MySQLdb

import datetime
import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import random
import os
from matplotlib.pyplot import figure
from fastprogress import master_bar, progress_bar
import torch.nn as nn
import time
from sklearn.metrics import mean_squared_error
import warnings
import itertools
from kalmanfilter import KalmanFilter

sns.set_style("darkgrid")
warnings.filterwarnings("ignore")

class Predictor:
    def __init__(self, state_transition=1, process_noise=1, observation_model=1, observation_noise=5):
        # mysql db
        #self.db = MySQLdb.connect("localhost", "testuser", "test123", "TESTDB", charset='utf8' )

        self.kf = KalmanFilter(state_transition=state_transition, process_noise=process_noise,
           observation_model=observation_model, observation_noise=observation_noise)

    def preprocessing(self, ds):
        ds['进站时间'] = pd.to_datetime(ds['进站时间'])
        df = ds.set_index('进站时间')

        min_day = df.index.min().date().strftime('%Y-%m-%d')
        max_day = df.index.max().date().strftime('%Y-%m-%d')

        timl = pd.date_range(min_day, max_day, freq='1D')
        names = set(list(df['进站名称']))
        pairs = [i for i in itertools.combinations(names, 2)]
        result = {}
        print("processing data...")
        for dt in timl:
            record = dict.fromkeys(pairs , 0)
            fg = df[df.index.date == datetime.date(dt.year, dt.month, dt.day)]
            for n in names:
                x = fg[fg['进站名称'] == n].groupby('出站名称').count()['用户ID']
                if len(x) > 0:
                    for m,j in zip(x.to_frame().index, x):
                        if (m, n) in record:
                            record[(m,n)] += j
                        elif (n, m) in record:
                            record[(n,m)] += j
            result[dt] = record

        multiple_time_series = {}
        for k, v in result.items():
            for ke, ve in v.items():
                if ke not in multiple_time_series:
                    multiple_time_series[ke] = []
                multiple_time_series[ke].append(ve)
        return timl, multiple_time_series

    def do_predict(self, ob_pair, last_days=7, predict_days=3, sql_cmd="", mode="predict_all", output_dir="./output/"):
        """
        cursor = self.db.cursor()
        cursor.execute(sql_cmd) # SQL语句，用户自定义
        data = cursor.fetchone()
        # 做一些逻辑处理
        # ...
        # 数据格式pandas：id, 进站名称, 进站时间, 出战名称, 出站时间, 渠道编号, 价格
        """
        ds = pd.read_csv("trips.csv", encoding='gb18030')
        timl, multiple_time_series = self.preprocessing(ds)
        if ob_pair in multiple_time_series:
            pdata = multiple_time_series[ob_pair]
        else:
            ob_pair = (ob_pair[1], ob_pair[0])
            pdata = multiple_time_series[ob_pair]
        if mode == "predict_all":
            pred = []
            y = []
            index = timl[-(len(pdata)-last_days-1):]
            for i in range(len(pdata) - last_days - 1):
                x = self.kf.predict(pdata[:last_days + i], 1)
                r = x.observations.mean[0]
                pred.append(r)
                y.append(pdata[last_days + i + 1])
            plt.plot(index, y, 'b-', label="observation")
            plt.plot(index, pred, 'r-', label="predict")
            plt.title('Predict OD [{} and {}]'.format(ob_pair[0], ob_pair[1]),fontsize=21)
            plt.xticks(fontsize=18)
            plt.yticks(fontsize=18)
            plt.ylabel('Passageer Volume', fontsize=18)
            plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
            plt.gcf().autofmt_xdate()
            plt.legend()
            plt.savefig(output_dir + '/predict_image_all.jpg')
        elif mode == "predict_future":
            if predict_days > len(pdata):
                raise Exception
            new_stamps = []
            for i in range(1, predict_days+1):
                new_stamps.append(timl[-1] + datetime.timedelta(days=i))
            x = self.kf.predict(pdata, predict_days)
            r = 0.5 * np.array(x.observations.mean) + 0.5 * np.array(pdata[-3:][::-1])
            plt.plot(timl, pdata, 'r-', label="observation")
            plt.plot(new_stamps, r, 'b-', label="prediction")
            plt.title('Predict OD [{} and {}]'.format(ob_pair[0], ob_pair[1]), fontsize=21)
            plt.xticks(fontsize=18)
            plt.yticks(fontsize=18)
            plt.ylabel('Next {} Days Passageer Volume'.format(predict_days), fontsize=18)
            plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
            plt.gcf().autofmt_xdate()
            plt.legend()
            plt.savefig(output_dir + '/predict_image_future.jpg')
        else:
            raise Exception("Not support yet.")

if __name__ == "__main__":
    p_instance = Predictor()
    p_instance.do_predict(ob_pair=('Sta134', 'Sta63'))


# Mech Vib Tsak 2 (SDOF, Wind Vibration, Analytic Sol) #

#************************************************************************************************

    ## The required Libraries to be imported:

import numpy as np
from numpy import rad2deg
from numpy.linalg import inv
import matplotlib.pyplot as plt
from math import pi, sqrt, sin, cos, atan
from cmath import phase
from sympy import symbols
from sympy.physics.vector import init_vprinting, vlatex
init_vprinting(use_latex='mathjax', pretty_print=False)

#************************************************************************************************

    ## System Parameters:

        ### Materials:
E_s = 2.1e11 # Modulus of rigidity of steel [Pa]
E_c = 0.17e11 # Modulus of rigidity of concrete [Pa]
E = 0.2*E_s + 0.8*E_c # Total modulus of rigidity [Pa]
rho = 1000 # RC density [kg/m^3]

        ### Dimensions:
L = 5 # Floor length [m]
W = 5 # Floor width [m]
h = 0.5 # Floor thickness [m]
H  = 5 # Story hight [m]
l = 0.65 # Column length [m]
w = 0.65 # Column width [m]
V = L * W * h # Floor volume

        ### Dynamic Parametrs:
m = rho * V # Floor mass [kg]
I = (l * w**4) / 12 # Column inertia [m^4]
ko = (3 * E * I) / H**3 # Column stiffness (cantilever beam)
k = 4*ko # Equivelant columns stiffnesses
zeta = 0.05 # Daming ratio of the building
zeta_list = [0.05, 0.1, 0.15, 0.25, 0.5, 1, 2]
ze1, ze2, ze3, ze4, ze5, ze6, ze7 = symbols('zeta1, zeta2, zeta3, zeta4, zeta5, zeta6, zeta7')
ze_list = [ze1, ze2, ze3, ze4, ze5, ze6, ze7]
R = np.linspace(0.00001,4.00001,120) # Frequency ratio list
wn = sqrt(k/m) # Natural frequency
wd = sqrt(1-zeta**2)*wn # Damped Natural frequency
cc = 2*sqrt(m*k) # Critical damping
c = zeta * cc # Damping coefficient

#************************************************************************************************

    ## Input Data (Sin wave):

        ### Wind (f = Fo * cos(w_w*t), direct force):
freq_w = 10 # Excitation frequency [Hz]
w_w = 2 * pi * freq_w # Cicular frequency [rad/s]
v_w = 140 # Wind velocity [m/s]
rho_a = 1.2 # Air density [kg/m^3]
A_w = L*H # Wind faced area of the building
Fwo = 0.5 * rho_a * v_w**2 * A_w # Wind force [N]

#************************************************************************************************

    ## Simulation:

        ### Time Installation:
dt = 1/(8*freq_w) # Time step
end_time = 1 # Finish time
time = np.arange(0, end_time, dt) # Time range

        ### Initial Conditions:
xo = 0.0
vo = 0.0

        ### Response Histories & Amplitudes:
r_w = w_w/wn

Fw_history = [] # Wind input history

x_st = Fwo/k # Static defliction

Xw_history = [] # Wind response history
Xw = Fwo / ((k-m*w_w**2) + complex(0,w_w*c))

        ### Time Responses:
for t in time:
#--------------------------------------------------
    fw = Fwo * cos(w_w*t) # input wind force
    Fw_history.append(fw*1e-6) # MN
#--------------------------------------------------
    x_w = abs(Xw)*cos(w_w*t-phase(Xw)) # Steady state response from wind
    Xw_history.append(x_w*1e2) # cm
#------------------------------------------------------------------------------------------
M_history = np.zeros((len(zeta_list),len(R))) # Amplification Factor
PH_history = np.zeros((len(zeta_list),len(R))) # Phase
for i in range(0,len(zeta_list)):
    for j in range(0,len(R)):
        ze = zeta_list[i]
        r = R[j]
        M = 1 / sqrt( (1-r**2)**2 + (2*ze*r)**2 )
        M_history[i,j] = M
        th = rad2deg(atan(2*ze*r**2 / (1-r**2)))
        if th >= 0:
            PH = rad2deg(atan(2*ze*r**2 / (1-r**2)))
        elif th < 0:
            PH = 180+rad2deg(atan(2*ze*r**2 / (1-r**2)))
        else:
            PH = 0
        PH_history[i,j] = PH

#************************************************************************************************

    ## Results and Responses:

        ### Abs response x_eq(t):

fig1 = plt.figure()
rect = fig1.patch

graphW = fig1.add_subplot(1,1,1)
graphW.plot(time, Xw_history, linewidth=2.0)
graphW.plot(time, Fw_history, linewidth=1.0)
graphW.legend(['x_w(t) [cm]', 'f(t) [MN]'], fontsize=12, loc='lower right')
graphW.set_title(' "Wind System Response" ', fontsize=25)
graphW.set_xlabel('Time [s]', fontsize=20)
graphW.grid(True)

        ### Amplification Factor:

fig2a = plt.figure()
rect = fig2a.patch
fig2b = plt.figure()
rect = fig2b.patch

for i in range(0,len(zeta_list)):
    graphW2a = fig2a.add_subplot(1,1,1)
    graphW2a.plot(R, M_history[i,0:], linewidth=1.5)
    graphW2a.legend( ["${}$".format( vlatex(ze_list[k])+' = '+str(zeta_list[k])) for k in range(0,len(zeta_list))  ] , fontsize=13, loc='upper right' )
    graphW2a.set_title(' "Amplification factor with r for Wind" ', fontsize=25)
    graphW2a.set_ylabel('M = Xw/x_st', fontsize=20)
    graphW2a.set_xlabel('r = w_w/wn', fontsize=20)
    graphW2a.grid(True)
    graphW2b = fig2b.add_subplot(1,1,1)
    graphW2b.plot(R, PH_history[i,0:], linewidth=1.5)
    graphW2b.legend( ["${}$".format( vlatex(ze_list[k])+' = '+str(zeta_list[k])) for k in range(0,len(zeta_list))  ] , fontsize=13, loc='lower right' )
    graphW2b.set_title(' "Phase with r for Wind" ', fontsize=25)
    graphW2b.set_ylabel('Phase [deg]', fontsize=20)
    graphW2b.set_xlabel('r = w_w/wn', fontsize=20)
    graphW2b.grid(True)

plt.show()


from zquantum.core.interfaces.optimizer import Optimizer
from scipy.optimize import OptimizeResult
import cma
import copy

class CMAESOptimizer(Optimizer):

    def __init__(self, options):
        if "sigma_0" not in options.keys():
            raise RuntimeError("Error: CMAESOptimizer input options dictionary must contain \"sigma_0\" field")
        else:
            self.sigma_0 = options.pop('sigma_0')
        self.options = options

    def minimize(self, cost_function, initial_params):
        """Minimize using the Covariance Matrix Adaptation Evolution Strategy
        (CMA-ES).

        Args:
            cost_function: the function whose return value is to be minimized.
            initial_params (numpy.ndarray): initial guess for the ansatz parameters.

        Returns:
            tuple: A tuple containing an optimization results dict and a numpy array
                with the optimized parameters.
        """

        # Optimization Results Object
        history = []
        def wrapped_cost_function(params):

            value = cost_function(params)
            history.append({'params': params, 'value': value})
            print(f'Function evaluation {len(history)}: {value}', flush=True)
            print(f'{params}', flush=True)

            return value

        strategy = cma.CMAEvolutionStrategy(initial_params, self.sigma_0, self.options)
        result = strategy.optimize(wrapped_cost_function).result

        optimization_results = {}
        optimization_results['opt_value'] = result.fbest
        optimization_results['opt_params'] = result.xbest
        optimization_results['history'] = history
        optimization_results['nfev'] = result.evaluations
        optimization_results['nit'] = result.iterations
        optimization_results['cma_xfavorite'] = list(result.xfavorite)

        return OptimizeResult(optimization_results)


import cobra
import sys

#Read the model in
model_file_path = str(model_file_path)
toycon = cobra.io.read_sbml_model(model_file_path)
#Get objective and assert it is in the model
objective = str(objective)
try:
    toycon.reactions.get_by_id(objective)
except KeyError:
    print("The selected objective reaction is not in the model!")
#Change objective
toycon.objective = objective
#Resetting the model case
solution = toycon.optimize()
flux = solution.objective_value
fluxes = solution.fluxes.round(2).to_dict()


# This file plots the difference between \sin{x} and x to analyze small angle approximation.
import math
import matplotlib.pyplot as plt
import numpy as np

x = np.linspace(0,math.pi/2,100)
y_x = x
y_sin = np.sin(x)

plt.figure()
plt.subplot(1,2,1)
plt.xlabel("theta in degrees")
plt.ylabel("Y values")
plt.plot(x*180/math.pi,y_x)
plt.plot(x*180/math.pi,y_sin)
plt.xticks(np.arange(0,90,10))

plt.subplot(1,2,2)
plt.xlabel("theta in degrees")
plt.ylabel("Diff in Y values")
plt.plot(x*180/math.pi,y_x-y_sin)
plt.xticks(np.arange(0,90,10))
plt.yticks(np.arange(0,0.6,0.05))
plt.tight_layout()
plt.savefig("SmallAngle.png")
plt.show()


import argparse
import multiprocessing
import sys
import time
import re
from multiprocessing import Queue, Process, Event
from queue import Empty

import pygame.sprite

from Engine.Level import Level

# def connect_to_reomte_game(remote_queues, player_queue, kill_signal):
#     while not kill_signal.is_set():
#         try:
#             player = player_queue.get_nowait()
#             for remote_queue in remote_queues:
#                 remote_queue.put(player)
#         except Empty:
#             pass
#     print("Connection Ending")

parser = argparse.ArgumentParser()
parser.add_argument("--server", help="Set the game up in server mode, "
                                     "use this if you have forwarded a port to accept connections",
                    action="store_true", default=False)
parser.add_argument("--host-ip", help="The ip address on which the connections will be made. "
                                      "For server, use the localhost, for the client use the public ip of the server.",
                    default='localhost', )
parser.add_argument("--fps", help="The target framerate to run at",
                    default=30, type=int)
parser.add_argument("--char", help="The character to play as",
                    default="Batman", type=str)
parser.add_argument("--port", help="The ip address on which the connections will be made. "
                                      "For server, use the localhost, for the client use the public ip of the server.",
                    default=33012, type=int)
parser.add_argument("-m", "--magnification", help="The level of magnifcation to use for rendering the game",
                    default=1.5, type=int)

def main():

    args = parser.parse_args()
    level = Level(
        map_name="test_map_new_format",
        player_character_name=args.char,
        player_start_pos=(20 + 32 if args.server else 64, 80),
        window_size=(800, 400),
        magnification=args.magnification,
        server=args.server,
        host=args.host_ip,
        port=args.port,
    )
    level.run(framerate=args.fps)

    #   EXANPLE OF 4 WAY MULTIPLAYER, NOT READY TO DELETE JUST YET
    # multiprocessing.set_start_method("spawn")
    # pygame.init()
    # WINDOW_SIZE = (600, 400)
    #
    # kill_signal = Event()
    # if args.server:
    #     chars = {0: "Treeman", 1: "Batman", }
    # else:
    #     chars = {0: "Batman", 1: "Treeman", }
    #
    # char_queues = {player_idx: [Queue() for c in chars if c is not player_idx] for player_idx, player_name in chars.items()}
    #
    # remote_queues = {}
    # for player_idx in char_queues.keys():
    #     remote_queues[player_idx] = []
    #     for remote_idx, queues in char_queues.items():
    #         if remote_idx != player_idx:
    #             remote_queues[player_idx].append(queues[player_idx%len(chars)-1])
    #
    # levels = [Level(
    #     map_name="test_map_new_format",
    #     player_character_name=player_name,
    #     player_start_pos=(20 + player_idx * 32, 80),
    #     npc_names=[npc_name for npc_idx, npc_name in chars.items() if player_name != npc_name],
    #     npc_start_positions=[(20 + npc_idx * 320, 80) for npc_idx, npc_name in chars.items() if player_name != npc_name],
    #     window_size=WINDOW_SIZE,
    #     player_state_queue=char_queues[player_idx],
    #     npc_state_queues=remote_queues[player_idx],
    #     server=args.server,
    # ) for player_idx, player_name in chars.items()]
    #
    # processes = [Process(target=level.run, args=(kill_signal, 60)) for level in levels]
    #
    # for proc in processes:
    #     proc.start()
    #
    # while not kill_signal.is_set():
    #     print("Tick", kill_signal.is_set())
    #     time.sleep(1)

if __name__=="__main__":
    main()


import os
import sys
import requests
import jwt
import datetime
from dotenv import load_dotenv
from time import time
# load API key and secret from .bashrc
load_dotenv()

API_KEY = os.getenv("Zoom_KEY")
API_SEC = os.getenv("Zoom_SECRET")

# get the arguments from the command line
meeting_id = sys.argv[1]
email = sys.argv[2]
first_name = sys.argv[3]
last_name = sys.argv[4]

# encode the api key and secret
header = {"alg": "HS256", "typ": "JWT"}
payload = {
    "iss": API_KEY,
    "exp": time() + 5000
}
token = jwt.encode(payload, API_SEC, algorithm="HS256", headers=header)

# url for adding a registrant to a meeting
url = f"https://api.zoom.us/v2/meetings/{meeting_id}/registrants"

# headers
headers = {
    "Authorization": f"Bearer {token}",
    "Content-Type": "application/json"
}

# data for the post request
data = {
    "email": email,
    "first_name": first_name,
    "last_name": last_name
}

# make the post request
response = requests.post(url, headers=headers, json=data)

# print the response
print(response.json())