from PIL import Image
import cv2
import torch
from dataset.dataloader import ResNetDataset
import torch.optim as optim
import torch.nn as nn
import torchvision.models as models
import os
import time
import numpy as np
import argparse
from network.models import *
from network.metrics import *
# from torchsampler import ImbalancedDatasetSampler

# ---------------------Parsing options---------------------
parser = argparse.ArgumentParser(description="Train face_mask, face_no_mask classification")
#--------------------Traing strategy--------------------
parser.add_argument("--optimizer", type=str, default="Adam", help="Adam, SGD")
parser.add_argument("--learning-rate", type=float, default=0.001)
parser.add_argument("--momentum", type=float, default=0.9)
parser.add_argument("--batch-size", type=int, default=4)
parser.add_argument("--num-classes", type=int, default=3)
parser.add_argument("--start-epoch", type=int, default=0)
parser.add_argument("--end-epoch", type=int, default=400)
parser.add_argument("--gamma", type=float, default=0.05)
parser.add_argument("--step", nargs='+', type=int, default=[6,12,24,48,96,192])
parser.add_argument("--with-arc", type=bool, default=False)
parser.add_argument("--dataloader", type=int, default=1)
#--------------------Data options--------------------
parser.add_argument("--alt-data-dir", type=str, default='data', required=False)
parser.add_argument("--base-data-dir", type=str, default='data')
parser.add_argument("--data-path", type=str, default='/mnt/sdb1/data/widerface/train/cls_data')
#--------------------model options--------------------
parser.add_argument("--start-switch", type=str, default=None)
parser.add_argument("--model", type=str, default="MobileFaceNetUltraLite")
parser.add_argument("--model-restore", type=str, default="/mnt/sdb3/Git_mess/mask_on_face_cls/checkpoints/best.pth")
parser.add_argument("--head-restore", type=str, default=None)
parser.add_argument("--model-save", type=str, default="checkpoints/mBFN.pth")
parser.add_argument("--head-save", type=str, default="checkpoints/mBFN_head.pth")
#--------------------ArcLoss options --------------------
parser.add_argument("--m", type=float, default=10)
parser.add_argument("--s", type=float, default=0.3)

args = parser.parse_args()
print(args)

criterion = nn.CrossEntropyLoss()
# save_path = "checkpoints/model.pth"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
if args.with_arc == False:
    if args.model == 'resnet18':
        Net = models.resnet18(num_classes=1000, pretrained=True)
        Net.fc = nn.Sequential(nn.Dropout(0.2),
                        nn.ReLU(),
                        nn.BatchNorm1d(512),
                        nn.Linear(512, args.num_classes)
                        )

    elif args.model == 'resnet34':
        Net = models.resnet34(num_classes=1000, pretrained=True)
        Net.fc = nn.Sequential(nn.Dropout(0.2),
                        nn.ReLU(),
                        nn.BatchNorm1d(512),
                        nn.Linear(512, args.num_classes)
                        )

    elif args.model == "MobileFaceNet":
        Net = MobileFaceNet(embedding_size=args.num_classes)

    elif args.model == "MobileFaceNetLite":
        Net = MobileFaceNetLite(embedding_size=args.num_classes)

    elif args.model == "MobileFaceNetSuperLite":
        Net = MobileFaceNetSuperLite(embedding_size=args.num_classes)

    elif args.model == "MobileFaceNetUltraLite":
        Net = MobileFaceNetUltraLite(embedding_size=args.num_classes)

    else:
        print("Model not implemented {}".format(args.model))

    if args.model_restore is not None:
        if os.path.exists(args.model_restore):
            # Net.load_state_dict(torch.load(NET_PTH))
            Net = torch.load(args.model_restore, map_location=torch.device('cpu'))
            print("Model loaded from {}".format(args.model_restore))
        else:
            print("Model restore not exist")

    Net.to(device)
    Net.eval()
    params = [*Net.parameters()]

    if args.optimizer == "Adam":
        opt = optim.Adam(params, lr=args.learning_rate)
    elif args.optimizer == "SGD":
        opt = optim.SGD(params, lr=args.learning_rate, momentum=args.momentum)
    else:
        print("Not implemented Optimizer {}".format(args.optimizer))

    my_lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=args.step, gamma=args.gamma)
    Uta = ResNetDataset(path=args.data_path, repl=(args.base_data_dir, args.alt_data_dir))
    data = torch.utils.data.DataLoader(Uta, batch_size=args.batch_size, shuffle=True)
    best = 0
    print("init net done")

    def save_progress(state, epoch, train_loss, train_acc):
        file = open("./progress.txt", "a")
        txt = "{}  epoch: {}       loss: {:.5}      acc: {:.5} ".format(state, epoch, train_loss, train_acc)
        file.write(txt+"\n")
        file.close()

    for epoch in range(args.start_epoch, args.end_epoch):
        start = time.time()
        total = 0
        correct = 0
        vtotal = 0
        vcorrect = 0

        classes_correct = [0 ,0]
        classes_total = [0, 0]
        for i, d in enumerate(data):
            [X, Y] = d[0].to(device), d[1].to(device)

            opt.zero_grad()

            out = Net(X)
            idx = np.zeros(len(Y))
            # print(out[0].shape)
            # print(Y.shape)
            for j, o in enumerate(out):
                idx[j] = o.data.cpu().numpy().argmax()
            idx = torch.from_numpy(idx).type(torch.int64)

            for j in range((len(idx))):
                if Y[j]==0:
                    classes_total[0]+=1
                else:
                    classes_total[1]+=1

                if idx[j] == Y[j] == 0:
                    classes_correct[0] += 1
                    correct += 1
                elif idx[j] == Y[j] == 1:
                    classes_correct[1] += 1
                    correct += 1
                else:
                    continue

            total += len(idx)

        acc = 100*correct/total
        print(f"[INFO]: acc {acc} classes_acc = [{classes_correct[0]/classes_total[0]},{classes_correct[1]/classes_total[1]}]")
        print(classes_total[0], classes_total[1], total)
# # Train with arcloss
# else:
#     if args.model == "resnet18" :
#         Net = models.resnet18(num_classes=1000, pretrained=True)
#         Net = nn.Sequential(*(list(Net.children())[:-2]))
#         Net.avgpool = nn.Sequential(nn.Flatten(),
#                     nn.Linear(512*7*7, 512))
#         head = ArcMarginModel(emb_size=512, num_classes=args.num_classes, margin_s=args.m, margin_m=args.s)

#     elif args.model == "MobileFaceNet":
#         if args.start_switch is not None:
#             Net = torch.load(args.start_switch)
#             Net = nn.Sequential(*(list(Net.children())[:-2]))
#             Net.flat = nn.Flatten()
#             Net.linear = nn.Sequential(nn.Linear(512*8*8, 2048),
#                                 nn.ReLU(),
#                                 nn.BatchNorm1d(2048))
#             print("ArcFace model loaded from swtich")
#         else:
#             Net = MobileFaceNet(embedding_size=3)
#             Net = nn.Sequential(*(list(Net.children())[:-2]))
#             Net.flat = nn.Flatten()
#             Net.linear = nn.Sequential(nn.Linear(512*8*8, 2048),
#                             nn.ReLU(),
#                             nn.BatchNorm1d(2048))
#         head = ArcMarginModel(emb_size=2048, num_classes=args.num_classes, margin_s=args.m, margin_m=args.s)

#     elif args.model == "MobileFaceNetLite":
#         head = ArcMarginModel(emb_size=64, num_classes=args.num_classes, margin_s=args.m, margin_m=args.s)
#         if args.start_switch is not None:
#             Net = torch.load(args.start_switch)
#             print(Net)
#             head.weight = Net.linear.weight
#             Net = nn.Sequential(*(list(Net.children())[:-1]))
#             Net.flat = nn.Flatten()
#             print("ArcFace model loaded from swtich")
#         else:
#             Net = MobileFaceNetLite(embedding_size=3)
#             Net = nn.Sequential(*(list(Net.children())[:-1]))
#             Net.flat = nn.Flatten()

#     elif args.model == "MobileFaceNetSuperLite":
#         head = ArcMarginModel(emb_size=64, num_classes=args.num_classes, margin_s=args.m, margin_m=args.s)
#         if args.start_switch is not None:
#             Net = torch.load(args.start_switch)
#             print(Net)
#             head.weight = Net.linear.weight
#             Net = nn.Sequential(*(list(Net.children())[:-1]))
#             Net.flat = nn.Flatten()
#             print("ArcFace model loaded from swtich")
#         else:
#             Net = MobileFaceNetSuperLite(embedding_size=3)
#             Net = nn.Sequential(*(list(Net.children())[:-1]))
#             Net.flat = nn.Flatten()

#     elif args.model == "MobileFaceNetUltraLite":
#         head = ArcMarginModel(emb_size=64, num_classes=args.num_classes, margin_s=args.m, margin_m=args.s)
#         if args.start_switch is not None:
#             Net = torch.load(args.start_switch)
#             print(Net)
#             head.weight = Net.linear.weight
#             Net = nn.Sequential(*(list(Net.children())[:-1]))
#             Net.flat = nn.Flatten()
#             print("ArcFace model loaded from swtich")
#         else:
#             Net = MobileFaceNetUltraLite(embedding_size=3)
#             Net = nn.Sequential(*(list(Net.children())[:-1]))
#             Net.flat = nn.Flatten()

#     else:
#         print("Model not implemented {}".format(args.model))

#     if args.model_restore is not None and args.start_switch is None:
#         if os.path.exists(args.model_restore):
#             # Net.load_state_dict(torch.load(NET_PTH))
#             Net = torch.load(args.model_restore)
#             print("Model loaded from {}".format(args.model_restore))
#         else:
#             print("Model restore not exist")

#     if args.head_restore is not None:
#         if os.path.exists(args.head_restore):
#             # Net.load_state_dict(torch.load(NET_PTH))
#             head.weight = torch.load(args.head_restore)
#             print("Head loaded from {}".format(args.head_restore))
#         else:
#             print("Head restore not exist")

#     Net.cuda()
#     Net.train()
#     params = [*Net.parameters()] + [head.weight]

#     if args.optimizer == "Adam":
#         opt = optim.Adam(params, lr=args.learning_rate)
#     elif args.optimizer == "SGD":
#         opt = optim.SGD(params, lr=args.learning_rate, momentum=args.momentum)
#     else:
#         print("Not implemented Optimizer {}".format(args.optimizer))

#     my_lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=args.step, gamma=args.gamma)

#     Uta = ResNetDataset(path=args.data_path, repl=(args.base_data_dir, args.alt_data_dir))
#     data = torch.utils.data.DataLoader(Uta, batch_size=args.batch_size, shuffle=True)
#     # Valid = ResNetDataset(path="./data/valid")
#     # valid_data = torch.utils.data.DataLoader(Valid, batch_size=4, shuffle=True)
#     best = 0
#     print("init net done")

#     def save_progress(state, epoch, train_loss, train_acc):
#         file = open("./progress.txt", "a")
#         txt = "{}  epoch: {}       loss: {:.5}      acc: {:.5} ".format(state, epoch, train_loss, train_acc)
#         file.write(txt+"\n")
#         file.close()

#     for epoch in range(args.start_epoch, 400):
#         running_loss = 0
#         train_loss = 0
#         valid_loss = 0
#         start = time.time()
#         total = 0
#         correct = 0
#         vtotal = 0
#         vcorrect = 0
#         for i, d in enumerate(data):
#             [X, Y] = d[0].to(device), d[1].to(device)
#             opt.zero_grad()

#             out = Net(X)
#             out = head(out, Y)
#             loss = criterion(out, Y)
#             loss.backward()
#             opt.step()
#             idx = np.zeros(len(Y))
#             # print(out[0].shape)
#             # print(Y.shape)
#             for j, o in enumerate(out):
#                 idx[j] = o.data.cpu().numpy().argmax()
#             idx = torch.from_numpy(idx).type(torch.int64)

#             for j in range((len(idx))):
#                 if idx[j] - Y[j] == 0:
#                     correct += 1
#             total += len(idx)

#             running_loss += loss.item()
#             train_loss += loss.item()*len(Y)
#             if i % 10 == 9:
#                 print("[{}, {}], loss {:.10} in {:.5}s acc: {:.5}%".format(epoch+1, i+1, running_loss/10, time.time()-start, 100*correct/total))
#                 save_progress(state="   BATCH [{}, {}]".format(i-10, i), epoch= epoch+1, train_loss=running_loss/10, train_acc=100*correct/total)
#                 start = time.time()
#                 running_loss = 0.0

#         print ("====== Epoch {} Loss: {:.5}, acc: {:.5}% ======".format(epoch+1, train_loss/len(data.sampler), 100*correct/total))
#         my_lr_scheduler.step()
#         if best <= 100*correct/total:
#             torch.save(Net, args.model_save)
#             # torch.save(metric_fc.weight, HEAD_PTH)
#             print("model saved to {}".format(args.model_save))
#             best = 100*correct/total
#             save_progress(state="SAVED   ", epoch= epoch+1, train_loss=train_loss/len(data.sampler), train_acc=best)

#         else:
#             print("model not saved as best >= acc, current best : {}".format(best))
#             save_progress(state="FAIL    ", epoch= epoch+1, train_loss=train_loss/len(data.sampler), train_acc=100*correct/total)


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Output, Input
import plotly.express as px
import dash_bootstrap_components as dbc
import pandas as pd
from raceplotly.plots import barplot

def human_format(num):
    num = float('{:.3g}'.format(num))
    magnitude = 0
    while abs(num) >= 1000:
        magnitude += 1
        num /= 1000.0
    return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude])

pd.options.display.max_columns = None
pd.options.display.max_rows = None

df = pd.read_csv('macrodata.csv')
indicators = ["GDP, $", "Population, total", "GDP per capita, $", "GDP per capita, PPP (current international $)",
              "Adjusted net national income (current $)", "Adjusted net national income per capita (current $)",
              "Agriculture, forestry, and fishing, value added (% of GDP)",
              "Industry (incl construction), value added (% of GDP)", "Manufacturing, value added (% of GDP)",
              "Services, value added (% of GDP)", "National currency/USD ex-rate"]
countries = df['country'].unique()
years = df['year'].unique()

default_countries = ['Finland', 'Indonesia', 'United States of America', 'Russian Federation', 'Bulgaria']
default_indicator = 'Services, value added (% of GDP)'
default_years = [2010, 2019]

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP],
                meta_tags=[{'name': 'viewport',
                            'content': 'width=device-width, initial-scale=1.0'}]
                )
server = app.server

country_dropdown = dcc.Dropdown(id='country-selector', multi=True,
                                value=default_countries,
                                options=[{'label': country, 'value': country} for country in countries],
                                placeholder='Select one ore more countries')

indicator_dropdown = dcc.Dropdown(id='indicator-selector', multi=False, value=default_indicator,
                                  options=[{'label': indicator, 'value': indicator} for indicator in indicators],
                                  placeholder='Select the indicator', clearable=False)

year_slider = dcc.RangeSlider(id='year-slider', min=2001, max=2019, step=1, value=default_years,
                              marks={str(year): str(year) for year in years})

macro_graph = dbc.Card(
    dbc.CardBody([
        dcc.Graph(id='macro-graph', figure={},
                  config={
                      'staticPlot': False,  # True, False
                      'scrollZoom': True,  # True, False
                      'doubleClick': 'reset+autosize',  # 'reset', 'autosize' or 'reset+autosize', False
                      'showTips': True,  # True, False
                      'displayModeBar': True,  # True, False, 'hover'
                      'displaylogo': False
                  })
    ])
)

macro_pie = dbc.Card(
    dbc.CardBody([
        dcc.Graph(id='macro-pie', figure={},
                  config={
                      'displayModeBar': True,  # True, False, 'hover'
                      'displaylogo': False,
                  })
    ])
)

bar_chart_race = dbc.Card(
    dbc.CardBody([dcc.Graph(id='bar-chart-race', figure={})])
)

app.layout = dbc.Container([
    dbc.Row(
        dbc.Col(
            html.H1('Macro Data Dashboard', className='text-center text-primary'),
            width=12)
    ),
    dbc.Row([
        dbc.Col(
            dbc.Card(
                dbc.CardBody([
                        dbc.Row([
                            dbc.Col(country_dropdown, width=8),
                            dbc.Col(indicator_dropdown, width=4),
                        ]),
                        dbc.Row([dbc.Col(year_slider, width=12)])
                ])),
            width=12,
        )
    ]),
    dbc.Row([
        dbc.Col(macro_graph, width=6),
        dbc.Col(macro_pie, width=6)
    ]),
    dbc.Row([
        dbc.Col(bar_chart_race, width=12)
    ])

], fluid=True)

# Callback section: connecting the components
# ************************************************************************
# 5-macro-graph
@app.callback(
    Output(component_id='macro-graph', component_property='figure'),
    Input(component_id='country-selector', component_property='value'),
    Input(component_id='year-slider', component_property='value'),
    Input(component_id='indicator-selector', component_property='value'),
)
def update_macro_graph(countries, years, indicator):
    if not countries:
        return {}
    dff = df[df['country'].isin(countries)]
    dff = dff[(dff['year'] >= years[0]) & (dff['year'] <= years[1])]
    figure = px.line(data_frame=dff, x='year', y=indicator, color='country',
                     title=f'{indicator} in {" - ".join([str(year) for year in years])}',
                     # category_orders={'country': countries},
                     labels={
                         'year': 'Year',
                         'country': 'Country'

                     })
    figure.update_traces(mode='lines+markers', hovertemplate=None)
    figure.update_layout({'hovermode': 'x unified'})
    return figure

@app.callback(
    Output(component_id='macro-pie', component_property='figure'),
    Input(component_id='macro-graph', component_property='hoverData'),
    Input(component_id='macro-graph', component_property='clickData'),
    Input(component_id='macro-graph', component_property='selectedData'),
    Input(component_id='country-selector', component_property='value'),
    Input(component_id='year-slider', component_property='value'),
    Input(component_id='indicator-selector', component_property='value'),
)
def update_macro_pie(hov_data, clk_data, slctd_data, countries, years, indicator):
    if not countries:
        return {}
    dff2 = df[df['country'].isin(countries)]
    if slctd_data:
        first_year = slctd_data['points'][0]['x']
        last_year = slctd_data['points'][-1]['x']
        dff2 = dff2[(dff2['year'] >= first_year) & (dff2['year'] <= last_year)]
        dff2 = dff2.groupby(['country'], as_index=False)[[indicator]].mean()
        dff2 = dff2.sort_values(by=['country'])
        period = f'{first_year} - {last_year}'
    elif not hov_data:
        dff2 = dff2[(dff2['year'] >= years[0]) & (dff2['year'] <= years[1])]
        period = " - ".join([str(year) for year in years])
    else:
        hov_year = hov_data['points'][0]['x']
        period = hov_year
        dff2 = dff2[dff2.year == hov_year]
    figure2 = px.pie(data_frame=dff2, values=indicator, names='country',
                     title=f'{indicator} in {period}',
                     color='country'
                     )

    figure2.update_layout(legend=dict(orientation="h"))

    figure2.update_traces(hovertemplate=
                          "%{label} - %{percent}

" +
                          "Value: %{value:,.0f}
" +
                          "<extra></extra>",
                          sort=False)
    # figure2.update_layout(legend_traceorder="normal")
    return figure2

@app.callback(
    Output(component_id='bar-chart-race', component_property='figure'),
    Input(component_id='country-selector', component_property='value'),
    Input(component_id='year-slider', component_property='value'),
    Input(component_id='indicator-selector', component_property='value')
)
def update_bar_chart_race(countries, years, indicator):
    dff = df[df['country'].isin(countries)]
    dff = dff[(dff['year'] >= years[0]) & (dff['year'] <= years[1])]

    my_raceplot = barplot(dff, item_column='country', value_column=indicator, time_column='year',
                          top_entries=len(countries))

    fig = my_raceplot.plot(title=f'{indicator} per Country from {years[0]} to {years[1]}',
                           value_label=indicator, time_label='Year: ', frame_duration=800)

    fig.update_layout(plot_bgcolor=None, template='ggplot2',
                      autosize=False, height=500)
    fig.update_xaxes(automargin=True)

    fig.update_traces(hovertemplate=
                      "%{y} - %{x:.4s}" +
                      "<extra></extra>")

    return fig

if __name__ == '__main__':
    app.run_server()


# import the necessary packages
import tensorflow.keras.backend as K
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.pyplot import figure

def make_pairs(images, labels):
    # initialize two empty lists to hold the (image, image) pairs and
    # labels to indicate if a pair is positive or negative
    pairImages = []
    pairLabels = []

    # calculate the total number of classes present in the dataset
    # and then build a list of indexes for each class label that
    # provides the indexes for all examples with a given label
    numClasses = len(np.unique(labels))
    idx = [np.where(labels == i)[0] for i in range(0, numClasses)]

    # loop over all images
    for idxA in range(len(images)):
        # grab the current image and label belonging to the current
        # iteration
        currentImage = images[idxA]
        label = labels[idxA]

        # randomly pick an image that belongs to the *same* class
        # label
        idxB = np.random.choice(idx[label])
        posImage = images[idxB]

        # prepare a positive pair and update the images and labels
        # lists, respectively
        pairImages.append([currentImage, posImage])
        pairLabels.append([1])

        # grab the indices for each of the class labels *not* equal to
        # the current label and randomly pick an image corresponding
        # to a label *not* equal to the current label
        negIdx = np.where(labels != label)[0]
        negImage = images[np.random.choice(negIdx)]

        # prepare a negative pair of images and update our lists
        pairImages.append([currentImage, negImage])
        pairLabels.append([0])

    # return a 2-tuple of our image pairs and labels
    return (np.array(pairImages), np.array(pairLabels))

def euclidean_distance(vectors):
    # unpack the vectors into separate lists
    (featsA, featsB) = vectors

    # compute the sum of squared distances between the vectors
    sumSquared = K.sum(K.square(featsA - featsB), axis=1,
        keepdims=True)

    # return the euclidean distance between the vectors
    return K.sqrt(K.maximum(sumSquared, K.epsilon()))

def plot_training(H, plotPath):
    # construct a plot that plots and saves the training history
    plt.style.use("ggplot")
    plt.figure()
    plt.plot(H.history["loss"], label="train_loss")
    plt.plot(H.history["val_loss"], label="val_loss")
    plt.title("Training Loss")
    plt.xlabel("Epoch #")
    plt.ylabel("Loss")
    plt.legend(loc="lower left")
    plt.savefig(plotPath)

def plot_confusion_matrix(cm,
						  plotPath,
                          target_names,
                          title='Confusion matrix',
                          cmap=None,
                          normalize=True):
    """
    A prettier confusion matrix
    """
    import matplotlib.pyplot as plt
    import numpy as np
    import itertools

    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    accuracy = np.trace(cm) / np.sum(cm).astype('float')
    misclass = 1 - accuracy

    if cmap is None:
        cmap = plt.get_cmap('Blues')

    plt.figure(figsize=(16, 14))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()

    if target_names is not None:
        tick_marks = np.arange(len(target_names))
        plt.xticks(tick_marks, target_names, rotation=45)
        plt.yticks(tick_marks, target_names)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 1.5 if normalize else cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt.text(j, i, "{:0.4f}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
        else:
            plt.text(j, i, "{:,}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))
    plt.savefig(plotPath)


import enum
import typing

class ChannelMode(enum.IntEnum):
    """Channel mode."""

    Servo = 0
    """The channel is a servo."""
    ServoMultiplied = 1  # unused, as far as I can tell
    """This value is defined within the Maestro Control Centre, but unused.

    It is not recommended to use this channel mode, as its purpose is unclear."""
    Output = 2
    """The channel is an output.

    Output channels are controlled with a value in the range [0, 1023], which maps onto
    the range [0, Vcc] volts."""
    Input = 3
    """The channel is an input.

    The value is in the range [0, 1), corresponding to the input voltage range [0, Vcc]
    volts."""

class USCParameter(enum.IntEnum):
    """Constants for getting and setting Maestro parameters."""

    Initialized = 0
    ServosAvailable = 1
    ServoPeriod = 2
    SerialMode = 3
    SerialTimeout = 6
    ChannelModes0To3 = 12
    ChannelModes4To7 = 13
    ChannelModes8To11 = 14
    ChannelModes12To15 = 15
    ChannelModes16To19 = 16
    ChannelModes20To23 = 17

    ServoMultiplier = 26

    # Servo 0 is at base, servo 1 is 9 up, servo 2 is 18 up, and so on.
    ServoHomeBase = 30
    ServoMinBase = 32
    ServoMaxBase = 33
    ServoNeutralBase = 34
    ServoRangeBase = 36
    ServoSpeedBase = 37
    ServoAccelerationBase = 38

_parameter_sizes: typing.Dict[USCParameter, int] = {
    USCParameter.Initialized: 8,
    USCParameter.ServosAvailable: 8,
    USCParameter.ServoPeriod: 8,
    USCParameter.SerialMode: 2,
    USCParameter.SerialTimeout: 16,
    USCParameter.ChannelModes0To3: 8,
    USCParameter.ChannelModes4To7: 8,
    USCParameter.ChannelModes8To11: 8,
    USCParameter.ChannelModes12To15: 8,
    USCParameter.ChannelModes16To19: 8,
    USCParameter.ChannelModes20To23: 8,
    USCParameter.ServoMultiplier: 8,
    USCParameter.ServoHomeBase: 15,
    USCParameter.ServoMinBase: 8,
    USCParameter.ServoMaxBase: 8,
    USCParameter.ServoNeutralBase: 15,
    USCParameter.ServoRangeBase: 6,
    USCParameter.ServoSpeedBase: 8,
    USCParameter.ServoAccelerationBase: 8,
}
"""Parameter sizes in bits

To find the number of bytes for the response, take math.ceil(size / 8.0).
"""

def get_parameter_size(parameter: USCParameter) -> int:
    if parameter >= USCParameter.ServoHomeBase:
        parameter = (
            (parameter - USCParameter.ServoHomeBase) % 9
        ) + USCParameter.ServoHomeBase

    if parameter in _parameter_sizes:
        return _parameter_sizes[parameter]
    else:
        raise KeyError(parameter)

class Request(enum.IntEnum):
    """Constants for Maestro USB control requests.

    See :doc:`protocol` for more details about control requests."""

    GetRawParameter = 129  # REQUEST_GET_PARAMETER
    SetRawParameter = 130  # REQUEST_SET_PARAMETER
    GetVariablesMicroMaestro = 131  # REQUEST_GET_VARIABLES
    SetServoVariable = 132  # REQUEST_SET_SERVO_VARIABLE
    SetTarget = 133  # REQUEST_SET_TARGET
    ClearErrors = 134  # REQUEST_CLEAR_ERRORS
    GetVariablesMiniMaestro = 135  # REQUEST_GET_SERVO_SETTINGS
    GetStack = 136  # REQUEST_GET_STACK
    GetCallStack = 137  # REQUEST_GET_CALL_STACK
    SetPWM = 138  # REQUEST_SET_PWM
    Reinitialize = 144  # REQUEST_REINITIALIZE
    EraseScript = 160  # REQUEST_ERASE_SCRIPT
    WriteScript = 161  # REQUEST_WRITE_SCRIPT
    SetScriptDone = 162  # REQUEST_SET_SCRIPT_DONE
    RestartScriptAtSubroutine = 163  # REQUEST_RESTART_SCRIPT_AT_SUBROUTINE
    RestartScriptAtSubroutineWithParameter = (
        164  # REQUEST_RESTART_SCRIPT_AT_SUBROUTINE_WITH_PARAMETER
    )
    RestartScript = 165  # REQUEST_RESTART_SCRIPT
    StartBootloader = 255  # REQUEST_START_BOOTLOADER

class ServoVariableMask(enum.IntEnum):
    Speed = 0
    Acceleration = 128


#!env python

import io
import urllib
import smtplib
import argparse
import numpy as np
import pandas as pd
import yahoo_fin.stock_info as yfsi
import matplotlib.pyplot as plt

parser = argparse.ArgumentParser(usage='Inquire the price for stock, need the stock ID argument')
parser.add_argument('--sid', '-s', type = str, help = 'stock ID')
args = parser.parse_args()
sid = args.sid
ctx = ''
ndone = 0

sh = 'http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=1'
sz = 'http://www.szse.cn/api/report/ShowReport?SHOWTYPE=xlsx&CATALOGID=1110&TABKEY=tab1&random=0.4178381198138461'

def parse1ashare(sid, p2s = False):
	global ndone
	if isinstance(sid, int) or isinstance(sid, np.int64):
		if sid < 10:
			sid = '00000' + str(sid) + '.SZ'
		elif sid < 100:
			sid = '0000' + str(sid) + '.SZ'
		elif sid < 1000:
			sid = '000' + str(sid) + '.SZ'
		elif sid < 10000:
			sid = '00' + str(sid) + '.SZ'
		elif sid >= 300000 and sid < 600000:
			sid = str(sid) + '.SZ'
		elif sid >= 600000:
			sid = str(sid) + '.SS'

	sdc = yfsi.get_data(sid, start_date='01/01/2012')[20:]
	if len(sdc) < 90:
		#print(f'{sid} too short history data')
		ndone += 1
		return
	del sdc['ticker']
	sdc = sdc.fillna(method = 'ffill')
	pprice = sdc['adjclose']
	t = sdc['open'] - sdc['close']
	pdiff = (t / t.max())
	pdiff[pdiff >= 0.45] = pprice.max()
	pdiff[pdiff < 0.45] = 0
	t = sdc['volume'] * ((sdc['close'] + sdc['open'] + sdc['high'] + sdc['low']) / 4)
	pvol = (t / t.max())
	pvol[pvol >= 0.45] = pprice.max() / 2
	pvol[pvol < 0.45] = 0

	tsd = pd.DataFrame(pprice)
	tsd['diff'] = pdiff
	tsd['vol'] = pvol
	sigdif = 0
	sigvol = 0
	if len(pdiff) > 0 and pdiff[-1] > pprice.max() / 2:
		sigdif = 1
	if len(pvol) > 0 and pvol[-1] > pprice.max() / 3:
		sigvol = 1
	if sigdif or sigvol:
		print(f'-----------\n{sid} signal diff={sigdif} vol={sigvol}')
		global ctx
		ctx += f'-----------\n{sid} signal diff={sigdif} vol={sigvol}\n'
	if p2s:
		tsd.plot()
		plt.title(f'{sid}')
		plt.show()
	ndone += 1

if sid and isinstance(sid, str) and len(sid) > 0:
	parse1ashare(sid, True)
else:
	shheader = {
		'X-Requested-With': 'XMLHttpRequest',
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
		'Referer': 'http://www.sse.com.cn/assortment/stock/list/share/'
	}
	req = urllib.request.Request(sh, headers = shheader)
	resp = urllib.request.urlopen(req)
	if not resp:
		print(f'Cannot open stock list on {sh}')
		exit(1)
	r = resp.read().decode('gb2312')
	if not r:
		print('Unknown char encoding on SH')
		exit(2)
	t = io.StringIO(r)
	a = pd.read_csv(t, sep = '\t')
	if a.empty or len(a.iloc[:, 0]) == 0 or a.iloc[0, 0] != 600000:
		print('Unexpected format, maybe SH website is changed')
		exit(3)
	sids = pd.Series(a.iloc[:, 0])

	urllib.request.urlretrieve(sz, 'sz.xlsx')
	b = pd.read_excel('sz.xlsx')
	if not r:
		print('Cannot open stock lsit file on SZ')
		exit(4)
	if b.empty or len(b.iloc[:, 4]) == 0 or b.iloc[0, 4] != 1:
		print('Unexpected format, maybe SZ website is changed')
		exit(5)

	sids = sids.append(pd.Series(b.iloc[:, 4]), ignore_index = True, verify_integrity = True)

	sids.apply(parse1ashare)
	print(len(sids))

	while (ndone < len(sids)):
		sleep(5)

	if len(ctx) > 0:
		print('sending email...')
		s = smtplib.SMTP(host = 'smtp.qq.com', port = 587)
		s.starttls()
		s.login('3168681165@qq.com', 'bsnidiijpefhdgbj')
		s.sendmail('3168681165@qq.com', ['3168681165@qq.com'], f'Subject: daily result\n\n{ctx}')
		s.quit()


import arcpy
import arcgis
import datetime
import os
import numpy

arcpy.env.overwriteOutput = True
def main():
    #TableImport()
    #caseBaseTable()
    violationsBaseTable()
    #actionsBaseTable()
    #Action()
    #Inspection()
    #Contact()
    Duration()
    #Compliance()
    #stDev()
    #Employee()
    #AddressField()
    #Geocode()
    #Update()

def TableImport():
    ceTables = {"CASE_MAIN" : ["STARTED", "1500"], "CASE_INSPECTIONS": ["COMPLETED_DATE", "1500"], "Case_Actions": ["COMPLETED_DATE", "1500"], "Case_Violations2": ["DATE_OBSERVED", "365"], "LICENSE2_Main": ["ISSUED", "1500"], "LICENSE2_Inspections": ["COMPLETED_DATE", "1500"], "LICENSE2_Actions": ["COMPLETED_DATE", "365"], "Permit_Reviews": ["DATE_SENT", "365"], "LICENSE2_REVIEWS": ["DATE_SENT", "365"], "Inspections": ["COMPLETED_DATE", "365"], "LICENSE2_Inspections": ["COMPLETED_DATE", "365"]}
    originalPath = "\\\\GISAPP\\Workspace\\sdeFiles\\LogosLive.sde"
    newPath = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb"
    for key, value in ceTables.items():
        originalFile = originalPath + "\\CRW_PROD.dbo." + key
        sql = value[0] + ">= CURRENT_DATE - " + value[1]
        in_mem_table = arcpy.management.CopyRows(originalFile, r"in_memory\tbl")
        arcpy.conversion.TableToTable(in_mem_table, newPath, key, sql)
        del in_mem_table

def caseBaseTable():
    #Creates Base Table to be consumed by Dashboard
    caseMain = ["CASE_NO", "STARTED", "CLOSED", "CaseSubType", "STATUS", "SITE_ADDR"]
    caseTable = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\CASE_MAIN"
    newTable = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb"
    #Creating Field mapping
    fieldmappings = arcpy.FieldMappings()
    fieldmappings.addTable(caseTable)
    #Removing all Fields that aren't needed in the end product
    for f in fieldmappings.fields:
            if f.name not in caseMain:
                fieldmappings.removeFieldMap(fieldmappings.findFieldMapIndex(f.name))
    arcpy.conversion.TableToTable(caseTable, newTable, "Dashboard_Case", "", fieldmappings)

def violationsBaseTable():
    #Creates Base Table to be consumed by Dashboard
    violationsMain = ["CASE_NO", "DATE_OBSERVED", "DATE_CORRECTED", "Violation_Type", "USERID", "STATUS"]
    violationsTable = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Case_Violations2"
    newTable = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb"
    #Creating Field mapping
    fieldmappings = arcpy.FieldMappings()
    fieldmappings.addTable(violationsTable)
    #Removing all Fields that aren't needed in the end product
    for f in fieldmappings.fields:
            if f.name not in violationsMain:
                fieldmappings.removeFieldMap(fieldmappings.findFieldMapIndex(f.name))
    arcpy.conversion.TableToTable(violationsTable, newTable, "Dashboard_Violations", "", fieldmappings)
    print("done")
def actionsBaseTable():
    #Creates Base Table to be consumed by Dashboard
    violationsMain = ["CASE_NO", "ACTION_BY", "ACTION_DATE", "COMPLETED_DATE", "ACTION_TYPE"]
    violationsTable = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Case_Actions"
    newTable = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb"
    #Creating Field mapping
    fieldmappings = arcpy.FieldMappings()
    fieldmappings.addTable(violationsTable)
    #Removing all Fields that aren't needed in the end product
    for f in fieldmappings.fields:
            if f.name not in violationsMain:
                fieldmappings.removeFieldMap(fieldmappings.findFieldMapIndex(f.name))
    arcpy.conversion.TableToTable(violationsTable, newTable, "Dashboard_Actions", "", fieldmappings)

def Action():
    firstAction = {}
    recentAction = {}
    #Identify First Action(Chronology or Inspection) taken on a case for use in the Dashboard
    #Search through Actions to find first response
    actionsLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Case_Actions"
    actionsFields = ["CASE_NO", "ACTION_BY", "ACTION_TYPE", "COMPLETED_DATE"]
    #Create Working Dictionary to determine which Action is first before adding to the First Action Dictionary
    workingDict = {}
    with arcpy.da.SearchCursor(actionsLayer, actionsFields) as Scur:
        for row in Scur:
            if row[0] in workingDict.keys():
                workingDict[row[0]].append(row[3])
            else:
                workingDict[row[0]] = [row[3]]

    dashLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Case"
    arcpy.management.AddField(dashLayer, "firstActionDate", "DATE")
    arcpy.management.AddField(dashLayer, "recentActionDate", "DATE")

    fieldsFA = ["CASE_NO", "firstActionDate", "recentActionDate"]
    with arcpy.da.UpdateCursor(dashLayer, fieldsFA) as Ucur:
        for row in Ucur:
            for key, value in workingDict.items():
                if key == row[0]:
                    fA = min(value)
                    rA = max(value)
                    row[1] = fA
                    row[2] = rA
                    Ucur.updateRow(row)

def Inspection():
    inspection = {}
    actionsLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Inspections"
    actionsFields = ["ActivityID", "COMPLETED_DATE"]

    workingDict = {}
    with arcpy.da.SearchCursor(actionsLayer, actionsFields) as Scur:
        for row in Scur:
            if row[0] in workingDict.keys():
                workingDict[row[0]].append(row[1])
            else:
                workingDict[row[0]] = [row[1]]

    dashLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Case"
    arcpy.management.AddField(dashLayer, "firstInspectionDate", "DATE")
    arcpy.management.AddField(dashLayer, "recentInspectionDate", "DATE")
    fields = ["CASE_NO", "firstInspectionDate", "recentInspectionDate"]
    with arcpy.da.UpdateCursor(dashLayer, fields) as Ucur:
        for row in Ucur:
            for key, value in workingDict.items():
                if key == row[0]:
                    ins = min(value)
                    newins = max(value)
                    row[1] = ins
                    row[2] = newins
                    Ucur.updateRow(row)

def Contact():
    dashLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Case"
    arcpy.management.AddField(dashLayer, "firstContactDate", "DATE")
    fields = ["firstActionDate", "firstInspectionDate", "firstContactDate"]
    with arcpy.da.UpdateCursor(dashLayer, fields) as Ucur:
        for row in Ucur:
            if row[0] is not None and row[1] is not None:
                datelst = []
                datelst.append(row[0])
                datelst.append(row[1])
                newDate = min(datelst)
                row[2] = newDate
                Ucur.updateRow(row)
            if row[0] is not None and row[1] is None:
                row[2] = row[0]
                Ucur.updateRow(row)
            if row[0] is None and row[1] is not None:
                row[2] = row[1]
                Ucur.updateRow(row)

    dashLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Case"
    arcpy.management.AddField(dashLayer, "recentContactDate", "DATE")
    fields = ["recentActionDate", "recentInspectionDate", "recentContactDate"]
    with arcpy.da.UpdateCursor(dashLayer, fields) as Ucur:
        for row in Ucur:
            if row[0] is not None and row[1] is not None:
                datelst = []
                datelst.append(row[0])
                datelst.append(row[1])
                newDate = min(datelst)
                row[2] = newDate
                Ucur.updateRow(row)
            if row[0] is not None and row[1] is None:
                row[2] = row[0]
                Ucur.updateRow(row)
            if row[0] is None and row[1] is not None:
                row[2] = row[1]
                Ucur.updateRow(row)

def Duration():
    dashLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Case"
    arcpy.management.AddField(dashLayer, "DurationContact", "DOUBLE")
    fields = ["STARTED", "firstContactDate", "DurationContact"]
    with arcpy.da.UpdateCursor (dashLayer, fields) as Ucur:
        for row in Ucur:
            if row[1] is None and row[0] is not None:
                duration = datetime.datetime.today() - row[0]
                row[2] = duration.days
                Ucur.updateRow(row)
            if row[1] is not None and row[0] is not None:
                duration = row[1] - row[0]
                row[2] = duration.days
                Ucur.updateRow(row)
            else:
                pass

    dashLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Violations"
    arcpy.management.AddField(dashLayer, "DurationComply", "DOUBLE")
    fields = ["DATE_OBSERVED", "DATE_CORRECTED", "DurationComply"]
    with arcpy.da.UpdateCursor (dashLayer, fields) as Ucur:
        for row in Ucur:
            if row[1] is None and row[0] is not None:
                duration = datetime.datetime.today() - row[0]
                row[2] = duration.days
                Ucur.updateRow(row)
            if row[1] is not None and row[0] is not None:
                duration = row[1] - row[0]
                row[2] = duration.days
                Ucur.updateRow(row)
            else:
                pass

    dictFields = ["CASE_NO", "DurationComply"]
    testDict = {}
    with arcpy.da.SearchCursor(dashLayer, dictFields) as Scur:
        for row in Scur:
            if row[0] in testDict.keys():
                testDict[row[0]].append(row[1])
            else:
                testDict[row[0]] = [row[1]]
    testlst = []
    perclst = []
    for key, value in testDict.items():
        newset = set(value)
        for x in newset:
            testlst.append(x)

    percent = numpy.percentile(testlst, 90)
    print(percent)
    print(testlst)

def Compliance():
    #Calculate compliance of Violation 10 days ultimate goal/ 30 day fallback goal
    violationPath = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Violations"
    arcpy.management.AddField(violationPath, "Compliance10", "TEXT")
    fields = ["DATE_OBSERVED", "DATE_CORRECTED", "Compliance10"]
    with arcpy.da.UpdateCursor(violationPath, fields) as Ucur:
        for row in Ucur:
            if row[0] is not None and row[1] is not None:
                duration = row[1] - row[0]
                comply = duration.days
                if comply <= 10:
                    row[2] = "Full Compliance"
                    Ucur.updateRow(row)
                else:
                    row[2] = "Not within Compliance"
                    Ucur.updateRow(row)
            if row[0] is not None and row[1] is None:
                duration = datetime.datetime.today() - row[0]
                comply = duration.days
                if comply <= 10:
                    row[2] = "Compliance to be Determined"
                    Ucur.updateRow(row)
                if comply > 10:
                    row[2] = "Not within Compliance"
                    Ucur.updateRow(row)
        del Ucur

    arcpy.management.AddField(violationPath, "Compliance30", "TEXT")
    fields = ["DATE_OBSERVED", "DATE_CORRECTED", "Compliance30"]
    with arcpy.da.UpdateCursor(violationPath, fields) as Ucur:
        for row in Ucur:
            if row[0] is not None and row[1] is not None:
                duration = row[1] - row[0]
                comply = duration.days
                if comply <= 30:
                    row[2] = "Full Compliance"
                    Ucur.updateRow(row)
                else:
                    row[2] = "Not within Compliance"
                    Ucur.updateRow(row)
            if row[0] is not None and row[1] is None:
                duration = datetime.datetime.today() - row[0]
                comply = duration.days
                if comply <= 30:
                    row[2] = "Compliance to be Determined"
                    Ucur.updateRow(row)
                if comply > 30:
                    row[2] = "Not within Compliance"
                    Ucur.updateRow(row)

def stDev():
    mainTable = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Violations"
    fields = ["DurationComply"]
    outlst =[]
    percentlst = []
    with arcpy.da.SearchCursor(mainTable, fields) as Scur:
        for row in Scur:
            if row[0] != None:
                outlst.append(row[0])
            else:
                pass

    arcpy.management.AddField(mainTable, "num_status", "TEXT")
    stdv = numpy.std(outlst)
    avgOutlier= sum(outlst) / len(outlst)
    sdtvOutlier = avgOutlier + stdv * 3
    fields = ["DurationComply", "num_status"]
    with arcpy.da.UpdateCursor(mainTable, fields) as Scur:
        for row in Scur:
            if row[0] == None:
                pass
            elif row[0] >= sdtvOutlier:
                row[1] = "Outlier"
                Scur.updateRow(row)
            else:
                percentlst.append(row[0])

    avg = sum(percentlst) / len(percentlst)
    numPercentile = numpy.percentile(percentlst, 90)
    print(avg, numPercentile)
    statusField = ["num_status", "DurationComply"]
    with arcpy.da.UpdateCursor(mainTable, statusField) as Ucur:
        for row in Ucur:
            if row[1] == None:
                continue
            if row[1] >= sdtvOutlier or row[1] == None:
                pass
            if row[1] > numPercentile and row[1] < sdtvOutlier:
                row[0] = "Not within 90th Percentile"
                Ucur.updateRow(row)
            if row[1] <= numPercentile:
                row[0] = "Within 90th Percentile"
                Ucur.updateRow(row)

def Employee():
    employeeDict = {}
    dashLayer = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Violations"
    arcpy.management.AddField(dashLayer, "EmployeeName", "TEXT")
    fields = ["UserID", "EmployeeName"]
    with arcpy.da.UpdateCursor (dashLayer, fields) as Ucur:
        for row in Ucur:
            for key, value in employeeDict.items():
                if row[0] == key:
                    row[1] = value
                    Ucur.updateRow(row)

def AddressField():
    caseMain = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Case"
    caseMainFields = ["CASE_NO", "SITE_ADDR"]
    addrDict = {}
    with arcpy.da.SearchCursor(caseMain, caseMainFields) as Scur:
        for row in Scur:
            addrDict[row[0]] = row[1]
    print("a")

    caseViolations = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Violations"
    arcpy.management.AddField(caseViolations, "SITE_ADDR", "TEXT")
    caseViolationsFields =  ["CASE_NO", "SITE_ADDR"]
    with arcpy.da.UpdateCursor(caseViolations, caseViolationsFields) as Ucur:
        for row in Ucur:
            for key,value in addrDict.items():
                if key == row[0]:
                    row[1] = value
                    Ucur.updateRow(row)
    print("b")

    caseActions = "\\\\GISAPP\\Workspace\\GIS Staff Workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_Actions"
    arcpy.management.AddField(caseActions, "SITE_ADDR", "TEXT")
    caseActionsFields =  ["CASE_NO", "SITE_ADDR"]
    with arcpy.da.UpdateCursor(caseActions, caseActionsFields) as Ucur:
        for row in Ucur:
            for key,value in addrDict.items():
                if key == row[0]:
                    row[1] = value
                    Ucur.updateRow(row)

def Geocode():
    layerName = ["Case"]
    for i in layerName:

        if arcpy.Exists("\\\\gisapp\\workspace\\gis staff workspace\\cschultz\\CodeEnforcement.gdb\\Analytics_{}".format(i)) == True:
            arcpy.management.Delete("\\\\gisapp\\workspace\\gis staff workspace\\cschultz\\CodeEnforcement.gdb\\Analytics_{}".format(i))
        arcpy.geocoding.GeocodeAddresses("\\\\gisapp\\workspace\\gis staff workspace\\cschultz\\CodeEnforcement.gdb\\Dashboard_{}".format(i), "\\\\gisapp\\Workspace\\MyArcGIS\\Geocoders\\AM_Composite.loc", "'Single Line Input' SITE_ADDR VISIBLE NONE", "\\\\gisapp\\workspace\\gis staff workspace\\cschultz\\CodeEnforcement.gdb\\Analytics_{}".format(i), "STATIC", None, '', None, "LOCATION_ONLY")
        print(i)

def Update():

    prjPath = "\\\\GISAPP\\Workspace\\Horizon\\ArcGISPro_Projects\\FeatureServices\\CodeEnforcement\\CodeEnforcement.aprx"

    dict = {
            "License_Main": ["License_Main","City Boundaries, City Limits, ETJ, General, Administrative Boundaries, Polygon", "This layer contains the City Limits and Extra Territorial Jurisdiction (ETJ) for the City of Pearland.",0],
            "Permit_Reviews": ["Permit_Reviews","City Boundaries, City Limits, ETJ, General, Administrative Boundaries, Polygon", "This layer contains the City Limits and Extra Territorial Jurisdiction (ETJ) for the City of Pearland.",1],
            "License_Reviews": ["License_Reviews","City Boundaries, City Limits, ETJ, General, Administrative Boundaries, Polygon", "This layer contains the City Limits and Extra Territorial Jurisdiction (ETJ) for the City of Pearland.",2],
            "Inspections": ["Inspections","City Boundaries, City Limits, ETJ, General, Administrative Boundaries, Polygon", "This layer contains the City Limits and Extra Territorial Jurisdiction (ETJ) for the City of Pearland.",3],
            "Dashboard_Case": ["Dashboard_Case","City Boundaries, City Limits, ETJ, General, Administrative Boundaries, Polygon", "This layer contains the City Limits and Extra Territorial Jurisdiction (ETJ) for the City of Pearland.",4],
            "Dashboard_Actions": ["Dashboard_Actions","City Boundaries, City Limits, ETJ, General, Administrative Boundaries, Polygon", "This layer contains the City Limits and Extra Territorial Jurisdiction (ETJ) for the City of Pearland.",5],
            "Dashboard_Violations": ["Dashboard_Violations","City Boundaries, City Limits, ETJ, General, Administrative Boundaries, Polygon", "This layer contains the City Limits and Extra Territorial Jurisdiction (ETJ) for the City of Pearland.",6]
            }
    n=0

    for key, value in dict.items():
        sd_fs_name = "{}".format(key)
        portal = "https://pearland.maps.arcgis.com"
        user = ""
        password = ""

        shrOrg = True
        shrEveryone = True

        relPath = "\\\\GISAPP\\Workspace\\Horizon\\Scripts"
        sddraft = os.path.join(relPath, "WebUpdate.sddraft")
        sd = os.path.join(relPath, "WebUpdate.sd")

        arcpy.env.overwriteOutput = True
        prj = arcpy.mp.ArcGISProject(prjPath)
        m = prj.listMaps('Map')[0]
        mp = m.listTables()[value[3]]
        arcpy.mp.CreateWebLayerSDDraft(mp, sddraft, sd_fs_name, 'MY_HOSTED_SERVICES', 'FEATURE_ACCESS','CodeEnforcement', True, True, False, True)
        arcpy.StageService_server(sddraft, sd)

        gis = arcgis.GIS(portal, user, password)
        searchItem = gis.content.search(query="title:"+ value[0] + " AND owner: " + user, item_type="Service Definition", sort_field='title', sort_order='asc')
        print(searchItem)
        for item in searchItem:
            print(item["title"])
            if item["title"] == key:
                sdItem = item
        sdItem.update(data=sd)
        sdItem.update(item_properties={'tags': '{}'.format(value[1]), 'snippet': "This Table is updated automatically through a Python Script.",
                                        'description': '{}'.format(value[2])})

        fs = sdItem.publish(overwrite=True)
        fs.share(org=shrOrg, everyone=shrEveryone)
        print(sd_fs_name, "done")

main()
print(".")


import numpy as np
import matplotlib.pyplot as plt
import sympy as sy
from sympy.utilities.lambdify import lambdify

def f(x):
    x = sy.symbols('x')
    return sy.sin(x)/x

def g(x):
    return sy.exp(x)

def taylor_expansion(f, a=1, n=3):
    x = sy.symbols('x')
    func = f(x)
    result = func.subs(x, a)
    for i in range(1, n):
        result += sy.diff(func, x, i).subs(x, a) * ((x - a)**i)/(sy.factorial(i))
    sy.pretty_print(result)
    return lambdify(x, result)

x = np.linspace(-9.4, 9.4, 200)

func_to_plot = taylor_expansion(f, n=10)

plt.plot(x, np.sin(x), 'k', label='Analytic')
plt.plot(x, [func_to_plot(i) for i in x])
plt.legend()
plt.show()

taylor_expansion(g, n=6)


import spacy
import json
from csv import DictWriter
from spacy.matcher import Matcher

# Load English tokenizer, tagger, parser, NER and word vectors
nlp = spacy.load("en_core_web_sm")

def is_ambiguous(sent: str):
    doc = nlp(sent.lower())
    matches = matcher(doc)
    return len(matches) > 0

# Load the dataset
with open('data/requirements.json') as file:
    dataset = json.load(file)

# Load the patterns
with open('data/patterns.json') as file:
    patterns = json.load(file)

matcher = Matcher(nlp.vocab)
matcher.add("AMBIGUOUS_PATTERNS", patterns)

# Result dictionary
results = []

# Evaluate the algorithm
correct_predictions = 0
for requirement in dataset:
    prediction = is_ambiguous(requirement['requirement'])
    # Add result
    is_correct = prediction == requirement['is_ambiguous']
    results.append({"requirement": requirement["requirement"], "prediction": prediction,
                    "is_ambiguous": requirement['is_ambiguous'],
                    "is_correct": is_correct})
    if is_correct:
        correct_predictions += 1

# Save results to CSV
with open('results.csv', 'w', newline='') as file:
    writer = DictWriter(file, fieldnames=['requirement', 'prediction', 'is_ambiguous', 'is_correct'])
    writer.writeheader()
    writer.writerows(results)

accuracy = correct_predictions / len(dataset)
print(f"Accuracy: {accuracy:.2f}")


# -*- coding: utf-8 -*-
"""
Created on Sat Nov 30 11:34:57 2019

Using user input, identifies headwords that match other headwords
as variants of a single headword.

Outputs dataframe containing solely entry ids and variant types indicating
relations.

@author: Mark
"""

import pandas as pd
import json
from os import path
from shutil import copyfile
from PandasExcel import add_bom, remove_bom
from ast import literal_eval

in_file  = 'headword_matches_unprocessed.csv'
out_file = 'headword_matches_processed.csv'

def main():
    write_backup(in_file)
    write_backup(out_file)

    global in_df, out_df
    in_df =  pd.read_csv(in_file, header=0)
    out_df = pd.read_csv(out_file, header=0)

    # read col 'matches' as native python types
    in_df.loc[:, 'matches'] = [literal_eval(row) for row in in_df['matches']]

    remove_bom(in_df)
    remove_bom(out_df)

    order_matches()

    print(in_df)
    print(out_df)

    write_result(in_df,     in_file)
    write_result(out_df,    out_file)

# iterates thru matches data
# asks user to decide how to structure entries
# deletes all entries that have been processed from in_df
def order_matches():
    copy = in_df.copy()
    for index, row in copy.iterrows():
        if row['entry_id'] not in list( in_df['entry_id'] ):
            continue
        if not row['matches']:
            continue
        ordered, ids = get_order(row)
        if not ordered:
            # 'esc' was passed as input
            # print dfs as they are
            return
        edit_dfs(ordered, ids)

# prints entry id and headword form of all entries referenced in row
# asks user to specify which form is the main entry, which are the variants
# and what variant types they are
def get_order(row):
    ids = [row['entry_id']] + [m[1] for m in row['matches']]
    headwords = [row['headword']] + [m[0] for m in row['matches']]
    json_dicts = [get_json_dict(i) for i in ids]
    glosses = [get_gloss(j) for j in json_dicts]
    bibs    = [get_bib(j)   for j in json_dicts]
    dates   = [j['date']  for j in json_dicts]
    for i, (eid, hdwrd, gls, bib, date) \
    in enumerate( zip(ids, headwords, glosses, bibs, dates) ):
        print(f"{i}: {hdwrd}- {gls}- {bib}- {date}")
    print("How would you like to organize these entries?")
    print("Please input a string of integers indicating your choice "+\
          "for each entry.")
    print("2 to set as headword, 1 to set as variant, 0 to set as unrelated.")
    print("Type 'esc' to leave the program and save progress.")
    print( ''.join( str(i) for i, _ in enumerate(ids) ) , end='' )
    result = input()
    result = validate_result(result, len(ids))
    if result == -1:
        return None, None
    return result, ids

# given result, a string describing hierarchy of entry ids in list ids
# edit in_df and out_df to this data heirarchy
def edit_dfs(result, ids):
    global in_df, out_df

    # delete main entry if ignored, but keep any instances where it appears as
    # a variant later
    if result[0] == '0':
        del_id(ids[0])
        result = result[1:]
        ids = ids[1:]

    # base case, no headword or variants
    # ignore all entries
    if '2' not in result:
        return

    entry_id = ''
    variants = {}
    # iterate thru rest of result, recording data for new entry_id and variants
    for i, res in zip(ids, result):
        # result val of 1 indicates id is variant of entry_id
        if res == '1':
            # ask user to input variant type
            variants[i] = input_var_type(i)
            del_id(i, True)

        # result val of 2 indicates id is entry_id
        elif res == '2':
            assert not entry_id
            entry_id = i
            del_id(i, True)

        else:
            assert res == '0'

    out_df.loc[ len(out_df) ] = {'entry_id':entry_id, 'variants':variants}

def input_var_type(var):
    print(f"What variant type would you like to give {var}?")
    print("Type '1' for Barbosa, '2' for SIL Dict, '3' for Martins, 4 for Weir.")
    print("Or type a custom field in.", end='')
    var_type = input()
    var_type = var_type.strip()

    if var_type == '1':
        return 'Barbosa 2005'
    elif var_type == '2':
        return 'SIL Dict 2011'
    elif var_type == '3':
        return 'Martins 2005'
    elif var_type == '4':
        return 'Weir'

    else:
        return var_type

# function shouldn't be needed, left as a comment in case that changes
"""def id_in_df(eid):
    match_ids = [m[1] for match in in_df['matches'] for m in match]
    result = eid in match_ids or eid in list(in_df['entry_id'])
    if not result:
        print(in_df['entry_id'])
        print(f'id not found {eid}')
    return result"""

# deletes row containing eid in 'entry_id' col
# if matched, delete eid for any instance it occurs matched to another
# entry (in the 'matches' col)
def del_id(eid, matched=False):
    global in_df
    index =  in_df.index[in_df['entry_id'] == eid].tolist()
    assert len(index) <= 1, index
    in_df = in_df.drop(index)

    if matched:
        for i, row in in_df.iterrows():
            matches = row['matches']
            index = [this_id for this_id, m in enumerate(matches) if m[1] == eid]
            if not index:
                continue
            assert len(index) == 1
            index = index[0]
            new_matches = matches.remove( matches[index] )
            if new_matches:
                in_df.loc[i, 'matches'] = new_matches
            else:
                in_df = in_df.drop(i)

# checks if string result fits acceptance conditions
# prompts user for correction until conditions are met
# returns string
def validate_result(result, size):
    if result == 'esc':
        return -1
    while True:
        try:
            assert len(result) == size
            for char in result:
                assert char in ('0', '1', '2')
            assert result.count('2') <= 1
            # can't have variants if no headword is identified
            if '2' not in result:
                assert '1' not in result
            int(result)
            return result
        except (ValueError, AssertionError):
            print("Please type only ints in the range 0-2 in a string of len",\
                  size)
            result = input()

def get_json_dict(eid):
    filehead = eid
    filehead = rep_all(filehead, '/ ', '_')
    filehead = rep_all(filehead, '()[]? ', '')
    filehead = filehead.replace('_=', '=')
    filename = 'entries/' + filehead + '.json'
    while True:
        try:
            with open(filename, 'r', encoding='utf8') as f:
                lemma = json.load(f)
            return lemma
        except FileNotFoundError:
            print(f'Could not find {filename}. Please enter the correctly'+\
                  ' typed json file below:')
            filename = input()

def get_gloss(lemma):
    for sense in lemma['sense']:
        # return gloss if gloss has nonzero value
        if sense['gloss']:
            # gloss is stored as a dictionary w/ language as keys
            # which language is returned is unimportant, so
            # return a random value
            # somehow it's always portuguese
            # lmão
            return sense['gloss'].popitem()[1]
        # return definition if def has nonzero value
        elif sense['def']:
            return sense['def'].popitem()[1]
    # tough luck, no data on lexeme meaning
    else:
        return None

def get_bib(lemma):
    if lemma['note']:
        for note in lemma['note']:
            if note[1] == 'bibliography':
                return note[0]
    return None

def rep_all(s, chars, tgt):
    for c in chars:
        s = s.replace(c, tgt)
    return s

# checks that backup file does not currently exist
# write backup as copy of main
def write_backup(f):
    f_backup = f.split(sep='.')
    f_backup[0] += 'BACKUP'
    f_backup = '.'.join(f_backup)

    assert not path.exists(f_backup), f"Make sure {f} is satisfactory "\
    +f"and then delete {f_backup}"

    copyfile(f, f_backup)

# check that files are not open
# wait for user input if so, then write df
def write_result(df, file):
    while True:
        try:
            add_bom(df)
            df.to_csv(file, encoding='utf8', index=False)
            break
        except:
            input(f"{file} is currently in use by another program. Please "\
                  +"close it so that the results may be saved.")

if __name__ == '__main__':
    main()


import asyncio
from pyppeteer import launch
async def intercept_network_response(response):
    # In this example, we only care about HTML responses!
    if "text/html" in response.headers.get("content-type", ""):
        # Print some info about the responses
        print("URL:", response.url)
        print("Method:", response.request.method)
        print("Response headers:", response.headers)
        print("Request Headers:", response.request.headers)
        print("Response status:", response.status)
        # Print the content of the response
        print("Content: ", await response.json())
        # NOTE: Use await response.json() if you want to get the JSON directly
async def main():
    browser = await launch()
    page = await browser.newPage()
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36')
    await page.goto('https://www.woolworths.com.au')

    page.on('response', lambda response: asyncio.ensure_future(intercept_network_response(response)))

    await page.goto('https://www.woolworths.com.au/shop/productdetails/339939')
    await browser.close()
asyncio.get_event_loop().run_until_complete(main())


import matplotlib.pyplot as plt
import numpy as np
import umap
import torch
from collections import OrderedDict
from torch import nn

def get_invariant_feats(model_feats,model_da, device, test_loader,use_ssl=1,only_ssl=1,no_da=0):
    'no_da: no domain adaptation'
    model_feats.eval()
    model_da.eval()
    test_loader
    alpha = 1
    pred_array = []
    y_lab_array = []
    params = OrderedDict(model_feats.named_parameters())
    #params = params.to(device)
    pred_array = np.asarray(pred_array)
    y_lab_array = np.asarray(y_lab_array)
    with torch.no_grad():
        for data in test_loader:
            x_src = data['x_src'].to(device).to(device)
            y_src = data['y_src'].to(device).to(device)
            if use_ssl:

                x_src_feats = model_feats(x_src, params)
                #x_src_feats = x_src[:,:,:]
                #reps = model(x,params).detach().cpu().numpy().squeeze(0)

                #x_src_feats = model_feats(z, params)
                #x_src_feats = x_src_feats.reshape(x_src.shape[0],-1,x_src_feats.shape[-1])
            else:
                x_src_feats = x_src
            if no_da:
                params_da =  OrderedDict(model_feats.named_parameters())
                invrt_feats = model_da(x_src_feats,params_da )
            elif only_ssl:
                invrt_feats = x_src_feats
            else:
                invrt_feats = model_da.get_invariant_feats(x_src_feats)

    return invrt_feats,y_src
def visualize_reps(model_feats,model_da,device,test_loader,rob_c_idx,rob_b_idx,use_ssl=1,only_ssl=1,src_type ='B',trgt_type='C',no_da=0):
    reducer = umap.UMAP()
    #reducer = TSNE(n_components=2)
    if only_ssl:
        assert use_ssl == 1, "use ssl should be 1 if only ssl is 1"
    reps,y_src = get_invariant_feats(model_feats = model_feats, model_da=model_da,device=device,test_loader=test_loader,
                                     use_ssl=use_ssl,only_ssl=only_ssl)
    #reps = reps[:,:,0:32]
    if use_ssl == 1:
        ssl_sv_str = 'with_ssl'
    else:
        ssl_sv_str = 'without_ssl'
    if only_ssl == 1:
        only_ssl_str = 'with_only_ssl'
    else:
        only_ssl_str = 'without_ssl'

    y_src = y_src.cpu().numpy()
    rob_lbl = np.zeros(y_src.shape)
    rob_lbl[rob_b_idx,:,:] = 1
    y_src2= np.copy(y_src)
    y_src2[rob_b_idx,:,:] = y_src2[rob_b_idx,:,:] + 10
    reps = reps.cpu().numpy().reshape(-1,reps.shape[-1])
    rob_lbl = rob_lbl.reshape(-1,rob_lbl.shape[-1])
    y_src2 = y_src2.reshape(-1,y_src2.shape[-1])
    y_src = y_src.reshape(-1, y_src.shape[-1])
    transformed = reducer.fit_transform(reps)
    save_path_rob = f'figures/DA/rob_lab_{ssl_sv_str}_{only_ssl_str}_src_{src_type}_trgt_{trgt_type}.pdf'
    save_path_terr = f'figures/DA/rob_ter_{ssl_sv_str}_{only_ssl_str}_src_{src_type}_trgt_{trgt_type}.pdf'
    fig1, axs1 = plt.subplots(1, 3, figsize=(15, 5))
    fig2, axs2 = plt.subplots(1, 3, figsize=(15, 5))
    axs1[0].scatter(transformed[:, 0], transformed[:, 1], c=rob_lbl,alpha=0.1)

    axs1[1].scatter(transformed[(rob_lbl == 0).reshape(-1,), 0], transformed[(rob_lbl == 0).reshape(-1,), 1], c='tab:blue', alpha=0.1,label='Robot c')
    axs1[1].legend()
    axs1[2].scatter(transformed[(rob_lbl == 1).reshape(-1,), 0], transformed[(rob_lbl == 1).reshape(-1,), 1], c='tab:orange', alpha=0.1,label='Robot b')
    axs1[2].legend()
    axs2[0].scatter(transformed[:, 0], transformed[:, 1], c=y_src2,
                    label=['b0', 'b1', 'b2', 'b3', 'b4', 'C0', 'C1', 'C2', 'C3', 'C4'], alpha=0.2)
    clr_list = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:brown']
    label_list = ['Stair Down', 'flat', 'Stair up', 'Slope down', 'Slope Up']
    for j in range(0, 5):
        j_idx = np.where(((rob_lbl == 0) & (y_src == j)))[0]
        axs2[1].scatter(transformed[j_idx, 0], transformed[j_idx, 1], c=clr_list[j], label=label_list[j],alpha=0.2)
    for j in range(0, 5):
        j_idx = np.where(((rob_lbl == 1)  &(y_src == j)))[0]
        axs2[2].scatter(transformed[j_idx, 0], transformed[j_idx, 1], c=clr_list[j], label=label_list[j],alpha=0.2)
    axs2[1].set_title("Robot C")
    axs2[2].set_title("Robot B")
    axs2[0].set_title("Combined Robots")
    axs2[2].legend()
    '''
    for i in range(0,reps.shape[0]):
        temp_reps = reps[i, :, :]
        if (i not in rob_c_idx) and ( i not in rob_b_idx) :
            continue
        else:
            transformed = reducer.fit_transform(temp_reps)
        if i in list(rob_b_idx):
            lbl = 'rob_b'
            clr ='b'
            clr_list = ['tab:blue','tab:orange','tab:green','tab:red','tab:brown']
            label_list = ['B1','B2','B3','B4','B5']
        elif i in list(rob_c_idx):
            lbl = 'rob_c'
            clr = 'r'
            clr_list = ['tab:pink' , 'tab:gray', 'tab:olive','tab:cyan','k']
            label_list = ['C1', 'C2', 'C3', 'C4','C5']
        for j in range(0,5):
            j_idx = np.where(y_src[i,:].reshape(-1,) == j)
            axs[0].scatter(transformed[j_idx,0],transformed[j_idx,1],c= clr_list[j],label = label_list[j])
        for k in range(10,15):
            j_idx = np.where(y_src[i, :].reshape(-1, ) == j)
            axs[0].scatter(transformed[j_idx, 0], transformed[j_idx, 1], c=clr_list[j], label=label_list[j])
        axs[1].scatter(transformed[:,0],transformed[:,1],c = clr,label = lbl)
        '''
    fig1.suptitle(f'Robot Type - Src {src_type} Trgt {trgt_type}')
    fig2.suptitle(f'Terrai Type - Src {src_type} Trgt {trgt_type}')
    fig1.savefig(save_path_rob)
    print(save_path_terr)
    return fig2
    #fig2.savefig(save_path_terr)

def visualize_reps_with_labels(reps,y_unlab, y_src,y_prop=None):
    #reps = reps.cpu().numpy().reshape(-1, reps.shape[-1])
    reducer = umap.UMAP()
    y_src = y_src.reshape(-1, )
    transformed = reducer.fit_transform(reps)
    fig1, axs1 = plt.subplots(1, 3, figsize=(15, 5))

    clr_list = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:brown']
    label_list = ['Stair Down', 'flat', 'Stair up', 'Slope down', 'Slope Up']
    for j in range(0, 5):
        j_idx = np.where( y_src == j)[0]
        axs1[1].scatter(transformed[j_idx, 0], transformed[j_idx, 1], c=clr_list[j], label=label_list[j], alpha=0.2)
    for j in range(0, 5):
        j_idx = np.where((y_prop == j))[0]
        axs1[2].scatter(transformed[j_idx, 0], transformed[j_idx, 1], c=clr_list[j], label=label_list[j], alpha=0.2)

    clr_list = ['k','tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:brown',
            ]
    label_list = ['unlabelled','Stair Down', 'flat', 'Stair up', 'Slope down', 'Slope Up',]

    for j in range(-1, 5):
        j_idx = np.where((y_unlab == j))[0]
        axs1[0].scatter(transformed[j_idx, 0], transformed[j_idx, 1], c=clr_list[j+1], label=label_list[j+1], alpha=0.2)
    plt.savefig(f'figures/DA/rob_lab_propagated_vel_b_01')
def plot_ts_reps(model_feats,model_clfr, device, x,y,title='figure',window=50):
    #fig1, axs1 = plt.subplots(4, 1, figsize=(15, 5))
    params = OrderedDict(model_feats.named_parameters())
    model_feats.to(device)
    x = torch.from_numpy(x)[:,:].to(device).unsqueeze(0)

    if window == -1:
        #src_lbl_clfr = model_clfr(x_src_feats)
        reps = model_feats(x,params) #.detach().cpu().numpy().squeeze(0)
        src_lbl_clfr = model_clfr(reps)
        reps = reps.detach().cpu().numpy().squeeze(0)
        src_lbl_clfr_reshaped = src_lbl_clfr.reshape(-1, src_lbl_clfr.shape[-1])
        probs = torch.nn.functional.softmax(src_lbl_clfr_reshaped, dim=1).detach()
        entropy = torch.distributions.Categorical(probs=probs).entropy().cpu().numpy()
        preds = torch.argmax(probs,dim=1).cpu().numpy()
        #reps = x.cpu().numpy().squeeze(0)
    else:
        z = x.squeeze(0).unfold(0, window, window)
        z = z.swapaxes(1, 2)
        reps = model(z, params).detach().cpu()
        reps = reps.reshape(-1,reps.shape[-1])

    fig, ax = plt.subplots(5, 1, sharex=True)
    ax[0].plot(x[0,:,:].detach().cpu().numpy())
    ax[0].title.set_text("Input data")
    ax[1].plot(reps)
    ax[1].title.set_text("Learned Reps")
    ax[2].plot(entropy)
    ax[2].title.set_text("Entropy (soft max)")
    ax[3].plot(preds)
    ax[3].title.set_text("Predicted class")

    ax[4].plot(y)
    ax[4].title.set_text("True class")
    #plt.show()
    plt.tight_layout()
    fig_path = f'figures/DA/rob_lab_1{title}_{str(window)}.pdf'
    #fig.suptitle(title)
    #plt.show()
    return fig
    #fig.savefig(fig_path)


from selenium import webdriver
import unittest
import HtmlTestRunner

class GoogleSearch(unittest.TestCase):

	@classmethod
	def setUpClass(self):
		self.driver = webdriver.Chrome(executable_path='../../ChromeDriver/chromedriver.exe')
		self.driver.implicitly_wait(10)
		self.driver.maximize_window()

	def test_search_automation(self):
		self.driver.get('https://google.com')
		self.driver.find_element_by_name('q').send_keys('Automation Testing')
		self.driver.find_element_by_name('btnK').click()
		assert 'Automation Testing' in self.driver.page_source

	def test_search_python_for_new_bie(self):
		self.driver.get('https://google.com')
		self.driver.find_element_by_name('q').send_keys('Python for beginner')
		self.driver.find_element_by_name('btnK').click()
		assert 'Python for beginner' in self.driver.page_source

	@classmethod
	def tearDownClass(self):
		self.driver.close()
		self.driver.quit()
		print("Test Completed")

if __name__ == '__main__':
	unittest.main(testRunner=HtmlTestRunner.HTMLTestRunner(output='C:/Phu/Python/Selenium/Report'))


from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import scipy.cluster.hierarchy as sch
from sklearn.cluster import AgglomerativeClustering

def k_means_clustering(X):
    print ("Using the elbow method to perform K means clustering..")
    wcss = []
    for i in range(1,11):
        kmeans = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=0)
        kmeans.fit(X)
        wcss.append(kmeans.inertia_)
    max_ratio = 0
    optimum_clusters = 0
    for i in range(1,9):
        if (wcss[i] - wcss[i - 1]) / (wcss[i + 1] - wcss[i - 1]) > max_ratio:
            max_ratio = (wcss[i] - wcss[i - 1]) / (wcss[i + 1] - wcss[i - 1])
            optimum_clusters = i + 1
    kmeans = KMeans(n_clusters=optimum_clusters, init='k-means++', max_iter=300, n_init=10, random_state=0)
    y_kmeans = kmeans.fit_predict(X)
    return y_kmeans

def hierarchical_clustering(X):
    print('Plotting Dendrogram,.')
    dendrogram = sch.dendrogram(sch.linkage(X, method='ward'))
    plt.title('Dendrogram')
    plt.ylabel('Euclidean Distances')
    plt.show()
    optimum_clusters = input('Input the optimum cluster...')
    print('Performing Hierarchical Clustering..')
    hc = AgglomerativeClustering(n_clusters=optimum_clusters, affinity='euclidean', linkage='ward')
    y_hc = hc.fit_predict(X)
    return y_hc


import io
import zipfile
import pandas as pd

def extract_zip(content):
    with zipfile.ZipFile(io.BytesIO(content)) as thezip:
        for zipinfo in thezip.infolist():
            with thezip.open(zipinfo) as thefile:
                yield zipinfo.filename, thefile

def extract_features_from_bureau(bureau_df, bureau_balances_df):
    bureau_df['AMT_CREDIT_SUM_DEBT'] = bureau_df['AMT_CREDIT_SUM_DEBT'].fillna(value=0)
    bureau_df['AMT_CREDIT_DEBT_RATIO'] =  bureau_df['AMT_CREDIT_SUM_DEBT']/data['bureau']['AMT_CREDIT_SUM']
    bureau_df['AMT_CREDIT_DEBT_RATIO'] = bureau_df['AMT_CREDIT_DEBT_RATIO'].fillna(value=0)

    bureau_df = bureau_df.set_index('SK_ID_BUREAU')

    DPD_STATUS_MAP = {
        'C': 0,
        'X': 0,
        '0': 0,
        '1': 1,
        '2': 2,
        '3': 3,
        '4': 4,
        '5': 5,
    }
    def sum_of_dpd(x):
        return x['STATUS'].map(DPD_STATUS_MAP).sum()

    dpd_counts_df = bureau_balances_df.groupby(['SK_ID_BUREAU']).apply(sum_of_dpd)

    dpd_counts_df = pd.DataFrame(dpd_counts_df, columns=['DPD_COUNTS']);
    bureau_with_dpds = pd.concat([bureau_df, dpd_counts_df], axis=1)

    bureau_with_dpds = bureau_with_dpds.set_index('SK_ID_CURR')

    return bureau_with_dpds[:][['AMT_CREDIT_DEBT_RATIO', 'CREDIT_DAY_OVERDUE', 'DPD_COUNTS']]


import numpy as np
import pandas as pd
import os
import re
from sklearn import model_selection

def create_folds(data, num_splits):
    # we create a new column called kfold and fill it with -1
    data["kfold"] = -1

    num_bins = int(np.floor(1 + np.log2(len(data))))

    # bin targets
    data.loc[:, "bins"] = pd.cut(
        data["score"], bins=num_bins, labels=False
    )

    # the next step is to randomize the rows of the data
    data = data.sample(frac=1).reset_index(drop=True)

    # initiate the kfold class from model_selection module
    kf = model_selection.StratifiedGroupKFold(n_splits=num_splits, shuffle=True, random_state=0xFACED)

    # fill the new kfold column
    # note that, instead of targets, we use bins!
    for f, (t_, v_) in enumerate(kf.split(X=data, y=data['bins'].values, groups=data['anchor'].values)):
        data.loc[v_, 'kfold'] = f

    data = data.drop("bins", axis=1)
    # return dataframe with folds
    return data

def get_cpc_texts():
    contexts = []
    pattern = '[A-Z]\d+'
    for file_name in os.listdir('/home/toefl/K/ptp/dataset/context/CPCSchemeXML202105'):
        result = re.findall(pattern, file_name)
        if result:
            contexts.append(result)
    contexts = sorted(set(sum(contexts, [])))
    results = {}
    for cpc in ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'Y']:
        with open(f'/home/toefl/K/ptp/dataset/context/CPCTitleList202202/cpc-section-{cpc}_20220201.txt') as f:
            s = f.read()
        pattern = f'{cpc}\t\t.+'
        result = re.findall(pattern, s)
        cpc_result = result[0].lstrip(pattern)
        for context in [c for c in contexts if c[0] == cpc]:
            pattern = f'{context}\t\t.+'
            result = re.findall(pattern, s)
            results[context] = cpc_result + ". " + result[0].lstrip(pattern)
    return results

cpc_texts = get_cpc_texts()

# read training data
df = pd.read_csv("/home/toefl/K/ptp/dataset/train.csv")
df['context_text'] = df['context'].map(cpc_texts)

# create folds
df = create_folds(df, num_splits=5)

print(df.kfold.value_counts())
print(df.head())

df.to_csv("train_folds.csv", index=False)


import torch
from torch import optim
import time
import numpy as np

from quantumdraw.sampler.metropolis import Metropolis
from quantumdraw.solver.neural_solver import NeuralSolver
from quantumdraw.solver.user_solver import UserSolver
from quantumdraw.wavefunction.neural_wave_function import NeuralWaveFunction
from quantumdraw.wavefunction.user_wave_function import UserWaveFunction

domain = {'xmin': -5., 'xmax': 5.}

def get_user_score(user_guess, current_pot):
    uwf = UserWaveFunction(current_pot, domain, xpts=list(map(lambda g: g[0], user_guess)), ypts=list(map(lambda g: g[1], user_guess)))

    # sampler
    sampler = Metropolis(nwalkers=100, nstep=100,
                         step_size=0.5, domain=domain)

    usolver = UserSolver(wf=uwf, sampler=sampler)
    data = usolver.feedback()
    points = list(zip(data['x'],data['y']))

    return points, usolver.get_score()

def get_solution(current_pot):
    uwf = UserWaveFunction(current_pot, domain, xpts=[], ypts=[])

    # sampler
    sampler = Metropolis(nwalkers=100, nstep=100,
                         step_size=0.5, domain=domain)

    usolver = UserSolver(wf=uwf, sampler=sampler)
    data = usolver.get_solution()
    data['y'] /= np.max(data['y'])
    points = list(zip(data['x'].tolist(),data['y'].tolist()))

    return points

def get_ai_score(current_pot, max_iterations=100, duration=30):
    sampler = Metropolis(nwalkers=500, nstep=2000,
                         step_size=0.5, domain=domain)

    # wavefunction
    wf = NeuralWaveFunction(current_pot, domain, 11, fcinit='random', sigma=0.5)

    # optimizer
    opt = optim.Adam(wf.parameters(), lr=0.05)

    # scheduler
    scheduler = optim.lr_scheduler.StepLR(opt, step_size=100, gamma=0.75)

    # define solver
    solver = NeuralSolver(wf=wf, sampler=sampler, optimizer=opt, scheduler=scheduler)
    # solver = NeuralSolver(wf=wf, sampler=sampler, optimizer=None, scheduler=None)

    pos = solver.run(1)
    solver.sampler.nstep = solver.resample.resample

    end_time = time.time() + duration
    while time.time() < end_time:
        pos = solver.run(1, pos=pos, with_tqdm=False)
        num_samples = 50
        low_x = -5
        high_x = 5
        x_points = [low_x + sample_num * (high_x - low_x) / num_samples for sample_num in range(0, num_samples)]
        x = torch.tensor(x_points)
        y = solver.wf(x)
        y /= y.max()
        y_points = y.detach().numpy().T[0].tolist()
        points = list(zip(x_points, y_points))
        yield points, solver.get_score()


from guacho_utils import *
import matplotlib as mpl
from mpl_toolkits.axes_grid1 import ImageGrid
from matplotlib.colors import LogNorm
from matplotlib import rc
import matplotlib.pyplot as plt
#import yt
plt.rc('font', family='serif')
plt.rc('font',**{'family':'serif','serif':['{Palatino}']})

rc('text', usetex=True)

mpl.rc('xtick',labelsize=10)
mpl.rc('ytick',labelsize=10)
mpl.rc('text',usetex=True)

firstrun= True
plt.ion()

path='/datos_diable/matias/Guacho_Working/Guacho-master/A1-5GP-GC/BIN/'
nout=45
lx=11.472
ly=5.7359
extent=(-lx,lx,-lx,lx)

circ1=plt.Circle((0.,0.),radius=1.,color='k',fill=False,ls='dashed')
#circ2=plt.Circle((0.,0.),radius=0.035,color='k',fill=False,ls='dashed')

rhosc = get_scalings(nout=0, path=path, verbose=True )[2]
tempsc = get_scalings(nout=0, path=path, verbose=True )[2]
nxtot, nytot, nztot = get_boxsize(nout=0, path=path, verbose=True )
rho = readbin3d_all(nout=nout,neq=0,path=path,verbose=False,mhd=True)
vx = readbin3d_all(nout=nout,neq=1,path=path,verbose=False,mhd=True)
vy = readbin3d_all(nout=nout,neq=2,path=path,verbose=False,mhd=True)
vz = readbin3d_all(nout=nout,neq=3,path=path,verbose=False,mhd=True)
pgas = readbin3d_all(nout=nout,neq=4,path=path,verbose=False,mhd=True)
bx = readbin3d_all(nout=nout,neq=5,path=path,verbose=False,mhd=True)
by = readbin3d_all(nout=nout,neq=6,path=path,verbose=False,mhd=True)
bz = readbin3d_all(nout=nout,neq=7,path=path,verbose=False,mhd=True)
rho_n = readbin3d_all(nout=nout,neq=8,path=path,verbose=False,mhd=True)*rhosc

v=np.sqrt(vx*vx+vy*vy+vz*vz)
b=np.sqrt(bx*bx+by*by+bz*bz)
t=pgas*1./(8.3145e7*(2*rho-rho_n))
yhp=1.-rho_n/rho
#Phi=Phi+1e-30
#Psi= (Phi*rho_n)*2.1789e-11

#------------------------------------------------------
fig= plt.figure(1)
fig.clf()

im = plt.imshow(rho[:,nytot/2,:], norm=LogNorm(), origin='lower', cmap='Blues',
           extent=extent )

cont= plt.contour(yhp[:,nytot/2,:], levels=[.9, .99, .999], colors='k',extent=extent )

plt.xlabel(r'$x~[\mathrm{R_*}]$')
plt.ylabel(r'$z~[\mathrm{R_*}]$')
plt.title(r'Density')
#plt.text(-0.13, 0.13,'A1 @ t='+str(nout*0.025).format('f')+' days')

ax= fig.add_subplot(1,1,1)
ax.add_patch(circ1)
#ax.add_patch(circ2)

cb= plt.colorbar(im,label='', pad=0.01, extend='max')
cb.set_label(r'$\rho_\mathrm{H}~[\mathrm{g cm^{-3}}]$')

plt.savefig('./PDF/A1b-f1.pdf',dpi=300,transparent=True,bbox_inches='tight')

#-------------------------------------------------------
fig= plt.figure(2)
fig.clf()

im = plt.imshow(t[:,nytot/2,:], origin='lower', cmap='gist_heat',
     extent=extent, vmin= 1e4, vmax = 2e6, norm= LogNorm())

plt.xlabel(r'$x~[\mathrm{R_*}]$')
plt.ylabel(r'$z~[\mathrm{R_*}]$')

plt.title(r'Temperature')

cb= plt.colorbar(im,label='', pad=0.01, extend='max')
cb.set_label(r'$T~[\mathrm{K}]$')
#plt.text(-0.13, 0.13,'P4c @ t='+str(nout*0.025).format('f')+' days')

plt.savefig('./PDF/A1b-f2.pdf',dpi=300,transparent=True,bbox_inches='tight')
#-------------------------------------------------------
fig= plt.figure(3)
fig.clf()

im = plt.imshow(b[:,nytot/2,:], origin='lower', cmap='Spectral', #bds_highcontrast',
     extent=extent, norm= LogNorm())#, vmin=1e-18 , vmax= 1e-12 )
plt.xlabel(r'$x~[\mathrm{R_*}]$')
plt.ylabel(r'$z~[\mathrm{R_*}]$')
plt.title(r'Magnetic field')
#plt.text(-0.13, 0.13,'P4c @ t='+str(nout*0.025).format('f')+' days')

cb= plt.colorbar(im,label='', pad=0.01, extend='max')
#cb.set_label(r'$\psi~[\mathrm{erg~cm^{-3}~s^{-1}}]$')
cb.set_label(r'$B~[G]$')

plt.savefig('./PDF/A1b-f3.pdf',dpi=300,transparent=True,bbox_inches='tight')

#--------------------------------------------------------------------------
fig= plt.figure(4)
fig.clf()

im = plt.imshow(v[:,nytot/2,:]/1e5,origin='lower', cmap='rainbow', extent=extent )

plt.xlabel(r'$x~[\mathrm{R_*}]$')
plt.ylabel(r'$z~[\mathrm{R_*}]$')
plt.title(r'Velocity')
#plt.text(-0.13, 0.13,'P4c @ t='+str(nout*0.025).format('f')+' days')

cb= plt.colorbar(im,label='', pad=0.01, extend='max')
cb.set_label(r'$v~[\mathrm{km s^{-1}}]$')

plt.savefig('./PDF/A1b-f4.pdf',dpi=300,transparent=True,bbox_inches='tight')

'''
    data = dict(density = (denT, "g/cm**3"), temperature=(temp, "K"),ionization_fraction= (yhp, "K") )
    bbox = np.array([[-.15, .15], [-.0375, .0375] ,[-.15, .15]])
    ds = yt.load_uniform_grid(data, denT.shape, length_unit="au", bbox=bbox, nprocs=1)

  slc = yt.SlicePlot(ds, "y", ["density"], width=((.3,'au'),(0.3,'au')) )

  slc.set_cmap("density", "Blues_r")
  slc.set_cmap("temperature", "gist_heat")
  slc.set_xlabel('x[AU]')
  slc.set_ylabel('z[AU]')
  slc.set_colorbar_label("density","$n_\mathrm{H}~[\mathrm{cm^{-3}}]$")

  slc.set_zlim("density",zmax=1e6,zmin=1e2)

  slc.annotate_sphere([0.,0.,0.,],(0.00511, 'au'),circle_args={'ls': 'solid'})
  slc.annotate_sphere([0.,0.,0.,],(0.035, 'au'),circle_args={'ls': 'dashed'})
  slc.annotate_contour('ionization_fraction',ncont=1,clim=(0.5,0.5))
  slc.annotate_contour('ionization_fraction',ncont=1,clim=(0.99,0.99))
  slc.annotate_contour('ionization_fraction',ncont=1,clim=(0.999,0.999),plot_args={"colors": 'black',"ls":'.'})

  slc.save('dens.png')

'''


from pandas import read_csv, get_dummies, DataFrame
df = read_csv("dataset/train.csv")
val_df = read_csv("dataset/test.csv")

features = df.columns.drop(["Id","SalePrice"])
target = ["SalePrice"]

X, y = df[features], df[target]
X= X.fillna(X.mean())
X = get_dummies(X)

vX= val_df[features]
vX= vX.fillna(vX.mean())
vX = get_dummies(vX)

X, vX = X.align(vX, join='inner', axis=1)

from sklearn.model_selection import train_test_split
train_X, test_X, train_y, test_y= train_test_split(X, y, test_size = 0.1)

from sklearn.ensemble import RandomForestRegressor
model = RandomForestRegressor()
model.fit(train_X,train_y)

pred_y = model.predict(test_X)
from sklearn.metrics import mean_squared_error
import numpy as np
rmse = mean_squared_error(np.log(test_y), np.log(pred_y))**0.5

val_y = model.predict(vX)
result = DataFrame({"Id" : list(range(1461,1461+len(val_y))), "SalePrice" : val_y})
result.set_index("Id",inplace=True)
result.to_csv("v0.csv")


import logging
from logging.handlers import RotatingFileHandler
import datetime

#Logger
log = logging.getLogger('root')
log.setLevel(logging.DEBUG)

fileHandler = RotatingFileHandler('GlitchFilterLog.log', maxBytes=100000)
fileHandler.setLevel(logging.INFO)

log.addHandler(fileHandler)

log.info('******************** {} - Session ********************'.format(datetime.datetime.now()))

formatter = logging.Formatter('%(asctime)s) - %(levelname)s: %(message)s')
fileHandler.setFormatter(formatter)


import keras
import numpy as np
import tensorflow as tf

def get_dataset(batch_size: int, train_samples: int):

    (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
    x_train = np.reshape(x_train, (-1, 784))
    x_test = np.reshape(x_test, (-1, 784))

    # Reserve 10,000 samples for validation.
    x_val = x_train[-train_samples:]
    y_val = y_train[-train_samples:]
    x_train = x_train[:-train_samples]
    y_train = y_train[:-train_samples]

    # Prepare the training dataset.
    train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
    train_dataset = train_dataset.shuffle(buffer_size=1024).batch(batch_size)

    # Prepare the validation dataset.
    val_dataset = tf.data.Dataset.from_tensor_slices((x_val, y_val))
    val_dataset = val_dataset.batch(batch_size)

    return train_dataset, val_dataset


from typing import Any, List, Optional
from lib.parsers.data_parsers.data_parser import DataParser
from lib.parsers.id_parsers.id_parser import IdParser
from types_.category_id import CategoryId
from lib.db import update_supplier

class ParsingController:
    def __init__(self, id_parsers: List[IdParser]=None, data_parsers: List[DataParser]=None):
        self.fsm: Any = None
        self.cat_id: str = ''
        self.id_parsers = id_parsers or []
        self.data_parsers = data_parsers or []
        self.inns = []
        self.suppliers = {}

    def set_fsm(self, fsm):
        self.fsm = fsm

    def set_cat_id(self, cat_id: CategoryId):
        self.cat_id = cat_id

    def trigger(self, event: str):
        if (self.fsm):
            self.fsm.trigger(event)

    async def parse_inns(self):
        self.inns = []
        self.suppliers = {}
        for id_parser in self.id_parsers:
            for inn in await id_parser.parse(self.cat_id):
                self.inns.append(inn)
        self.trigger('fetching_ids_finish')
        return self.inns

    async def parse_supplier(self, inn):
        result = {}
        for data_parser in self.data_parsers:
            data = await data_parser.parse(inn)
            if data_parser.key_name is None:
                for (k, v) in data.items():
                    result[k] = v
            else:
                result[data_parser.key_name] = data
        self.suppliers[inn] = result
        return result

    async def parse_suppliers(self):
        for inn in self.inns:
            print('fetching supplier', inn)
            await self.parse_supplier(inn)
        self.trigger('fetching_data_finish')

    async def save_data(self):
        for inn in self.inns:
            supplier = self.suppliers[inn]
            await update_supplier(inn, supplier)
        self.trigger('uploading_data_finish')


import logging

from ibm_cloud_sdk_core import ApiException

from ..base_client import BaseClient
from ..consts import VPC_DEFAULT_PAGINATION_LIMIT
from ..exceptions import IBMInvalidRequestError

LOGGER = logging.getLogger(__name__)

class ImagesClient(BaseClient):
    """
    Client for images related APIs
    """

    def __init__(self, cloud_id, region):
        super(ImagesClient, self).__init__(cloud_id=cloud_id, region=region)

    def list_images(self, visibility=None, name=None, resource_group_id=None,
                    limit=VPC_DEFAULT_PAGINATION_LIMIT):
        """
        :param visibility:
        :param name:
        :param resource_group_id:
        :param limit: Number of Resources Per Page
        :return:
        """
        if visibility and visibility not in ["private", "public"]:
            raise IBMInvalidRequestError("Parameter 'visibility' should be one of ['private', 'public']")

        params = {
            "visibility": visibility,
            "name": name,
            "resource_group_id": resource_group_id,
            "limit": limit
        }

        response = self.service.list_images(**params).get_result()
        return response.get("images", [])

    def create_image(self, image_json):
        """
        :param image_json:
        :return:
        """
        if not isinstance(image_json, dict):
            raise IBMInvalidRequestError("Parameter 'image_json' should be a dictionary")

        # TODO: Create schema for input and validate it

        return self.service.create_image(image_prototype=image_json).get_result()

    def delete_image(self, image_id):
        """
        :param image_id:
        :return:
        """
        return self.service.delete_image(id=image_id)

    def get_image(self, image_id):
        """
        :param image_id:
        :return:
        """
        return self.service.get_image(id=image_id).get_result()

    def update_image(self, image_id, image_json):
        """
        :param image_id:
        :param image_json:
        :return:
        """
        response = {}
        try:
            response = self.service.update_image(id=image_id, image_patch=image_json).get_result()
        except ApiException as e:
            LOGGER.info("Update Image failed with status code " + str(e.code) + ": " + e.message)

        return response

    def list_operating_systems(self, limit=VPC_DEFAULT_PAGINATION_LIMIT):
        """
        :param limit: Number of Resources Per Page
        :return:
        """
        params = {
            "limit": limit
        }

        response = {}
        try:
            response = self.service.list_operating_systems(**params).get_result()
        except ApiException as e:
            LOGGER.info("List Operating Systems failed with status code " + str(e.code) + ": " + e.message)

        return response.get("operating_systems", [])

    def get_operating_system(self, operating_system_name):
        """
        :param operating_system_name:
        :return:
        """

        response = {}
        try:
            response = self.service.get_operating_system(name=operating_system_name).get_result()
        except ApiException as e:
            LOGGER.info("Get Operating System failed with status code " + str(e.code) + ": " + e.message)

        return response


from typer.testing import CliRunner

from .fixture_cli import env_gitlab_token
from gitlabtree.main import app

runner = CliRunner()

def test_help() -> None:
    result = runner.invoke(app, ["pipeline", "--help"])
    assert result.exit_code == 0
    assert "pipeline" in result.stdout
    assert "last pipeline run" in result.stdout

def test_no_command() -> None:
    result = runner.invoke(app, ["pipeline"])
    assert result.exit_code == 2
    assert "Missing argument 'START'" in result.output