r"""Code example for a custom model, using PyTorch.

This demo shows how to use a custom model with LIT, in just a few lines of code.
We'll use a transformers model, with a minimal amount of code to implement the
LIT API. Compared to models/glue_models.py, this has fewer features, but the
code is more readable.

This demo is equivalent in functionality to simple_tf2_demo.py, but uses PyTorch
instead of TensorFlow 2. The models behave identically as far as LIT is
concerned, and the implementation is quite similar - to see changes, run:
  git diff --no-index simple_tf2_demo.py simple_pytorch_demo.py

The transformers library can load weights from either,
so you can use any saved model compatible with the underlying model class
(AutoModelForSequenceClassification). To train something for this demo, you can:
- Use quickstart_sst_demo.py, and set --model_path to somewhere durable
- Or: Use tools/glue_trainer.py
- Or: Use any fine-tuning code that works with transformers, such as
https://github.com/huggingface/transformers#quick-tour-of-the-fine-tuningusage-scripts

To run locally:
  python -m lit_nlp.examples.simple_pytorch_demo \
      --port=5432 --model_path=/path/to/saved/model

Then navigate to localhost:5432 to access the demo UI.

NOTE: this demo still uses TensorFlow Datasets (which depends on TensorFlow) to
load the data. However, the output of glue.SST2Data is just NumPy arrays and
plain Python data, and you can easily replace this with a different library or
directly loading from CSV.
"""

from collections.abc import Sequence
import sys
from typing import Optional

from absl import app
from absl import flags
from absl import logging

from lit_nlp import dev_server
from lit_nlp import server_flags
from lit_nlp.api import model as lit_model
from lit_nlp.api import types as lit_types
# Use the regular GLUE data loaders, because these are very simple already.
from lit_nlp.examples.datasets import glue
from lit_nlp.lib import file_cache
from lit_nlp.lib import utils

import torch
import transformers

# NOTE: additional flags defined in server_flags.py

FLAGS = flags.FLAGS

FLAGS.set_default("development_demo", True)

_MODEL_PATH = flags.DEFINE_string(
    "model_path",
    "https://storage.googleapis.com/what-if-tool-resources/lit-models/sst2_tiny.tar.gz",
    "Path to trained model, in standard transformers format, e.g. as "
    "saved by model.save_pretrained() and tokenizer.save_pretrained()"
)

SequenceClassifierOutput = (
    transformers.modeling_outputs.SequenceClassifierOutput
)

def _from_pretrained(cls, *args, **kw):
  """Load a transformers model in PyTorch, with fallback to TF2/Keras weights."""
  try:
    return cls.from_pretrained(*args, **kw)
  except OSError as e:
    logging.warning("Caught OSError loading model: %s", e)
    logging.warning(
        "Re-trying to convert from TensorFlow checkpoint (from_tf=True)")
    return cls.from_pretrained(*args, from_tf=True, **kw)

class SimpleSentimentModel(lit_model.Model):
  """Simple sentiment analysis model."""

  LABELS = ["0", "1"]  # negative, positive

  def __init__(self, model_name_or_path):
    self.tokenizer = transformers.AutoTokenizer.from_pretrained(
        model_name_or_path)
    model_config = transformers.AutoConfig.from_pretrained(
        model_name_or_path,
        num_labels=2,
        output_hidden_states=True,
        output_attentions=True,
    )
    # This is a just a regular PyTorch model.
    self.model = _from_pretrained(
        transformers.AutoModelForSequenceClassification,
        model_name_or_path,
        config=model_config)
    self.model.eval()

  ##
  # LIT API implementation
  def max_minibatch_size(self):
    # This tells lit_model.Model.predict() how to batch inputs to
    # predict_minibatch().
    # Alternately, you can just override predict() and handle batching yourself.
    return 32

  def predict_minibatch(self, inputs):
    # Preprocess to ids and masks, and make the input batch.
    encoded_input = self.tokenizer.batch_encode_plus(
        [ex["sentence"] for ex in inputs],
        return_tensors="pt",
        add_special_tokens=True,
        max_length=128,
        padding="longest",
        truncation="longest_first")

    # Check and send to cuda (GPU) if available
    if torch.cuda.is_available():
      self.model.cuda()
      for tensor in encoded_input:
        encoded_input[tensor] = encoded_input[tensor].cuda()
    # Run a forward pass.
    with torch.no_grad():  # remove this if you need gradients.
      out: SequenceClassifierOutput = self.model(**encoded_input)

    # Post-process outputs.
    batched_outputs = {
        "probas": torch.nn.functional.softmax(out.logits, dim=-1),
        "input_ids": encoded_input["input_ids"],
        "ntok": torch.sum(encoded_input["attention_mask"], dim=1),
        "cls_emb": out.hidden_states[-1][:, 0],  # last layer, first token
    }
    # Return as NumPy for further processing.
    detached_outputs = {k: v.cpu().numpy() for k, v in batched_outputs.items()}
    # Unbatch outputs so we get one record per input example.
    for output in utils.unbatch_preds(detached_outputs):
      ntok = output.pop("ntok")
      output["tokens"] = self.tokenizer.convert_ids_to_tokens(
          output.pop("input_ids")[1:ntok - 1])
      yield output

  def input_spec(self) -> lit_types.Spec:
    return {
        "sentence": lit_types.TextSegment(),
        "label": lit_types.CategoryLabel(vocab=self.LABELS, required=False)
    }

  def output_spec(self) -> lit_types.Spec:
    return {
        "tokens": lit_types.Tokens(),
        "probas": lit_types.MulticlassPreds(parent="label", vocab=self.LABELS,
                                            null_idx=0),
        "cls_emb": lit_types.Embeddings()
    }

def get_wsgi_app() -> Optional[dev_server.LitServerType]:
  """Returns a LitApp instance for consumption by gunicorn."""
  FLAGS.set_default("server_type", "external")
  FLAGS.set_default("demo_mode", True)
  # Parse flags without calling app.run(main), to avoid conflict with
  # gunicorn command line flags.
  unused = flags.FLAGS(sys.argv, known_only=True)
  if unused:
    logging.info(
        "simple_pytorch_demo:get_wsgi_app() called with unused "
        "args: %s", unused)
  return main([])

def main(argv: Sequence[str]) -> Optional[dev_server.LitServerType]:
  if len(argv) > 1:
    raise app.UsageError("Too many command-line arguments.")

  # Normally path is a directory; if it's an archive file, download and
  # extract to the transformers cache.
  model_path = _MODEL_PATH.value
  if model_path.endswith(".tar.gz"):
    model_path = file_cache.cached_path(
        model_path, extract_compressed_file=True)

  # Load the model we defined above.
  models = {"sst": SimpleSentimentModel(model_path)}
  # Load SST-2 validation set from TFDS.
  datasets = {"sst_dev": glue.SST2Data("validation")}

  # Start the LIT server. See server_flags.py for server options.
  lit_demo = dev_server.Server(models, datasets, **server_flags.get_flags())
  return lit_demo.serve()

if __name__ == "__main__":
  app.run(main)


import os
import warnings
import numpy as np
import pandas as pd

from typing import List, Dict, Union

from tqdm import tqdm
from summarizer import Summarizer
from sklearn.decomposition import PCA
from jinja2 import Environment, FileSystemLoader
from transformers import AutoModel, AutoTokenizer, AutoConfig

from covid_encoding import encode_texts, train_doc2vec_model
from covid_preprocessing import clean_text
from covid_clustering import get_cluster_labels
from covid_keywords import extract_common_phrases
from covid_visualisation import noninteractive_visualisation

INPUT_FILE = "../datasets/covid_articles/covid19_articles.csv"

class ArticleClustering():
    def __init__(self, clustering_method: str = "hdbscan", encoding_method: str = "roberta",
                 keyword_extraction_method: str = "lda", num_keywords: int = 20,
                 verbose: bool = True, doc2vec_model: Union[None, str] = None, use_summary: bool = False,
                 report_filename: str = "report.html"):
        self.clustering_method = clustering_method
        self.encoding_method = encoding_method
        self.keyword_extraction_method = keyword_extraction_method
        self.num_keywords = num_keywords
        self.verbose = verbose
        self.doc2vec_model = doc2vec_model
        self.use_summary = use_summary
        self.report_filename = report_filename

    def analyze(self, input_file: str, n_samples: Union[int, None] = None) -> None:
        self.n_samples = n_samples
        if self.encoding_method == "doc2vec" and not self.doc2vec_model:
            print("No doc2vec model detected")
            exit(1)
        article_data = self.open_csv_file(input_file, n_samples=n_samples, verbose=self.verbose)
        texts = list(article_data["content"].to_numpy())
        if self.use_summary:
            texts = self.get_summary(texts)

        texts = clean_text(texts, verbose=self.verbose)
        article_data["filtered_content"] = texts
        embeddings = encode_texts(texts, self.encoding_method, verbose=self.verbose)
        embeddings = self.dimensionality_reduction(embeddings, 300, verbose=self.verbose)

        cluster_labels = get_cluster_labels(embeddings, method=self.clustering_method)
        article_data["cluster_label"] = cluster_labels

        noninteractive_visualisation(embeddings, cluster_labels, verbose=self.verbose)

        if self.verbose:
            print("Clustering finished\nKeyword extraction started")
            with warnings.catch_warnings():
                warnings.filterwarnings(action="ignore", category=FutureWarning)
                tqdm.pandas()
            article_data_clustered = article_data.groupby("cluster_label").progress_apply(self.dataframe_regroup)
        else:
            article_data_clustered = article_data.groupby("cluster_label").apply(self.dataframe_regroup)
        article_data_clustered = article_data_clustered.sort_values(by=["cluster_size"], ascending=False)
        self.generate_html_report(article_data_clustered)

    @staticmethod
    def open_csv_file(path: str, *, n_samples: Union[None, int] = None, verbose: bool = False) -> pd.DataFrame:
        dataframe = pd.read_csv(path)
        if n_samples:
            dataframe = dataframe.sample(n=n_samples)
        if verbose:
            if n_samples:
                print(f"Chosen {n_samples} random samples from articles dataset")
            else:
                print(f"Articles dataset opened. Length: {dataframe.shape[0]}")
        return dataframe

    def train_doc2vec_model(self, model_name: str, dataset_path: str) -> None:
        train_doc2vec_model(dataset_path, self.verbose, model_name)
        self.doc2vec_model = model_name

    def get_summary(self, texts: List[str]) -> List[str]:
        config = AutoConfig.from_pretrained('allenai/biomed_roberta_base')
        config.output_hidden_states = True
        tokenizer = AutoTokenizer.from_pretrained('allenai/biomed_roberta_base')
        model = AutoModel.from_pretrained('allenai/biomed_roberta_base', config=config)
        summarizer = Summarizer(custom_model=model, custom_tokenizer=tokenizer)
        if self.verbose:
            print("Summary extraction started:")
            texts = tqdm(texts)
        result = [summarizer(text) for text in texts]
        return result

    def dimensionality_reduction(self, embeddings: np.ndarray, max_pca_components: int = 300, verbose: bool = False) -> np.ndarray:
        pca = PCA(n_components=(min(self.n_samples or max_pca_components, max_pca_components)))
        result = pca.fit_transform(embeddings)
        if verbose and len(embeddings[0]) != result.shape[1]:
            print(f"Dimensionality reduction: {len(embeddings)}x{len(embeddings[0])} -> {result.shape[0]}x{result.shape[1]}")
        return result

    def dataframe_regroup(self, dataframe: pd.DataFrame) -> Dict:
        titles = dataframe["title"].to_numpy()
        size = dataframe.shape[0]
        texts = dataframe["filtered_content"].to_list()
        common_phrases = extract_common_phrases(texts, self.keyword_extraction_method, n_keywords=self.num_keywords)

        return pd.Series({'titles': titles,
                          'cluster_size': size,
                          'common_phrases': common_phrases})

    def generate_html_report(self, dataframe: pd.DataFrame) -> None:
        env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__))))
        template = env.get_template("template.html")
        original_list = dataframe.to_dict('records')
        new_list = [{key: values for key, values in record.items()
                    if key in ['cluster_size', 'titles', 'common_phrases']}
                    for record in original_list]
        template_vars = {"values": new_list}
        html_out = template.render(template_vars)
        with open(self.report_filename, "w", encoding="utf-8") as f:
            f.write(html_out)

def main():
    pipeline = ArticleClustering(encoding_method="doc2vec",
                                 doc2vec_model="doc2vec.model",
                                 keyword_extraction_method="rake")
    pipeline.analyze(INPUT_FILE, 1000)

if __name__ == "__main__":
    main()


import tensorflow as tf
import copy
import numpy as np
import pointer_net
from flask import request, Response
from flask import Flask
from gevent.pywsgi import WSGIServer
import time
import os
import random
import sys
import json
import glob
from copy import deepcopy

tf.app.flags.DEFINE_integer("batch_size", 1,"Batch size.")
tf.app.flags.DEFINE_integer("max_input_sequence_len", 1000, "Maximum input sequence length.")
tf.app.flags.DEFINE_integer("max_output_sequence_len", 50, "Maximum output sequence length.")
tf.app.flags.DEFINE_integer("rnn_size", 1024, "RNN unit size.")
tf.app.flags.DEFINE_integer("attention_size", 64, "Attention size.")
tf.app.flags.DEFINE_integer("num_layers", 2, "Number of layers.")
tf.app.flags.DEFINE_integer("beam_width", 10, "Width of beam search .")
tf.app.flags.DEFINE_float("learning_rate", 0.001, "Learning rate.")
tf.app.flags.DEFINE_float("max_gradient_norm", 5.0, "Maximum gradient norm.")
tf.app.flags.DEFINE_boolean("forward_only", True, "Forward Only.")
tf.app.flags.DEFINE_string("models_dir", "./models/model1/solid", "Log directory")
tf.app.flags.DEFINE_string("data_path", "", "Training Data path.")
tf.app.flags.DEFINE_string("test_data_path", "", "Test Data path.")
tf.app.flags.DEFINE_string("pred_out_path", "", "Test Data path.")

FLAGS = tf.app.flags.FLAGS

class EntityLinker(object):
    def __init__(self, forward_only):
        self.forward_only = True
        self.epoch = 0
        self.bestf1 = 0
        self.testgraph = tf.Graph()
        with self.testgraph.as_default():
            config = tf.ConfigProto()
            config.gpu_options.allow_growth = True
            config.operation_timeout_in_ms=6000
            self.testsess = tf.Session(config=config)
            self.build_model()

    def build_model(self):
        self.testmodel = None
        with self.testgraph.as_default():
            self.testmodel = pointer_net.PointerNet(batch_size=1,
                    max_input_sequence_len=FLAGS.max_input_sequence_len,
                    max_output_sequence_len=FLAGS.max_output_sequence_len,
                    rnn_size=FLAGS.rnn_size,
                    attention_size=FLAGS.attention_size,
                    num_layers=FLAGS.num_layers,
                    beam_width=FLAGS.beam_width,
                    learning_rate=FLAGS.learning_rate,
                    max_gradient_norm=FLAGS.max_gradient_norm,
                    forward_only=True)
            self.testsess.run(tf.global_variables_initializer())
            ckpt = tf.train.get_checkpoint_state(FLAGS.models_dir)
            print(ckpt, FLAGS.models_dir)
            if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
                print("Load model parameters from %s" % ckpt.model_checkpoint_path)
                self.testmodel.saver.restore(self.testsess, ckpt.model_checkpoint_path)

    def getvector(self,d):
        inputs = []
        enc_input_weights = []
        for question in d:
            questioninputs = []
            enc_input_len = len(question)
            if enc_input_len > FLAGS.max_input_sequence_len:
                print("Length too long, skip")
                continue
            #qemb = question[0][0][1:769]
            for idx,word in enumerate(question):
                questioninputs.append(word[0])
            for i in range(FLAGS.max_input_sequence_len-enc_input_len):
                questioninputs.append([0]*969)
            inputs.append(questioninputs)
            weight = np.zeros(FLAGS.max_input_sequence_len)
            weight[:enc_input_len]=1
            enc_input_weights.append(weight)
        self.test_inputs = np.stack(inputs)
        self.test_enc_input_weights = np.stack(enc_input_weights)

linker = EntityLinker(FLAGS.forward_only)
app = Flask(__name__)
print("serving api ...")

@app.route('/erlinker', methods=['POST'])
def erlinker():
    d = request.get_json(silent=True)
    citem = copy.deepcopy(d)
    inputvecs = []
    linkedvecs = []
    sepseencount = 0
    for cans,canv in zip(d['candidatestring'],d['candidatevectors']):
        inputvecs.append([canv,cans])
        if cans == '[SEP]':
            sepseencount += 1
        if sepseencount < 2:
            linkedvecs.append([canv,cans])
    linkedvecs.append([969*[-1.0],'[SEP]']) # This for now only holds thhe question token + [SEP]. The linked ents and rels will be appended shortly.

    batch = [inputvecs]
    vector = linker.getvector(batch)
    try:
        print(linker.test_inputs.shape)
        predicted,_ = linker.testmodel.step(linker.testsess, linker.test_inputs, linker.test_enc_input_weights, update=False)
        predentrels = set()
        for entrelnum in list(predicted[0][0]):
            if entrelnum <= 0:
                continue
            predentrels.add(inputvecs[entrelnum-1][1])
            linkedvecs.append([inputvecs[entrelnum-1][0],inputvecs[entrelnum-1][1]])
        print(predentrels)
        citem['predentrels'] = list(predentrels)
        del citem['candidatevectors']
        citem['linkedentrelvecs'] = linkedvecs
        citem['linkedentrelstring'] = [x[1] for x in linkedvecs]
        return json.dumps(citem, indent=4, sort_keys=True)
    except Exception as err:
        print("ERROR:",err)
        citem['predentrels'] = []
        del citem['candidatevectors']
        citem['linkedentrelvecs'] = linkedvecs
        citem['linkedentrelstring'] = [x[1] for x in linkedvecs]
        return json.dumps(citem, indent=4, sort_keys=True)

if __name__ == '__main__':
    http_server = WSGIServer(('', 2223), app)
    http_server.serve_forever()


import warnings
from functools import partial

import numpy as np
from sklearn.utils.extmath import squared_norm

from regain.math import create_from_diagonal, fill_diagonal, get_diagonal
from regain.update_rules import update_rho
from regain.utils import Convergence

try:
    from prox_tv import tv1_1d, tvp_1d, tvgen, tvp_2d
except ImportError:
    # fused lasso prox cannot be used
    pass

def soft_thresholding(a, lamda):
    """Soft-thresholding."""
    return np.sign(a) * np.maximum(np.abs(a) - lamda, 0)

def soft_thresholding_off_diagonal(a, lamda):
    """Soft-thresholding."""
    soft = soft_thresholding(a, lamda)
    fill_diagonal(soft, get_diagonal(a))
    return soft

def soft_thresholding_vector(a, lamda):
    """Soft-thresholding for vectors."""
    return np.maximum(1 - lamda / np.linalg.norm(a, axis=-1, keepdims=True), 0) * a

def blockwise_soft_thresholding(a, lamda):
    """Proximal operator for l2 norm column-wise."""
    out = soft_thresholding_vector(a.swapaxes(-1, -2), lamda).swapaxes(-1, -2)
    return out

def blockwise_soft_thresholding_symmetric(a, lamda):
    """Proximal operator for l2 norm, for symmetric matrices (last 2 axes)."""
    col_norms = np.linalg.norm(a, axis=-2)
    out = np.matmul(
        a, create_from_diagonal((1 - lamda / col_norms) * (col_norms > lamda))
    )
    return out

def prox_linf_1d(a, lamda):
    """Proximal operator for the l-inf norm, defined as:

    min ||x||_inf + (1/2t)||x-w||^2

    Since there is no closed-form, we can minimize it with scipy.

    Matlab reference
    ------
    %% Perform prox operator:   min_x (1/2t)||x-w||^2 subject to |x|<=radius
    function [ z ] = project_1ball( z,radius )
    %  By Moreau's identity, projection onto 1-norm ball can be computed
    %  using the proximal of the conjugate problem, which is L-infinity
    %  minimization.
      z = z -  prox_infinityNorm(z,radius);
    end

    %% Perform prox operator:   min ||x||_inf + (1/2t)||x-w||^2
    function [ xk ] = prox_infinityNorm( w,t )
        N = length(w);
        wabs = abs(w);
        ws = (cumsum(sort(wabs,'descend'))- t)./(1:N)';
        alphaopt = max(ws);
        if alphaopt>0
          xk = min(wabs,alphaopt).*sign(w); % truncation step
        else
          xk = zeros(size(w)); % if t is big, then solution is zero
        end
    end
    """
    from scipy.optimize import minimize

    def f(x):
        return lamda * np.linalg.norm(x, np.inf) + 0.5 * squared_norm(a - x)

    return minimize(f, a).x

def prox_linf(a, lamda):
    """Proximal operator for l-inf norm."""
    x = np.zeros_like(a)
    for t in range(a.shape[0]):
        x[t] = np.array([prox_linf_1d(a[t, :, j], lamda) for j in range(a.shape[1])]).T
    return x

def prox_logdet(a, lamda):
    """Time-varying latent variable graphical lasso prox."""
    es, Q = np.linalg.eigh(a)
    xi = (-es + np.sqrt(np.square(es) + 4.0 / lamda)) * lamda / 2.0
    # return np.linalg.multi_dot((Q, np.diag(xi), Q.T))

    # support ndim > 2
    Xi = create_from_diagonal(xi)
    QXiQt = np.matmul(np.matmul(Q, Xi), np.swapaxes(Q, -1, -2))
    return QXiQt

def prox_logdet_ala_ma(a, lamda):
    es, Q = np.linalg.eigh(a)
    xi = (-es + np.sqrt(np.square(es) + 4.0 * lamda)) / 2.0
    # return np.linalg.multi_dot((Q, np.diag(xi), Q.T))

    # support ndim > 2
    Xi = create_from_diagonal(xi)
    QXiQt = np.matmul(np.matmul(Q, Xi), np.swapaxes(Q, -1, -2))
    return QXiQt

def prox_trace_indicator(a, lamda):
    """Time-varying latent variable graphical lasso prox."""
    es, Q = np.linalg.eigh(a)
    xi = np.maximum(es - lamda, 0)
    # return np.linalg.multi_dot((Q, np.diag(xi), Q.T))

    # support ndim > 2
    Xi = create_from_diagonal(xi)
    QXiQt = np.matmul(np.matmul(Q, Xi), np.swapaxes(Q, -1, -2))
    return QXiQt

def prox_laplacian(a, lamda):
    """Prox for l_2 square norm, Laplacian regularisation."""
    return a / (1 + 2.0 * lamda)

def prox_node_penalty(A_12, lamda, rho=1, tol=1e-4, rtol=1e-2, max_iter=500):
    """Lamda = beta / (2. * rho).

    A_12 = np.vstack((A_1, A_2))
    """
    n_time, _, n_dim = A_12.shape

    U_1 = np.full((A_12.shape[0], n_dim, n_dim), 1.0 / n_dim, dtype=float)
    U_2 = np.copy(U_1)
    Y_1 = np.copy(U_1)
    Y_2 = np.copy(U_1)

    C = np.hstack((np.eye(n_dim), -np.eye(n_dim), np.eye(n_dim)))
    inverse = np.linalg.inv(C.T.dot(C) + 2 * np.eye(3 * n_dim))

    V = np.zeros_like(U_1)
    W = np.zeros_like(U_1)
    V_old = np.zeros_like(U_1)
    W_old = np.zeros_like(U_1)

    for iteration_ in range(max_iter):
        A = (
            Y_1 - Y_2 - W - U_1 + (W.transpose(0, 2, 1) - U_2).transpose(0, 2, 1)
        ) / 2.0
        V = blockwise_soft_thresholding_symmetric(A, lamda=lamda)

        A = np.concatenate(((V + U_2).transpose(0, 2, 1), A_12), axis=1)
        D = V + U_1
        # Z = np.linalg.solve(C.T*C + eta*np.identity(3*n), - C.T*D + eta* A)
        Z = np.empty_like(A)
        for i, (A_i, D_i) in enumerate(zip(A, D)):
            Z[i] = inverse.dot(2 * A_i - C.T.dot(D_i))
        W, Y_1, Y_2 = (Z[:, i * n_dim : (i + 1) * n_dim, :] for i in range(3))

        # update residuals
        delta_U_1 = V + W - (Y_1 - Y_2)
        delta_U_2 = V - W.transpose(0, 2, 1)
        U_1 += delta_U_1
        U_2 += delta_U_2

        # diagnostics
        rnorm = np.sqrt(squared_norm(delta_U_1) + squared_norm(delta_U_2))
        snorm = rho * np.sqrt(
            squared_norm(W - W_old) + squared_norm(V + W - V_old - W_old)
        )
        check = Convergence(
            obj=np.nan,
            rnorm=rnorm,
            snorm=snorm,
            e_pri=np.sqrt(2 * V.size) * tol
            + rtol
            * max(
                np.sqrt(squared_norm(W) + squared_norm(V + W)),
                np.sqrt(squared_norm(V) + squared_norm(Y_1 - Y_2)),
            ),
            e_dual=np.sqrt(2 * V.size) * tol
            + rtol * rho * np.sqrt(squared_norm(U_1) + squared_norm(U_2)),
        )
        W_old = W.copy()
        V_old = V.copy()

        # if np.linalg.norm(delta_U_1, 'fro') < tol and \
        #         np.linalg.norm(delta_U_2, 'fro') < tol:
        if check.rnorm <= check.e_pri and check.snorm <= check.e_dual:
            break
        rho_new = update_rho(rho, rnorm, snorm, iteration=iteration_)
        # scaled dual variables should be also rescaled
        U_1 *= rho / rho_new
        U_2 *= rho / rho_new
        rho = rho_new
    else:
        warnings.warn("Node norm did not converge.")
    return Y_1, Y_2

def prox_FL(a, beta, lamda, p=1, symmetric=False, use_matlab=False, optimize=True):
    """Fused Lasso prox.

    It is calculated as the Total variation prox + soft thresholding
    on the solution, as in
    http://ieeexplore.ieee.org/abstract/document/6579659/
    """
    # if any([any(np.diag(x) < 0) for x in a]):
    #     for a_i in a:
    #         np.fill_diagonal(a_i, np.sum(np.abs(a_i), axis=1))
    if optimize:
        Y = tvgen(a, [beta], [1], [p], n_threads=32, max_iters=30)

    else:
        Y = np.empty_like(a)
        # if use_matlab:
        #     from regain.wrapper.tv_condat import wrapper
        #     func = wrapper.total_variation_condat
        # else:
        func = tv1_1d if p == 1 else partial(tvp_1d, p=p)

        if symmetric:
            x, y = np.triu_indices_from(a[0])
            b = np.vstack(a.transpose(1, 2, 0))
            upper_ind = x * a.shape[1] + y
            Z = np.zeros_like(b)
            Z[upper_ind] = [func(row, beta) for row in b[upper_ind]]

            e = np.array(np.split(Z, a.shape[1], axis=0)).transpose(2, 0, 1)
            Y = (e + e.transpose(0, 2, 1)) / (
                np.array([np.eye(a.shape[1]) for i in range(a.shape[0])]) + 1
            )
        else:
            for i in range(np.power(a.shape[1], 2)):
                solution = func(a.flat[i :: np.power(a.shape[1], 2)], beta)
                Y.flat[i :: np.power(a.shape[1], 2)] = solution

    # fused-lasso (soft-thresholding on the solution)
    Y_soft = soft_thresholding(Y, lamda)

    # restore diagonal
    for y_s, y_fv in zip(Y_soft, Y):
        np.fill_diagonal(y_s, np.diag(y_fv))
    return Y_soft

    # Y = np.empty_like(a)
    # for i in range(a.shape[1]):
    #     for j in range(i, a.shape[2]):
    #         solution = tv1_1d(a[:, i, j], beta)
    #         # fused-lasso (soft-thresholding on the solution)
    #         if i != j:
    #             solution = soft_thresholding_sign(solution, lamda)
    #         Y[:, i, j] = solution
    # return Y

    # not work
    # x = np.vstack(a.transpose((2, 1, 0)))  # each row is the time evolution
    # assert np.allclose(np.array(xx), x)
    # x = np.apply_along_axis(
    #     compose(partial(soft_thresholding_sign, lamda=lamda),
    #             partial(tv1_1d, w=beta)), 1, x)
    # # return np.array(np.vsplit(x, x.shape[1])).transpose((2, 1, 0))
    # Z = np.array(np.vsplit(x, a.shape[1]))#.transpose((2, 1, 0))


import numpy as np
import matplotlib.pyplot as plt
from ortools.sat.python import cp_model
import sys
import time

if len(sys.argv) < 2:
    quit()
try:
    istanza = int(sys.argv[1])
except:
    print("Metti un numero da 1 a 11 per runnure un'istanza, 0 per banale e -1 per banale ma non troppo!")
    quit()

file = ""
if istanza == 0:
    file = "inst_banale"
elif istanza == -1:
    file = "inst_banale_ma_non_troppo"
elif istanza < 10:
    file = "inst0{}".format(istanza)
else:
    file = "inst{}".format(istanza)
f = open('tsp_inst/' + file + ".dzn", "r")
text = f.read()
f.close()
content = text.split('\n')[:-2]

lower_boud = int(content[6][16:-1])

content = [content[i][4:-1] for i in range(len(content)) if content[i][0] != '%']
content[2] = content[2][1:-3]
content[3] = content[3][1:-3]
content[4] = content[4][1:-3]
content[5] = content[5][1:-3]

n = int(content[1])
m = int(content[0])

l = [int(L) for L in content[2].split(',')]
s = [int(L) for L in content[3].split(',')]
x = [int(L) for L in content[4].split(',')]
y = [int(L) for L in content[5].split(',')]

x = x + [x[-1]]*(m-1)
y = y + [y[-1]]*(m-1)

N = n + m
ITEMS = list(range(n))
NODES = list(range(N))
COURIERS = list(range(m))

D = np.array([[0 for j in range(N)] for i in range(N)])

for i in range(N):
    for j in range(N):
        if i > n and j > n:
            D[i, j] = 0
        elif i > n:
            D[i, j] = abs(x[n] - x[j]) + abs(y[n] - y[j])
        elif j > n:
            D[i, j] = abs(x[i] - x[n]) + abs(y[i] - y[n])
        else:
            D[i, j] = abs(x[i] - x[j]) + abs(y[i] - y[j])

model = cp_model.CpModel()

# VARIABLES
table = []
for i in NODES:
    rows = []
    for j in NODES:
        rows.append(model.NewBoolVar(name=f't[{i},{j}]'))
    table.append(rows)

# VARIABLES
courier = []
for k in COURIERS:
    rows = []
    for i in ITEMS:
        rows.append(model.NewBoolVar(name=f'courier[{k},{i}]'))
    courier.append(rows)

# ------ initialization constraints ----

# Ongi oggetto raggiunto esattamente una volta
for i in ITEMS:
    model.AddExactlyOne(table[i][j] for j in NODES)
    model.AddExactlyOne(table[j][i] for j in NODES)

for i in ITEMS:
    model.Add(table[i][i] == False)

for i in NODES:
    for j in NODES:
        if i != j:
            model.Add(table[i][j] == False).OnlyEnforceIf(table[j][i])

for i in range(n, N):
    for j in range(n, N):
        if i != j:
            model.Add(table[i][j] == False)

for i in ITEMS:
    model.AddExactlyOne(courier[k][i] for k in COURIERS)

for i in range(n, N):
    for j in ITEMS:
        model.Add(courier[i-n][j] == True).OnlyEnforceIf(table[i][j])

for k in COURIERS:
    for i in ITEMS:
        for j in ITEMS:
            model.Add(courier[k][i] == courier[k][j]).OnlyEnforceIf(table[i][j])

for k in COURIERS:
    model.Add(sum(courier[k][j]*s[j] for j in ITEMS) <= l[k])

"""
# SYMMETRY BREAKING
for i in range(n, N):
    model.Add(sum(table[i][j]*(j+1)*(j+1) for j in ITEMS) >= sum(table[j][i]*(j+1)*(j+1) for j in ITEMS))

for j in range(n, N):
    for jj in range(j+1, N):
        model.Add(sum(table[i][j]*(i+1)*(i+1) for i in ITEMS) <= sum(table[i][jj]*(i+1)*(i+1) for i in ITEMS))
"""

for k in COURIERS:
    for kk in range(k+1, m):
        if l[k] == l[kk]:
            model.Add(sum(courier[k][j]*(j+1)*(j+1) for j in ITEMS) <= sum(courier[kk][j]*(j+1)*(j+1) for j in ITEMS))
"""
        if l[k] >= l[kk]:
            model.Add(sum(courier[k][j]*s[j] for j in ITEMS) >= sum(courier[kk][j]*s[j] for j in ITEMS))
"""

arcs = []
for i in NODES:
    for j in NODES:
        arcs.append((i, j, table[i][j]))
model.AddCircuit(arcs)

obj = cp_model.LinearExpr.Sum([table[i][j]*D[i, j] for i in NODES for j in NODES])
model.Add(obj >= lower_boud)

model.Minimize(obj)
solver = cp_model.CpSolver()
# Massimo 5 minuti
solver.parameters.max_time_in_seconds = 300.0

solver.num_search_workers = 4
print("Start solving...")
status = solver.Solve(model)
print(status)

if status == cp_model.OPTIMAL or status == cp_model.FEASIBLE:

    print(f"\nPlan:\n")
    for i in NODES:
        for j in NODES:
            print(f'{solver.Value(table[i][j])}', end=" ")
        print("")
    print("--------------------------------------")

    for k in COURIERS:
        print(f"\nCourier{k}:", end=" ")
        for i in ITEMS:
            print(f'{solver.Value(courier[k][i])}', end=" ")

    print(f'\nMinimum of objective function: {solver.ObjectiveValue()}\n')
    for k in COURIERS:
        somma = 0
        for j in ITEMS:
            somma += solver.Value(courier[k][j])*s[j]
        print("sum[{}] = {} and l[{}] = {}".format(k, somma, k, l[k]))
    for i in range(len(x)):
        plt.gca().add_patch(plt.Circle((x[i],y[i]), 0.1, color='r'))
    x1s = []
    y1s = []
    x2s = []
    y2s = []
    color_i = 1
    color = 'C1'
    for k in range(m):
        x1s.clear()
        y1s.clear()
        x2s.clear()
        y2s.clear()
        for i in range(N):
            for j in range(N):
                if i != j:
                    if solver.Value(table[i][j]):
                        x1s.append(x[i])
                        y1s.append(y[i])
                        x2s.append(x[j]-x[i])
                        y2s.append(y[j]-y[i])
        plt.quiver(x1s, y1s, x2s, y2s, angles='xy', scale_units='xy', color=[color], scale=1, width=0.003)
        color_i += 1
        color = 'C'+str(color_i)
    plt.xlim(min(x)-2,max(x)+2)
    plt.ylim(min(y)-2,max(y)+2)
    plt.show()
else:
    print('No solution found.')

"""
plt.scatter(x[:-1], y[:-1], c='b')
for i in range(n):
    plt.annotate(f'q{i}', (x[i]+0.05, y[i]+0.05))

for k in COURIERS:
    for i in NODES:
        for j in NODES:
            if solver.Value(table[k][i][j]) == 1:
                plt.plot([x[i], x[j]], [y[i], y[j]], c='y', alpha=0.4)

plt.plot(x[-1], y[-1], c='r', marker='s')
plt.show()
"""


"""Shutil file extractor module"""
import logging
import shutil

from tools.triage.utils.extraction import ExtractorInterface

class ShutilExtractor(ExtractorInterface):
    """Responsible for handling the extraction of supported files using shutil"""

    def __init__(self):
        self.logger = logging.getLogger(__name__)

    def can_handle_file(self, filename: str) -> bool:
        """Overrides ExtractorInterface.can_handle_file()"""
        if not super().can_handle_file(filename):
            return False

        # Check to see if shutil supports the file extension
        supported_extensions = []
        for format in shutil.get_unpack_formats():
            supported_extensions.extend(format[1])
        for ext in supported_extensions:
            if filename.endswith(ext):
                return True
        return False

    def extract(self, filename, destination_directory) -> str:
        """Overrides ExtractorInterface.extract()"""
        if not self.can_handle_file(filename):
            message = f"This extractor does not support the file {filename}"
            self.logger.error(message)
            return
        try:
            shutil.unpack_archive(filename, destination_directory)
            return destination_directory
        except Exception as e:
            message = f"An error was encountered during shutil extraction of {filename} to {destination_directory}, the error was {e}"
            self.logger.error(message)


import argparse
import ntpath
import os
import time

import camkifu.core
import ckmain
import test.objects

"""
Script that helps with running successive detection tests in one go. To be called on a directory containing
video files and reference sgf files.

For each sgf reference (supposed to contain the expected results), look for a file with the same name but a
different extension, and run the vision algorithms on that file. Stop when all references have been processed.

Expand / modify the Report class depending on the kind of information needed.

"""

def list_bench(refdir, vid_dir) -> dict:
    """
    Map each sgf reference to the first matching "video" file (if found) in a dict.
    The "video" file is the first file with a matching name, the extension is not checked.

    refdir -- the directory containing the sgf files
    vid_dir -- the directory containing the video files

    """
    # step 0: arrange paths
    if not vid_dir.endswith("/"):
        vid_dir += "/"
    if refdir is None:
        refdir = vid_dir
    elif not refdir.endswith("/"):
        refdir += "/"

    # step 1: list video candidates (all files that are not sgf)
    vid_files = []
    for f in os.listdir(vid_dir):
        if not f.endswith('.sgf'):
            vid_files.append(f)

    # step 2: match sgf references to video candidates
    bench = {}
    unmatched = []
    for sgf in os.listdir(refdir):
        if sgf.endswith('.sgf'):
            matched = False
            key = ntpath.basename(sgf)[:-4]
            for vf in vid_files:
                if ntpath.basename(vf).startswith(key):
                    matched = True
                    if sgf not in bench:
                        bench[refdir + sgf] = vid_dir + vf
                    else:
                        msg_base = "Benchmark warning: multiple video candidates for reference file {}.sgf : {}"
                        print(msg_base.format(key, [ntpath.basename(bench[sgf]), vf]))
                        break  # at least one warning should be enough to get attention
            if not matched:
                unmatched.append(ntpath.basename(sgf))
    if len(unmatched):
        print("Benchmark warning: unmatched reference files {}".format(unmatched))
    return bench

def main(vid_dir, refdir=None, bf=None, sf=None):
    """
    vid_dir -- the directory containing the videos on which to run the benchmark. May also contain the reference sgf(s).

    """
    bench = list_bench(refdir, vid_dir)
    reports = []
    for i, (sgf, vid) in enumerate(bench.items()):
        msg_base = "{} ({}/{}) ----------------------------------------------------------------"
        print(msg_base.format(ntpath.basename(vid), i+1, len(bench)))
        controller = test.objects.ControllerVTest(sgf, video=vid)
        vmanager = camkifu.core.VManager(controller, imqueue=DummyQueue(), bf=bf, sf=sf)
        stop_condition = lambda: vmanager.hasrun and not vmanager.is_processing()
        test.objects.ProcessKiller(vmanager, stop_condition).start()
        start = time.time()
        vmanager.run()
        reports.append(Report(vmanager, time.time() - start))
    for r in reports:
        print(r)

class Report:

    def __init__(self, vmanager, duration):
        self.vmanager = vmanager
        self.sgf = vmanager.controller.kifu.ref.sgffile
        self.video = vmanager.controller.video
        self.matcher = vmanager.controller.kifu.check()
        self.duration = duration

    def __repr__(self):
        name = ntpath.basename(self.sgf)[:-4]
        percent = round(100*self.matcher.ratio(), 1)
        return "[{}: {}% in {} s]".format(name, percent, int(round(self.duration)))

class DummyQueue:
    def put(self, x):
        pass

    def put_nowait(self, x):
        pass

def get_argparser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser()
    ckmain.add_finder_args(parser)

    # compulsory argument
    parser.add_argument("dir", help="Directory containing the videos on which to run the benchmark.")

    # optional arguments
    refdir_help = "Directory containing the reference sgfs associated with each video (matched on filename)." \
                  " Defaults to \"dir\" if omitted."
    parser.add_argument("--refdir", help=refdir_help)

    return parser

if __name__ == '__main__':
    args = get_argparser().parse_args()
    main(args.dir, refdir=args.refdir, bf=args.bf, sf=args.sf)


import ee
import time
from datetime import datetime
from google.cloud import storage
import os
import rasterio

TASK_FINISHED_STATES = (ee.batch.Task.State.COMPLETED,
                        ee.batch.Task.State.FAILED,
                        ee.batch.Task.State.CANCELLED)

class getJsonEnv():
    """
    Grabs .env
    """
    def __init__(self):

        with open('gcsPrivatekey.json','w') as f:
            f.write(os.getenv('GCS_JSON'))

        with open('geePrivatekey.json','w') as f:
            f.write(os.getenv('GEE_JSON'))

class assetManagement(object):
    ## Connects to bucket, upload the image and once it is ready transfers it to GEE associated collection.
    """
    ImageObject:
    {
    'sources':['/Users/alicia/Downloads/results%2Fhistorical%2Fdecadal_test_historical_1991_2000_hdds.tiff'],
    'gcsBucket':'gee-image-transfer',
    'collectionAsset':'users/Aliciaarenzana/testcollection',
    'assetName':'t2000',
    'bandNames':[{'id': 'R'}, {'id': 'G'}, {'id': 'B'}],
    'pyramidingPolicy':'MODE',
    'properties':{
        'my_imageProperties':'to add to the collection'
        }
    }
    """
    def __init__(self,imageObject):
        """checks the image and sets up the properties """
        getJsonEnv()
        self.meta=imageObject
        self.imageNames=self.getImageName()
        self.gcsBucket=self.setUpCredentials()
        self.sources = []

    def getImageName(self):
        """gets the listof names from sources"""
        return [os.path.basename(name) for name in self.meta['sources']]

    def checksImages(self):
        """Checks the images that we will compose have the same n bands as they are going to become one image and part of the image collection"""
        metadata=[]
        for image in self.meta['sources']:
            with rasterio.open(image) as src:
                metaData=src.meta

                assert metaData['driver'] == 'GTiff', "Driver is not supported: {0}".format(metaData['driver'])
                assert metaData['count'] == len(self.meta['bandNames']), "Nbands incorrect, expected: {0}, {1} provided".format(metaData['count'],len(self.meta['bandNames']))

                metadata.append({'dtype': metaData['dtype'], 'driver': metaData['driver'], 'nodata': metaData['nodata'], 'nBands': metaData['count'],'crs': src.crs.to_string()})

        assert len(set([item['dtype'] for item in metadata])) == 1, "Images list dtypes aren't compatibles. Expected: 1, {1} provided".format(metaData['count'],len(set([item['dtype'] for item in metadata])))
        assert len(set([item['driver'] for item in metadata])) == 1, "Images list drivers aren't compatibles. Expected: 1, 1 provided".format(metaData['count'],len(set([item['driver'] for item in metadata])))
        assert len(set([item['nodata'] for item in metadata])) == 1, "Images list nodata values aren't compatibles. Expected: 1, {1} provided".format(metaData['count'],len(set([item['nodata'] for item in metadata])))
        assert len(set([item['nBands'] for item in metadata])) == 1, "Images list nBands number aren't compatibles. Expected: 1, {1} provided".format(metaData['count'],len(set([item['nBands'] for item in metadata])))
        assert len(set([item['crs'] for item in metadata])) == 1, "Images list crs aren't compatibles. Expected: 1, {1} provided".format(metaData['count'],len(set([item['crs'] for item in metadata])))
        return metadata[0]

    def setUpCredentials(self):
        """Sets up the credentials"""
        credentials = ee.ServiceAccountCredentials(os.getenv('GEE_SACCOUNT'), 'geePrivatekey.json')
        ee.Initialize(credentials)
        storage_client=storage.Client.from_service_account_json('gcsPrivatekey.json')
        return storage_client.get_bucket(self.meta['gcsBucket'])

    def setUpGeeAsset(self):
        aclSet='{"all_users_can_read" : true}'
        collectionAsset=self.meta['collectionAsset'].split('/')
        assetHome ="{0}/{1}".format(collectionAsset[0],collectionAsset[1])
        if ee.data.getInfo(assetHome) == None:
            ee.data.createAssetHome(assetHome)
        if ee.data.getInfo(self.meta['collectionAsset']) == None:
            ee.data.createAsset({'type': 'ImageCollection'}, self.meta['collectionAsset'])
            ee.data.setAssetAcl(self.meta['collectionAsset'],aclSet)

    def uploadGCS(self, imageName):
        """Upload the image to google cloud storage"""
        imageIndex = self.imageNames.index(imageName)
        blob = self.gcsBucket.blob('{0}/{1}'.format(self.meta['collectionAsset'],imageName))
        blob.upload_from_filename(self.meta['sources'][imageIndex])
        blob.make_public()

        return {'primaryPath': 'gs://{gcsBucket}/{collectionName}/{imageNa}'.format(gcsBucket=self.meta['gcsBucket'],collectionName=self.meta['collectionAsset'],imageNa=imageName)}

    def transferGEE(self):
        """Transfers the images from google cloud storage to gee asset"""
        task_id = ee.data.newTaskId()[0]
        time = self.meta['properties']['system:time_start']

        self.meta['properties']['system:time_start'] = ee.Date(time).getInfo()['value']

        request = {
            'id':'{collectionAsset}/{assetName}'.format(collectionAsset= self.meta['collectionAsset'],assetName =self.meta['assetName']),
            'properties':self.meta['properties'],
            'tilesets': [{'sources': self.sources}],
            'pyramidingPolicy':self.meta['pyramidingPolicy'].upper(),
            'bands':self.meta['bandNames']
        }
        ee.data.startIngestion(task_id, request, True)
        return task_id

    def taskStatus(self, task_id, timeout=90, log_progress=True):
        """Waits for the specified task to finish, or a timeout to occur. (thanks to gee cli)"""
        start = time.time()
        elapsed = 0
        last_check = 0
        while True:
            elapsed = time.time() - start
            status = ee.data.getTaskStatus(task_id)[0]
            state = status['state']
            if state in TASK_FINISHED_STATES:
              error_message = status.get('error_message', None)
              print('Task %s ended at state: %s after %.2f seconds'
                    % (task_id, state, elapsed))
              if error_message:
                raise ValueError(error_message)
              return
            if log_progress and elapsed - last_check >= 30:
              print('[{:%H:%M:%S}] Current state for task {}: {}'
                    .format(datetime.now(), task_id, state))
              last_check = elapsed
            remaining = timeout - elapsed
            if remaining > 0:
              time.sleep(min(10, remaining))
            else:
              break
        print('Wait for task %s timed out after %.2f seconds' % (task_id, elapsed))

    def execute(self):
        #Checks if the images are correct
        self.checksImages()

        #sets up credentials and assets

        self.setUpGeeAsset()

        #Uploads file/s to GCS
        self.sources = list(map(self.uploadGCS, self.imageNames))

        #Transfers it from GCS to GEE
        task_id = self.transferGEE()

        print('TaskID: {0}'.format(task_id))
        print('Status: {0}'.format(ee.data.getTaskStatus(task_id)[0]))
        self.taskStatus(task_id)


#!/usr/bin/python3

# requirements
from tornado import websocket, web, ioloop
import datetime
import logging
import json
import os

class StatsManager:

    # general attributes
    bootTime = None

    # attributes for requests
    requests = {}
    requests["total"] = 0
    requests["failed"] = 0
    requests["successful"] = 0
    requests["avg_time"] = 0
    requests["paths"] = {}

    def __init__(self, conf):
        self.bootTime = datetime.datetime.now()
        self.conf = conf

    def getStats(self):
        return {"bootTime": str(self.bootTime),
                "upTime": str(datetime.datetime.now() - self.bootTime),
                "requests": self.requests,
                "config": self.conf.tools
        }

    def reset(self):
        self.total_requests = 0
        self.failed_requests = 0
        self.successful_requests = 0

class StatsProcessor():

    def __init__(self, stats):
        self.stats = stats
        self.logger = logging.getLogger("mediatorLogger")
        self.template = os.path.join(os.path.dirname(__file__), "templates/stats.html")

    def getStats(self):

        # get stats
        return self.stats.getStats()

    def get(self):

        if self.request.path == "/stats":

            # debug print
            self.logger.debug("Stats request")

            # get stats
            res = self.stats.getStats()

            # json or html?
            key = self.get_argument('format', None)
            if key == "json":

                # return a json dict
                self.set_header("Content-Type", "application/json")
                self.write(json.dumps(res))

            else:

                # render a web page
                self.render(self.template, stats=res)

        elif self.request.path == "/clearStats":

            # reset stats
            self.stats.reset()

            # return the stats
            res = self.stats.getStats()
            self.set_header("Content-Type", "application/json")
            self.write(json.dumps(res))


# https://leetcode.com/problems/check-if-all-the-integers-in-a-range-are-covered/
import pytest

class Solution:
    def isCovered(self, ranges: list[list[int]], left: int, right: int) -> bool:
        dict_nums = {i: 0 for i in range(left, right + 1)}

        for (i, j) in ranges:
            while i <= j:
                dict_nums[i] = 1
                i += 1

        return all(dict_nums.values())

@pytest.mark.parametrize(
    ("ranges", "left", "right", "result"),
    [
        ([[1, 2], [3, 4], [5, 6]], 2, 5, True),
        ([[1, 50]], 1, 50, True),
        ([[1, 2], [5, 6]], 2, 5, False),
    ],
)
def test_isCovered(
    ranges: list[list[int]], left: int, right: int, result: bool
) -> None:
    assert result == Solution().isCovered(ranges, left, right)

print(Solution().isCovered([[1, 2], [3, 4], [5, 6]], 2, 5))


import decisionlib
from typing import List, Optional, Dict, Any, Callable
from gha import GithubAction, GithubActionScript
from typing import List
from decisionlib import CONFIG

BUILD_ARTIFACTS_EXPIRE_IN = "1 week"
PAHKAT_REPO = "https://pahkat.uit.no/"

def linux_build_task(name, bundle_dest="repo", with_secrets=True):
    task = (
        decisionlib.DockerWorkerTask(name)
        .with_worker_type("linux")
        .with_provisioner_id("divvun")
        .with_docker_image("ubuntu:hirsute")
        # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
        .with_scopes("docker-worker:cache:divvun-*")
        .with_scopes("queue:get-artifact:private/*")
        .with_scopes("queue:get-artifact:public/*")
        .with_scopes("object:upload:divvun:*")
        .with_scopes("secrets:get:divvun" * with_secrets)
        .with_features("taskclusterProxy")
        .with_caches(
            **{
                "divvun-cargo-registry": "/root/.cargo/registry",
                "divvun-cargo-git": "/root/.cargo/git",
                "divvun-rustup": "/root/.rustup",
            }
        )
        .with_index_and_artifacts_expire_in(BUILD_ARTIFACTS_EXPIRE_IN)
        .with_max_run_time_minutes(60)
        .with_script("mkdir -p $HOME/tasks/$TASK_ID")
        .with_script("mkdir -p $HOME/tasks/$TASK_ID/_temp")
        .with_apt_update()
        .with_apt_install("curl", "git", "python3", "python3-pip")
        .with_pip_install("taskcluster", "pyYAML")
        .with_apt_install("wget", "nodejs", "awscli")
        .with_repo_bundle("repo", bundle_dest)
        .with_repo_bundle("ci", "ci")
        .with_script(f"cd $HOME/tasks/$TASK_ID/{bundle_dest}")
    )
    return task

def macos_task(name):
    return (
        decisionlib.MacOsGenericWorkerTask(name)
        .with_worker_type("macos")
        .with_scopes("queue:get-artifact:private/*")
        .with_scopes("queue:get-artifact:public/*")
        .with_scopes("object:upload:divvun:*")
        .with_scopes("secrets:get:divvun")
        .with_index_and_artifacts_expire_in(BUILD_ARTIFACTS_EXPIRE_IN)
        .with_max_run_time_minutes(60)
        .with_provisioner_id("divvun")
        .with_features("taskclusterProxy")
        .with_script("mkdir -p $HOME/tasks/$TASK_ID")
        .with_script("mkdir -p $HOME/tasks/$TASK_ID/_temp")
        .with_repo_bundle("ci", "ci")
        .with_repo_bundle("repo", "repo")
        .with_script("cd $HOME/tasks/$TASK_ID/repo")
    )

def windows_task(name):
    return (
        decisionlib.WindowsGenericWorkerTask(name)
        .with_worker_type("windows")
        .with_provisioner_id("divvun")
        .with_scopes("queue:get-artifact:private/*")
        .with_scopes("queue:get-artifact:public/*")
        .with_scopes("object:upload:divvun:*")
        .with_scopes("secrets:get:divvun")
        .with_index_and_artifacts_expire_in(BUILD_ARTIFACTS_EXPIRE_IN)
        .with_max_run_time_minutes(60)
        .with_script("mkdir %HOMEDRIVE%%HOMEPATH%\\%TASK_ID%")
        .with_script("mkdir %HOMEDRIVE%%HOMEPATH%\\%TASK_ID%\\_temp")
        .with_features("taskclusterProxy")
        .with_repo_bundle("ci", "ci")
        .with_repo_bundle("repo", "repo")
        .with_python3()
        .with_script("pip install --user taskcluster")
        .with_script("cd %HOMEDRIVE%%HOMEPATH%\\%TASK_ID%\\repo")
    )

def gha_setup():
    return GithubAction("Eijebong/divvun-actions/setup", {}).with_secret_input(
        "key", "divvun", "DIVVUN_KEY"
    )

def gha_pahkat(packages: List[str]):
    return GithubAction(
        "Eijebong/divvun-actions/pahkat/init",
        {
            "repo": "https://pahkat.uit.no/devtools/",
            "channel": "nightly",
            "packages": ",".join(packages),
        },
    )

def _generic_rust_build_upload_task(
    os_,
    task_name,
    cargo_toml_path,
    package_id,
    target_dir,
    bin_name,
    env,
    setup_uploader,
    rename_binary,
    features,
    version_action
):
    if os_ == "windows":
        target_dir = "\\".join(target_dir.split("/"))
        task_new = windows_task
        install_rust = GithubAction(
            "actions-rs/toolchain",
            {
                "toolchain": "stable",
                "profile": "minimal",
                "override": "true",
                "components": "rustfmt",
                "target": "i686-pc-windows-msvc",
            },
        )
        build = GithubAction(
            "actions-rs/cargo",
            {
                "command": "build",
                "args": f"--release {features} --manifest-path {cargo_toml_path} --target i686-pc-windows-msvc",
            },
        )
        dist = GithubActionScript(
            f"mkdir dist\\bin && move {target_dir}\\i686-pc-windows-msvc\\release\\{bin_name}.exe dist\\bin\\{rename_binary}.exe"
        )
        sign = GithubAction(
            "Eijebong/divvun-actions/codesign",
            {"path": f"dist/bin/{rename_binary}.exe"},
        )
        deploy = GithubAction(
            "Eijebong/divvun-actions/deploy",
            {
                "package-id": package_id,
                "type": "TarballPackage",
                "platform": "windows",
                "arch": "i686",
                "repo": PAHKAT_REPO + "devtools/",
                "version": "${{ steps.version.outputs.version }}",
                "channel": "${{ steps.version.outputs.channel }}",
                "payload-path": "${{ steps.tarball.outputs['txz-path'] }}",
            },
        )
    elif os_ == "macos":
        task_new = macos_task
        install_rust = GithubAction(
            "actions-rs/toolchain",
            {
                "toolchain": "stable",
                "profile": "minimal",
                "override": "true",
                "components": "rustfmt",
            },
        )
        build = GithubAction(
            "actions-rs/cargo",
            {
                "command": "build",
                "args": f"--release {features} --manifest-path {cargo_toml_path}",
            },
        )
        dist = GithubActionScript(
            f"mkdir -p dist/bin && mv {target_dir}/release/{bin_name} dist/bin/{rename_binary}"
        )
        sign = GithubAction(
            "Eijebong/divvun-actions/codesign", {"path": f"dist/bin/{rename_binary}"}
        )
        deploy = GithubAction(
            "Eijebong/divvun-actions/deploy",
            {
                "package-id": package_id,
                "type": "TarballPackage",
                "platform": "macos",
                "arch": "x86_64",
                "repo": PAHKAT_REPO + "devtools/",
                "version": "${{ steps.version.outputs.version }}",
                "channel": "${{ steps.version.outputs.channel }}",
                "payload-path": "${{ steps.tarball.outputs['txz-path'] }}",
            },
        )
    elif os_ == "linux":
        task_new = lambda name: linux_build_task(name).with_gha(
            "setup_linux", GithubActionScript("apt install -y musl musl-tools")
        )
        install_rust = GithubAction(
            "actions-rs/toolchain",
            {
                "toolchain": "stable",
                "profile": "minimal",
                "override": "true",
                "components": "rustfmt",
                "target": "x86_64-unknown-linux-musl",
            },
        )
        build = GithubAction(
            "actions-rs/cargo",
            {
                "command": "build",
                "args": f"--release {features} --manifest-path {cargo_toml_path}",
            },
        )
        dist = GithubActionScript(
            f"mkdir -p dist/bin && mv {target_dir}/release/{bin_name} dist/bin/{rename_binary}"
        )
        sign = GithubActionScript('echo "No code signing on linux"')
        deploy = GithubAction(
            "Eijebong/divvun-actions/deploy",
            {
                "package-id": package_id,
                "type": "TarballPackage",
                "platform": "linux",
                "arch": "x86_64",
                "repo": PAHKAT_REPO + "devtools/",
                "version": "${{ steps.version.outputs.version }}",
                "channel": "${{ steps.version.outputs.channel }}",
                "payload-path": "${{ steps.tarball.outputs['txz-path'] }}",
            },
        )
    else:
        raise NotImplementedError

    return (
        task_new(f"{task_name}: {os_}")
        .with_env(**env)
        .with_script(
            r'call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat"'
            if os_ == "windows"
            else ""
        )
        .with_gha("setup", gha_setup())
        # The actions-rs action is broken on windows
        .with_gha(
            "install_rustup",
            GithubActionScript(
                "choco install -y --force rustup.install && echo ::add-path::${HOME}/.cargo/bin"
            ),
            enabled=(os_ == "windows"),
        )
        .with_gha("version", version_action)
        .with_gha("install_rust", install_rust)
        .with_gha("build", build)
        .with_gha("dist", dist)
        .with_gha("sign", sign)
        .with_gha(
            "tarball",
            GithubAction("Eijebong/divvun-actions/create-txz", {"path": "dist"}),
        )
        .with_gha("setup_uploader", setup_uploader)
        .with_gha(
            "deploy",
            deploy.with_secret_input("GITHUB_TOKEN", "divvun", "GITHUB_TOKEN"),
        )
        .with_prep_gha_tasks()
        .find_or_create(f"build.{bin_name}.{os_}.{CONFIG.git_sha}")
    )

def generic_rust_build_upload_task(
    task_name: str,
    cargo_toml_path: str,
    package_id: str,
    target_dir: str,
    bin_name: str,
    env: Dict[str, Any],
    setup_uploader: Callable[[str], GithubAction],
    rename_binary: Optional[str]=None,
    get_features: Optional[Callable[[str], GithubAction]]=None,
    version_action: Optional[GithubAction]=None,
    only_os: Optional[List[str]]=None
):
    if rename_binary is None:
        rename_binary = bin_name
    if version_action is None:
        version_action = GithubAction("Eijebong/divvun-actions/version", {"cargo": cargo_toml_path, "nightly": "main, develop"}).with_secret_input("GITHUB_TOKEN", "divvun", "GITHUB_TOKEN")
    if only_os is None:
        only_os = ["macos", "windows", "linux"]

    for os_ in only_os:
        if get_features is not None:
            features = get_features(os_)
        else:
            features = ""
        _generic_rust_build_upload_task(
            os_,
            task_name,
            cargo_toml_path,
            package_id,
            target_dir,
            bin_name,
            env,
            setup_uploader(os_),
            rename_binary,
            features,
            version_action
        )


import cv2
import os
from PIL import Image
provinces = [
    "zh_cuan", "川",
    "zh_e", "鄂",
    "zh_gan", "赣",
    "zh_gan1", "甘",
    "zh_gui", "贵",
    "zh_gui1", "桂",
    "zh_hei", "黑",
    "zh_hu", "沪",
    "zh_ji", "冀",
    "zh_jin", "津",
    "zh_jing", "京",
    "zh_jl", "吉",
    "zh_liao", "辽",
    "zh_lu", "鲁",
    "zh_meng", "蒙",
    "zh_min", "闽",
    "zh_ning", "宁",
    "zh_qing", "靑",
    "zh_qiong", "琼",
    "zh_shan", "陕",
    "zh_su", "苏",
    "zh_sx", "晋",
    "zh_wan", "皖",
    "zh_xiang", "湘",
    "zh_xin", "新",
    "zh_yu", "豫",
    "zh_yu1", "渝",
    "zh_yue", "粤",
    "zh_yun", "云",
    "zh_zang", "藏",
    "zh_zhe", "浙"
]
PROVINCES = ("京", "闽", "粤", "苏", "沪", "浙", "川", "鄂", "赣", "甘", "贵", "桂", "黑", "冀", "津", "吉", "辽", "鲁", "蒙"
             , "宁", "青", "琼", "陕", "晋", "皖", "湘", "新", "豫", "渝", "云", "藏")
# 图像放大原来的两倍
# img = cv2.imread("/home/python/Desktop/opencv_test/opencv_test1/train_shouxie/Validation/#/__0_121440.png", 0)
# height, width = img.shape[:2]
# res = cv2.resize(img, (2 * width, 2 * height), interpolation=cv2.INTER_CUBIC)
# cv2.imshow("res", res)
# gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# cv2.imshow("img", img)
# cv2.imshow("gray_img", gray_img)
# cv2.waitKey(0)
# cv2.destroyAllWindows()

# path = "/home/python/Desktop/opencv_test/opencv_demo/opencv_test1/chinese_test/jpg/car9zheF77777.jpg"
# img = cv2.imread(path)
# img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# img = cv2.resize(img, (32, 40), interpolation=cv2.INTER_AREA)
# a = path.split(".")
# new_path = a[0] + ".bmp"
# cv2.imwrite(path, img)
# image = Image.open(path)
# new_pic = image.convert("L")
# new_pic.save(new_path)
# # os.remove(path)
# print("图片{}修改格式成功".format(os.path.basename(path)))
# cv2.imshow("img", img)
# # os.remove("/home/python/Desktop/opencv_test/opencv_test1/test_pic/car1.jpg")
# cv2.waitKey(0)

# tensorflow格式图片
# for root, dirs, files in os.walk("/home/python/Desktop/charsChinese3"):
#     if not os.path.basename(root).startswith("zh_"):
#         continue
#     if os.path.basename(root) in ["charsChinese1"]:
#         continue
#     for j, filename in enumerate(files):
#         filepath = os.path.join(root, filename)
#         a = filename.split(".")
#         new_filename = a[0] + ".bmp"
#         new_filepath = os.path.join(root, new_filename)
#         digit_img = cv2.imread(filepath)
#         img = cv2.resize(digit_img, (32, 40), interpolation=cv2.INTER_AREA)
#         cv2.imwrite(new_filepath, img)
#         digit_img = Image.open(new_filepath)
#         new_pic = digit_img.convert("L")
#         new_pic.save(new_filepath)
#         os.remove(filepath)
#         print("改写完一张{}".format(filename))

img = cv2.imread("/home/python/Desktop/opencv_test/opencv_demo/opencv_test1/chinese_test/jpg/car_100ganH3B386.jpg")
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = cv2.resize(img, (20, 20), interpolation=cv2.INTER_AREA)
cv2.imwrite("/home/python/Desktop/opencv_test/opencv_demo/opencv_test1/chinese_test/jpg/car_100ganH3B386_1.jpg", img)


import requests
import pandas as pd
from dingtalkchatbot.chatbot import DingtalkChatbot
from apscheduler.schedulers.background import BackgroundScheduler
import time
import datetime
from config import config

class Bigdata():
    def __init__(self):
        self.path_coin = 'C:\\Users\\Administrator\\Desktop\\research\\10\\rank\\data\\'
        self.path_defi = 'C:\\Users\\Administrator\\Desktop\\research\\10\\rank\\defi\\'
        self.path_concept = 'C:\\Users\\Administrator\\Desktop\\research\\10\\rank\\concept\\'

        self.scheduler = BackgroundScheduler()
        """
        self.header = {'User-Agent' :"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36",
                  "Referer": "https: // www.feixiaohao.cc/",
                  "Sec - Fetch - Site": "cross - site",
                  "Host": "dncapi.bqrank.net",
                  "Origin": "https: // www.feixiaohao.cc",
                  "Referer": "https: // www.feixiaohao.cc/",

                  }
        """
        self.header = {'User-Agent' :"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36"

                  }
        self.dingding = DingtalkChatbot(config.webhook)

    def get_rank_list(self):
        print("get_rank_list")

        #signal = 0
        try:
            res = requests.get(url = 'https://dncapi.soulbab.com/api/coin/web-coinrank?page=1&type=-1&pagesize=100&webp=1', headers = self.header, timeout=10)
            #print(res)
            signal = 1

        except:
            signal = 0

        print("signal: ", signal)
        if signal == 1:
            i = 1
            rank_df = pd.DataFrame(columns=['rank', 'name', 'current_price_usd', 'market_value_usd', 'supply', 'turnoverrate'])
            for coin in res.json()['data']:

                new = pd.DataFrame({'rank': coin['rank'], 'name': coin['name'], 'current_price_usd': coin['current_price_usd'], 'market_value_usd': coin['market_value_usd'], 'supply': coin['supply'], 'turnoverrate': coin['turnoverrate']}, index = [i])
                i+=1
                rank_df = rank_df.append(new, ignore_index=True)
            #print(rank_df)
            rank_df.to_csv(self.path_coin + time.strftime('%Y%m%d') + '.csv')
            #self.dingding.send_text(msg=f"成功下载市值排名数据, {time.strftime('%Y%m%d %H:%M:%S')}", is_at_all=False)

        else:
            print("error in getting rank")
            #self.dingding.send_text(msg=f"下载市值排名数据出现错误, {time.strftime('%Y%m%d %H:%M:%S')}", is_at_all=False)

            #rank_df = 0

    def get_defi_rank_list(self):

        rank_df = pd.DataFrame(columns=['rank', 'code', 'volume', 'changerate', 'percentage', 'typename'])

        try:
            res = requests.get(url = 'https://dncapi.bqrank.net/api/v2/Defi/newlockup/list/page?per_page=100&typeid=1&webp=1', headers = self.header, timeout=10)

            signal = 1
        except:

            signal = 0

        if signal == 1:
            data = res.json()['data']['list']
            rank = 0
            for coin in data[0]['lockupitem']:
        # print(coin)
                new = pd.DataFrame({'rank': rank+1, 'code': coin['code'], 'volume': coin['volume'],
                            'changerate': coin['changerate'], 'percentage': coin['percentage'],
                            'typename': coin['typename']}, index=[rank])
                rank += 1
                rank_df = rank_df.append(new, ignore_index=True)
            rank_df.to_csv(self.path_defi + time.strftime('%Y%m%d')+'.csv')
            self.dingding.send_text(msg=f"成功下载DEFI锁仓量排名数据, {time.strftime('%Y%m%d %H:%M:%S')}", is_at_all=False)

        else:
            print("error in getting defi data")
            #self.dingding.send_text(msg=f"下载DEFI锁仓量排名数据出现错误, {time.strftime('%Y%m%d %H:%M:%S')}", is_at_all=False)

    def get_hot_races(self):

        rank_df = pd.DataFrame(columns=['rank', 'name', 'change'])

        try:
            res = requests.get(url = 'https://dncapi.bqrank.net/api/concept/conceptapplies?pagesize=10&type=1&webp=1', headers = self.header, timeout=10)
            #print(res)
            signal = 1
        except:

            signal = 0
        #print(res.json()['data'])
        if signal == 1:
            print(res.json()['data'])
            message = []
            data = res.json()['data']
            rank = 0
            for concept in data:
        # print(coin)
                new = pd.DataFrame({'rank': rank+1, 'concpet': concept['name'], 'change': concept['change']}, index=[rank])
                message.append((concept['name'], concept['change']))
                rank += 1
                rank_df = rank_df.append(new, ignore_index=True)
            rank_df.to_csv(self.path_concept + time.strftime('%Y%m%d')+'.csv')
            self.dingding.send_text(msg=f"概念板块排名:{message}, {time.strftime('%Y%m%d')}", is_at_all=False)

        else:
            print("error in getting concept data")
        #    self.dingding.send_text(msg=f"下载概念板块排名数据出现错误, {time.strftime('%Y%m%d %H:%M:%S')}", is_at_all=False)

    def get_defi_tvl_rank(self):
        hearder = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",}
        try:
            res = requests.get(url='https://defi-tracker.dappradar.com/api/ethereum/defi/all?currency=USD&limit=25&page=1&sort=tvlInFiat&order=desc',
                           headers=hearder, timeout=10)
            print(res)
        except:
            print("no response")
        #print(res.json())

    def monitor(self):
        self.scheduler.add_job(self.get_rank_list, 'cron', hour='17', minute='19', second='01')
        self.scheduler.add_job(self.get_defi_rank_list, 'cron', hour='17', minute='10', second='01')
        self.scheduler.start()
        while True:
            time.sleep(5)

def func():
    print("yes")

if __name__ == '__main__':
    # 创建后台执行的 schedulers
    bigdata = Bigdata()
    #bigdata.monitor()
    #bigdata.get_hot_races()
    #bigdata.get_defi_chain_rank()
    #bigdata.get_defi_rank_list()
    bigdata.get_rank_list()
    #bigdata.get_defi_tvl_rank()

    #bigdata.get_rank_list()
    #bigdata.get_defi_rank_list()
    #scheduler = BackgroundScheduler()
    #scheduler.add_job(bigdata.get_rank_list, 'cron', hour='17', minute='07', second='01')
    #scheduler.add_job(bigdata.get_rank_list, 'cron', hour='17', minute='11', second='01')
    #scheduler.add_job(bigdata.get_defi_rank_list, 'cron', hour='17', minute='30', second='00')
    #scheduler.start()
    #while True:
    #    time.sleep(5)
#get_rank_list()

#https://dncapi.bqrank.net/api/concept/conceptapplies?pagesize=10&type=1&webp=1


import xlrd
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import linear_model
from sklearn.metrics import  mean_squared_error, r2_score
from mpl_toolkits.mplot3d import axes3d
import numpy as np
data=pd.read_excel('BI_Alumnos08.xlsx')
lista=pd.DataFrame()
alt=data['Altura']
ed=data['Edad']
lista['Altura']=alt
lista['Edad']=ed

XY=np.array(lista)
Z=data['Peso'].values
regLM=linear_model.LinearRegression()
regLM.fit(XY,Z)
z_pred=regLM.predict(XY)
print('Coeficientes de R',regLM.coef_)
print('error cuadrado medio: %.2f '% mean_squared_error(Z,z_pred))
print('puntaje de varianza: %.2f' % r2_score(Z,z_pred))
arreglo=np.array(data['Altura'])
arreglo2=np.array(data['Edad'])

for i in arreglo:
    for o in arreglo2:
        predPeso = regLM.predict([[i, o]])
        print("|  ",i," | ", o," |    ", int(predPeso),"   |")

arreglo3=np.array([int(predPeso)])

# Creamos la figura
fig = plt.figure()
# Agrrgamos un plano 3D
ax1 = fig.add_subplot(111,projection='3d')
ax1.scatter(arreglo3, arreglo2, arreglo, c='g', marker='o')
ax1.set_xlabel('Peso Predictivo')
ax1.set_ylabel('Edad')
ax1.set_zlabel('Altura')
ax1.set_title('Regresión Lineal con Múltiples Variables')
# Mostramos el gráfico
plt.show()


import boto3
import datetime
from datetime import date
import time
import csv
from pprint import pprint
from botocore.exceptions import ClientError

client = boto3.client('iam')
users=client.list_users()['Users']

final_data_from_raw_filter = []
result = []
for user in users:
    # print(user)
    Username=user['UserName']
    AccessKeyList_Username=client.list_access_keys(UserName=Username)
    raw_data=AccessKeyList_Username['AccessKeyMetadata']
    final_data_from_raw_filter.append(raw_data)

#print(final_data_from_raw_filter)

for each_list in final_data_from_raw_filter:
    for each in each_list:
        result.append(each['UserName'])

#print(result)

with open("C:/Temp/iam_tagging.csv", newline='') as csv_file:
    reader = csv.DictReader(csv_file)
    for row in reader:
#        print(row)
        for each_user in users:
            if each_user['UserName'] == row['Iam_Username']:
#                print(row['Iam_Username'], row['E-Mail'], row['E-Mail_Value'], row['ServiceAccount'])
#                Tags_var={row['E-Mail']:row['E-Mail_Value'], row['ServiceAccount']:row['ServiceAccount_Value'], row['Owner']:row['Owner_Value']}
                response = client.tag_user(
                    UserName=row['Iam_Username'], Tags = [
                    {
                        'Key': row['E-Mail'],
                        "Value": row['E-Mail_Value']
                    },
                    {
                        'Key': row['ServiceAccount'],
                        "Value": row['ServiceAccount_Value']
                    },
                    {
                        'Key': row['Owner'],
                        "Value": row['Owner_Value']
                    },
                    {
                        'Key': row['Environment'],
                        "Value": row['Environment_Value']
                    },
                    {
                        'Key': row['EngineerContact'],
                        "Value": row['EngineerContact_Value']
                    },
                    {
                        'Key': row['EngineeringTeam'],
                        "Value": row['EngineeringTeam_Value']
                    }
                ]
                )
                print("Tags Applied for the user account:", row['Iam_Username'])


from machine import I2C, Pin
from pca9685 import PCA9685
from motor import DCMotors
from ibus import IBus
import utime

PCA_ADDR = 0x41
SDA = Pin(0)
SCL = Pin(1)
#Frequency = 500
i2c=machine.I2C(0,sda=SDA, scl=SCL, freq=125000)
ibus_in = IBus(1)
I2C_PCA = I2C(id=0, sda=SDA, scl=SCL)
print("device found at address:", hex(I2C_PCA.scan()[0]))
# PCA = PCA9685(i2c=I2C_PCA,address=PCA_ADDR)
# PCA.freq(50)
Motor = DCMotors(i2c=I2C_PCA, address=PCA_ADDR, freq=1500)
I2C_PCA.writeto_mem(PCA_ADDR, 0x00, b'\x21')
I2C_PCA.writeto_mem(PCA_ADDR, 0x01, b'\x04')
I2C_PCA.writeto_mem(PCA_ADDR, 0xFE, b'\x79')

def map(x, i_min, i_max, o_min, o_max):
    return max(min(o_max, (x - i_min) * (o_max - o_min) // (i_max - i_min) + o_min), o_min)

def convert(X, Y, RL, RR):
    if X or Y:
        w_R = int(55*((-0.5*X)+(0.5*Y)))  # speed of LEFT WHEEL
        w_L = int(55*((-0.5*X)+(-0.5*Y)))  # speed of RIGHT WHEEL
        w_B = int(55*((1*X)+(0*Y)))  # speed of BACK WHEEL

        Motor.speed(0, map(w_L, -4000, 4000, -4095, 4095))  # wheel left
        Motor.speed(1, map(w_R, -4000, 4000, -4095, 4095))  # wheel right
        Motor.speed(2, map(w_B, -4000, 4000, -4095, 4095))  # wheel back
        print("WHEEL L: {}, WHEEL R: {}, WHEEL B: {}".format(w_L, w_R, w_B))
    elif RL==1:
        Motor.speed(0, -1000)  # wheel left
        Motor.speed(1, -1000)  # wheel right
        Motor.speed(2, -1000)  # wheel back
    elif RR==1:
        Motor.speed(0, 1000)  # wheel left
        Motor.speed(1, 1000)  # wheel right
        Motor.speed(2, 1000)  # wheel back
    else:
        Motor.brake(0)
        Motor.brake(1)
        Motor.brake(2)
while True:
    res = ibus_in.read()
    # if signal then display immediately
    if (res[0]):
        x_val = IBus.normalize(res[1])
        y_val = IBus.normalize(res[2])
        rl_val = IBus.normalize(res[5], type="dial")
        rr_val = IBus.normalize(res[8], type="dial")
        # print("Y-AXIS {} X-AXIS {}".format(rl_val,rr_val, end=""))
        convert(x_val, y_val, rl_val, rr_val)
    utime.sleep_ms(10)

Rabbit Robot/Flysky - Pico - PCA9685/Testing/Drive.py


from abc import ABC, abstractmethod
from typing import Optional

import torch
import torchbiggraph.model
from torchbiggraph.operators import AbstractDynamicOperator, AbstractOperator
from torchbiggraph.plugin import PluginRegistry
from torchbiggraph.types import FloatTensorType, LongTensorType

class AbstractRegularizer(ABC):
    """
    Computes a weighted penalty for embeddings involved in score computations.
    """

    def __init__(self, weight):
        self.weight = weight

    @abstractmethod
    def forward_dynamic(
        self,
        src_pos: FloatTensorType,
        dst_pos: FloatTensorType,
        src_operators: Optional[FloatTensorType],
        dst_operators: Optional[FloatTensorType],
    ) -> FloatTensorType:
        pass

    @abstractmethod
    def forward(
        self,
        src_pos: FloatTensorType,
        dst_pos: FloatTensorType,
        src_operators: Optional[FloatTensorType],
        dst_operators: Optional[FloatTensorType],
    ) -> FloatTensorType:
        pass

REGULARIZERS = PluginRegistry[AbstractRegularizer]()

@REGULARIZERS.register_as("N3")
class N3Regularizer(AbstractRegularizer):
    """N3 regularizer described in https://arxiv.org/pdf/1806.07297.pdf"""

    def reg_embs(
        self, src_pos: FloatTensorType, dst_pos: FloatTensorType
    ) -> FloatTensorType:
        a, b, rank = torchbiggraph.model.match_shape(src_pos, -1, -1, -1)
        torchbiggraph.model.match_shape(dst_pos, a, b, rank)
        total = 0
        for x in (src_pos, dst_pos):
            total += torch.sum(self.modulus(x, rank // 2) ** 3)
        return total * self.weight

    def forward_dynamic(
        self,
        src_pos: FloatTensorType,
        dst_pos: FloatTensorType,
        operator: AbstractDynamicOperator,
        rel_idxs: LongTensorType,
    ) -> FloatTensorType:
        total = 0
        operator_params = operator.get_operator_params_for_reg(rel_idxs)
        if operator_params is not None:
            operator_params = operator_params.to(src_pos.device)
            total += torch.sum(operator_params**3)
        for x in (src_pos, dst_pos):
            total += torch.sum(operator.prepare_embs_for_reg(x) ** 3)
        total *= self.weight
        return total

    def forward(
        self,
        src_pos: FloatTensorType,
        dst_pos: FloatTensorType,
        operator: AbstractOperator,
    ) -> FloatTensorType:
        total = 0
        operator_params = operator.get_operator_params_for_reg()
        if operator_params is not None:
            operator_params = operator_params.to(src_pos.device)
            batch_size = len(src_pos)
            total += torch.sum(operator_params**3) * batch_size
        for x in (src_pos, dst_pos):
            total += torch.sum(operator.prepare_embs_for_reg(x) ** 3)
        total *= self.weight
        return total


from .plugin import CredentialPlugin

from django.conf import settings
from django.utils.translation import gettext_lazy as _
from delinea.secrets.vault import PasswordGrantAuthorizer, SecretsVault

dsv_inputs = {
    'fields': [
        {
            'id': 'tenant',
            'label': _('Tenant'),
            'help_text': _('The tenant e.g. "ex" when the URL is https://ex.secretsvaultcloud.com'),
            'type': 'string',
        },
        {
            'id': 'tld',
            'label': _('Top-level Domain (TLD)'),
            'help_text': _('The TLD of the tenant e.g. "com" when the URL is https://ex.secretsvaultcloud.com'),
            'choices': ['ca', 'com', 'com.au', 'eu'],
            'default': 'com',
        },
        {
            'id': 'client_id',
            'label': _('Client ID'),
            'type': 'string',
        },
        {
            'id': 'client_secret',
            'label': _('Client Secret'),
            'type': 'string',
            'secret': True,
        },
    ],
    'metadata': [
        {
            'id': 'path',
            'label': _('Secret Path'),
            'type': 'string',
            'help_text': _('The secret path e.g. /test/secret1'),
        },
        {
            'id': 'secret_field',
            'label': _('Secret Field'),
            'help_text': _('The field to extract from the secret'),
            'type': 'string',
        },
    ],
    'required': ['tenant', 'client_id', 'client_secret', 'path', 'secret_field'],
}

if settings.DEBUG:
    dsv_inputs['fields'].append(
        {
            'id': 'url_template',
            'label': _('URL template'),
            'type': 'string',
            'default': 'https://{}.secretsvaultcloud.{}',
        }
    )

def dsv_backend(**kwargs):
    tenant_name = kwargs['tenant']
    tenant_tld = kwargs.get('tld', 'com')
    tenant_url_template = kwargs.get('url_template', 'https://{}.secretsvaultcloud.{}')
    client_id = kwargs['client_id']
    client_secret = kwargs['client_secret']
    secret_path = kwargs['path']
    secret_field = kwargs['secret_field']

    tenant_url = tenant_url_template.format(tenant_name, tenant_tld.strip("."))

    authorizer = PasswordGrantAuthorizer(tenant_url, client_id, client_secret)
    dsv_secret = SecretsVault(tenant_url, authorizer).get_secret(secret_path)

    return dsv_secret['data'][secret_field]

dsv_plugin = CredentialPlugin(name='Thycotic DevOps Secrets Vault', inputs=dsv_inputs, backend=dsv_backend)