import os, sys
import random
import time

import argparse
import numpy as np
import cv2
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim

from torch.utils.data import DataLoader

from helpers import Logger
from dataset import GLAS_dataloader, POLYPS_dataloader
from metrics import calculate_metric_percase
from models.LeViTUNet384 import Build_LeViT_UNet_384

"""Train and evaluate masksup on GLaS or Kvasir & CVC-ClinicDB datasets"""

########## Reproducibility ##########

SEED = 0
random.seed(SEED)
os.environ["PYTHONHASHSEED"] = str(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)
if torch.cuda.is_available():
    torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True

########## Get Args ##########

parser = argparse.ArgumentParser(description="settings")
# configuration
parser.add_argument("--exp_name", default="baseline")
# dataset
parser.add_argument("--dataset", default="isic2018", type=str)
# model
parser.add_argument("--alpha", default=1, type=float)
parser.add_argument("--beta", default=1, type=float)
parser.add_argument("--gamma", default=1, type=float)
args = parser.parse_args()

print(args)

########## Setup ##########

# Device
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(DEVICE)

# Log folder
# EXPERIMENT_NAME = args.exp_name+"_"+"a"+str(args.alpha)+"b"+str(args.beta)+"g"+str(args.gamma)+"_"+args.dataset #"levit192_isic2018"
EXPERIMENT_NAME = "glas_reproduce"

ROOT_DIR = os.path.abspath(".")
LOG_PATH = os.path.join(ROOT_DIR, "logs", EXPERIMENT_NAME)

if not os.path.exists(os.path.join(ROOT_DIR, "logs")):
    os.mkdir(os.path.join(ROOT_DIR, "logs"))

if not os.path.exists(LOG_PATH):
    os.mkdir(LOG_PATH)
    os.mkdir(os.path.join(LOG_PATH, "samples"))
    os.mkdir(os.path.join(LOG_PATH, "samples", "masked_imgs_cb"))
    os.mkdir(os.path.join(LOG_PATH, "samples", "masked_preds_cb"))
    os.mkdir(os.path.join(LOG_PATH, "samples", "masked_imgs_cb_ts"))
    os.mkdir(os.path.join(LOG_PATH, "samples", "masked_preds_cb_ts"))

# save config in log file
sys.stdout = Logger(os.path.join(LOG_PATH, "log_train.txt"))

########## Load data ##########

if "glas" in EXPERIMENT_NAME:
    print("Loading GLAS dataset")
    train_dataset = GLAS_dataloader("datasets/GLAS")
    test_dataset = GLAS_dataloader("datasets/GLAS", is_train=False)
else:
    print("Loading POLYP dataset")
    train_dataset = POLYPS_dataloader("datasets/POLYPS")
    test_dataset = POLYPS_dataloader("datasets/POLYPS", is_train=False)

train_dataloader = DataLoader(
    train_dataset, batch_size=8, shuffle=True, num_workers=8
)  # 8
test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=8)

print("Training on {} batches/samples".format(len(train_dataloader)))
print("Testing on {} batches/samples".format(len(test_dataloader)))

dt = next(iter(train_dataloader))
x = dt["image"]
y = dt["mask"]

print("Sample: ", x[0][:, :10][0][0][:3])

########## Get model ##########

# Define model
model = Build_LeViT_UNet_384(num_classes=1, pretrained=True)

# Send to GPU
model = model.to(DEVICE)
print(model)

# All parameters
all_params = sum(p.numel() for p in model.parameters())
print("All parameters ", all_params)

# Trainable parameters
all_train_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print("Trainable parameters ", all_train_params)

########## Setup optimizer and loss and loss scaling for AMP ##########

optimizer = optim.Adam(
    model.parameters(), lr=1e-4, weight_decay=1e-5, amsgrad=True
)  # prev 1e-4
amp = True
grad_scaler = torch.cuda.amp.GradScaler(enabled=amp)

criterion = (
    nn.BCEWithLogitsLoss()
)  # loss combines a Sigmoid layer and the BCELoss in one single class
criterion_mse = nn.MSELoss()

########## Trainer and validation functions ##########

def train(model, epoch):
    """
    Trains a segmentation model.
    """
    print("Trains a segmentation model.")

    assert model is not None, f"Should be a PyTorch model, got: {model}"

    model.train()
    for batch_idx, data in enumerate(train_dataloader):
        data1, data2, target = (
            data["image"].to(DEVICE),
            data["partial_image1"].to(DEVICE),
            data["mask"].to(DEVICE),
        )
        output1 = model.forward(data1.float())

        # Compute loss based on two outputs
        loss = criterion(output1.float(), target.float())

        # Update
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

def train_context_branch(model, epoch, save_masks=True):
    """
    Trains a segmentation model using context branch in a siamese style.
    """

    print("Trains a segmentation model using context branch in a siamese style.")

    assert model is not None, f"Should be a PyTorch model, got: {model}"

    model.train()
    for batch_idx, data in enumerate(train_dataloader):
        data1, data2, target = (
            data["image"].to(DEVICE),
            data["partial_image1"].to(DEVICE),
            data["mask"].to(DEVICE),
        )

        # Save masked image
        if save_masks:
            masked_img = data2[0]
            masked_img = (masked_img.permute(1, 2, 0).detach().cpu().numpy() + 1) / 2
            masked_img = (masked_img * 255).astype(np.uint8)
            masked_img = cv2.cvtColor(masked_img, cv2.COLOR_RGB2BGR)
            # masked_img[masked_img==127] = 0
            cv2.imwrite(
                "{}/samples/masked_imgs_cb/ep{}_b{}.png".format(
                    LOG_PATH, epoch, batch_idx
                ),
                masked_img,
            )

        # Make predictions
        output1 = model.forward(data1.float())
        output2 = model.forward(data2.float())

        # Compute loss based on two outputs
        loss1 = criterion(output1.float(), target.float())
        loss2 = criterion(output2.float(), target.float())

        # Loss coefficients
        alpha = 1
        beta = 1

        # Total loss
        loss = alpha * loss1 + beta * loss2

        # Update
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    print("With CB: ", alpha, beta)

def train_context_branch_with_task_sim(model, epoch, save_masks=True):
    """
    Trains a segmentation model using context branch (CB) and task similarity (TS) constraint.
    """

    print(
        "Trains a segmentation model using context branch (CB) and task similarity (TS) constraint."
    )

    model.train()
    for batch_idx, data in enumerate(train_dataloader):
        data1, data2, target = (
            data["image"].to(DEVICE),
            data["partial_image1"].to(DEVICE),
            data["mask"].to(DEVICE),
        )

        # Save masked image
        if save_masks:
            masked_img = data2[0]
            masked_img = (masked_img.permute(1, 2, 0).detach().cpu().numpy() + 1) / 2
            masked_img = (masked_img * 255).astype(np.uint8)
            masked_img = cv2.cvtColor(masked_img, cv2.COLOR_RGB2BGR)
            # masked_img[masked_img==127] = 0
            cv2.imwrite(
                "{}/samples/masked_imgs_cb_ts/ep{}_b{}.png".format(
                    LOG_PATH, epoch, batch_idx
                ),
                masked_img,
            )

        with torch.cuda.amp.autocast(enabled=amp):
            # Make predictions
            output1 = model.forward(data1.float())
            output2 = model.forward(data2.float())

            # Compute loss based on two outputs, and maximize similarity
            loss1 = criterion(output1.float(), target.float())
            loss2 = criterion(output2.float(), target.float())
            loss3 = criterion_mse(
                torch.sigmoid(output1.float()), torch.sigmoid(output2.float())
            )

        # Loss coefficients
        alpha = 1
        beta = 1
        gamma = 1

        # Total loss
        loss = alpha * loss1 + beta * loss2 + gamma * loss3

        # Update
        optimizer.zero_grad(set_to_none=True)
        grad_scaler.scale(loss).backward()
        grad_scaler.step(optimizer)
        grad_scaler.update()

    print("With CB + TS: ", alpha, beta, gamma)

def test(model):
    model.eval()

    with torch.no_grad():
        test_loss = 0
        jaccard = 0
        dice = 0

        for data in test_dataloader:
            data, target = data["image"].to(DEVICE), data["mask"].to(DEVICE)
            output = model(data.float())
            test_loss += criterion(output.float(), target.float()).item()
            dc, jc, _ = calculate_metric_percase(output, target)
            jaccard += jc
            dice += dc
        test_loss /= len(test_dataloader)
        jaccard /= len(test_dataloader)
        dice /= len(test_dataloader)

        losses.append(test_loss)
        jacs.append(jaccard)
        dices.append(dice)

        print("Average Loss: {:.3f}".format(test_loss))
        print("Jaccard Index / IoU : {:.3f}".format(jaccard * 100))
        print("Dice Coefficient / F1 : {:.3f}".format(dice * 100))
        print("==========================================")
        print("==========================================")
        return jaccard

########## Train and validate ##########

losses = []
jacs = []
dices = []
score = 0
best_score = 0

start_time = time.time()
N_EPOCHS = 200
for epoch in range(1, N_EPOCHS):
    # Train and eval
    print("Epoch: {}".format(epoch))

    ########## Trainer type ##########
    # 1. Base
    # train(model, epoch)
    # 2. Base with context branch
    # train_context_branch(model, epoch)
    # 3. MaskSup
    train_context_branch_with_task_sim(model, epoch)

    score = test(model)

    if score > best_score:
        # Save predictions
        if not os.path.exists(os.path.join(LOG_PATH, "vis")):
            os.mkdir(os.path.join(LOG_PATH, "vis"))
            os.mkdir(os.path.join(LOG_PATH, "vis", "imgs"))
            os.mkdir(os.path.join(LOG_PATH, "vis", "gts"))
            os.mkdir(os.path.join(LOG_PATH, "vis", "preds"))

        for batch_idx, data in enumerate(test_dataloader):
            img, target = data["image"].to(DEVICE), data["mask"].to(DEVICE)
            output = torch.sigmoid(model(img.float()))

            img = (img[0].permute(1, 2, 0).detach().cpu().numpy() + 1) / 2
            img = (img * 255).astype(np.uint8)
            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)

            gt = target.permute(0, 2, 3, 1).squeeze().detach().cpu().numpy()
            gt = (gt * 255).astype(np.uint8)
            gt = cv2.cvtColor(gt, cv2.COLOR_RGB2BGR)

            pred = output.permute(0, 2, 3, 1).squeeze().detach().cpu().numpy() > 0.5
            pred = (pred * 255).astype(np.uint8)
            pred = cv2.cvtColor(pred, cv2.COLOR_RGB2BGR)

            cv2.imwrite(
                os.path.join(LOG_PATH, "vis", "imgs/") + str(batch_idx) + ".png", img
            )
            cv2.imwrite(
                os.path.join(LOG_PATH, "vis", "gts/") + str(batch_idx) + ".png", gt
            )
            cv2.imwrite(
                os.path.join(LOG_PATH, "vis", "preds/") + str(batch_idx) + ".png", pred
            )

        # Save model
        print("########Saving model at IoU/Jaccard={:.3f}########".format(score))
        torch.save(model.state_dict(), "{}/{}.pth".format(LOG_PATH, EXPERIMENT_NAME))
        best_score = score

end_time = time.time()
print("--- Time taken to train : %s hours ---" % ((end_time - start_time) // 3600))
print("--- Time taken to train : %s mins ---" % ((end_time - start_time) // 60))

print("Max Jaccard/IoU and Dice/F1 scores: ", max(jacs), " and ", max(dices))

########## Save logs ##########

# Save losses
losses = np.array(losses)
np.savetxt("{}/{}_loss.txt".format(LOG_PATH, EXPERIMENT_NAME), losses, delimiter=",")
jacs = np.array(jacs)
np.savetxt("{}/{}_jacs.txt".format(LOG_PATH, EXPERIMENT_NAME), jacs, delimiter=",")
dices = np.array(dices)
np.savetxt("{}/{}_dices.txt".format(LOG_PATH, EXPERIMENT_NAME), dices, delimiter=",")

report = {}

report["Max Jaccard = "] = "{:.5f}".format(max(jacs))
report["Max Dice = "] = "{:.5f}".format(max(dices))

with open("{}/{}_bests.txt".format(LOG_PATH, EXPERIMENT_NAME), "w") as f:
    for k, v in report.items():
        f.write(str(k))
        # f.write("--->")
        f.write(str(v))
        # new line
        f.write("\n")
f.close()

########## Plot curves ##########

# b, g, r, y, o, -g, -m,
plt.figure(figsize=(15, 5))
plt.subplot(121)
plt.plot(losses, linewidth=4)
plt.title("{} loss".format("Exp name"))
# plt.ylabel('Loss')
plt.xlabel("Epoch")
plt.legend(["loss"], loc="upper left")
plt.grid(True)
# Plot training & validation iou_score values
plt.subplot(122)
plt.plot(jacs, linewidth=4)
plt.plot(dices, linewidth=4)
# plt.title('{} IOU score'.format(experiment_name))
# plt.ylabel('iou_score')
plt.xlabel("Epoch")
plt.grid(True)
plt.legend(["Jaccard", "Dice"], loc="upper left")
plt.savefig("{}/{}_graph.png".format(LOG_PATH, EXPERIMENT_NAME), dpi=300)
# plt.show()


"""
This is a set of utility functions for downloading and manipulating video/audio
"""
from pytube import YouTube
import skvideo.io  #requires 'brew install ffmpeg'
import torch
import librosa
import numpy as np
import os
from progressbar import ProgressBar
import json

def download_videos(URLs, DEST_PATH):
    """
    This function downloads the video given by URLs passed from YouTube.

    Params:
        -URLs: list of URLs to download
        -DEST_PATH: destination path for downloaded videos

    Returns:
        -Nothing
    """
    pbar = ProgressBar() # progress bar for downloading videos

    for url in pbar(URLs):
        try:
            yt = YouTube(url)
        except:
            print("Could not connect to YouTube...")

        # filter files for 'mp4' file type and get first file
        new_vid = yt.streams.filter(subtype='mp4').first()
        try:
            new_vid.download(DEST_PATH)
        except:
            print("Error downloading video...")

def video_to_array(VIDEO_FILE):
    """
    This function converts a mp4 file into a numpy array (takes a while)

    Params:
        -mp4 file

    Returns:
        -numpy array
    """
    return skvideo.io.vread(VIDEO_FILE)

def video_to_wave(VIDEO_FILE, timesteps):
    """
    This function returns a numpy array of sound bytes from a video
    """

def get_metadata(VIDEO_FILE):
    """
    This function returns a dict of the metadata for the mp4 file passed in.

    Params:
        -VIDEO_FILE: path to mp4 video

    Returns:
        -dict of metadata from file
    """
    return skvideo.io.ffprobe(video_location)

def print_metadata(metadata, mode='video'):
    """
    This function takes a dict of metadata and prints it to the terminal.

    Params:
        -metadata: dict of metadata to be printed
        -mode: type of metadata (either video or audio)

    Returns:
        -Nothing
    """
    if mode == 'video' or mode == 'audio':
        print("\n" + "---------- " + mode + " metadata ----------" + "\n")
        print(json.dumps(metadata[mode], indent=4))
        print("\n" + "---------- " + mode + " metadata ----------" + "\n")
    else:
        print("please enter valid mode (\"audio\" or \"video\")")

if __name__ == "__main__":

    destination = os.path.dirname(os.path.abspath(__file__))

    videos = ["https://www.youtube.com/watch?v=ToSe_CUG0F4"]

    # uncomment for downloading video
    # download_videos(videos, destination)

    video_location = "" # path to video
    print_metadata(get_metadata(video_location), "audio")

    #convert mp4 to array
    #video_array = video_to_array(video_location)
    #sound_bytes = get_sound_bytes(video_location)

    #print(type(video_array))
    #print(video_array.shape)


import numpy as np
import os
import pickle
import meshio
import sys
sys.path.append("../")
from utils.stress_utils import *
from utils.constants import OBJECT_NAMES

dgn_dataset_path = "/home/baothach/stress_field_prediction/sim_data/stress_prediction_data/dgn_dataset"
adjacent_tetrahedrals_save_path = "/home/baothach/shape_servo_data/stress_field_prediction/adjacent_tetrahedrals"
data_recording_path = "/home/baothach/shape_servo_data/stress_field_prediction/data"
os.makedirs(adjacent_tetrahedrals_save_path, exist_ok=True)

for object_name in OBJECT_NAMES:
# for object_name in ["strawberry"]:
# for i, object_name in enumerate(sorted(os.listdir(dgn_dataset_path))):
    file_name = os.path.join(data_recording_path, f"{object_name}_grasp_{0}_force_{1.0}.pickle")
    if not os.path.isfile(file_name):
        print(f"{file_name} not found")
        break

    with open(file_name, 'rb') as handle:
        data = pickle.load(handle)
        (tet_indices, tet_stress) = data["tet"]
        tet_indices = np.array(tet_indices).reshape(-1,4)

    save_data = get_adjacent_tetrahedrals_of_vertex(tet_indices)
    with open(os.path.join(adjacent_tetrahedrals_save_path, f"{object_name}.pickle"), 'wb') as handle:
        pickle.dump(save_data, handle, protocol=pickle.HIGHEST_PROTOCOL)


import sys
from collections import deque

sys.setrecursionlimit(10**6)

n = int(input())

dist = [-1] * (n+1)
check = [0] * (n+1) # 0 : not visited, 1: visited, 2: cycle
adj_list = [[] for _ in range(n+1)]

for i in range(1, n+1):
  u, v = map(int, input().split())
  adj_list[u].append(v)
  adj_list[v].append(u)

for i in range(1, n+1):
  adj_list[i].sort()

# p -> x
def find_cycle(x, p):
  # -2 : found cycle and not included
  # -1 : not a cycle
  # n : found cycle and start index (>=0)

  if check[x] == 1:
    # cycle
    return x

  check[x] = 1
  for y in adj_list[x]:
    if y == p:
      # previous node
      continue

    res = find_cycle(y, x)

    if res == -2:
      return -2
    if res >= 0:
      check[x] = 2
      if x == res:
        return -2
      else:
        return res
  return -1

def calc_dist():
  q = deque()

  for i in range(1, n+1):
    if check[i] == 2:
      q.append(i)
      dist[i] = 0
    else:
      dist[i] = -1

  while q:
    x = q.popleft()
    for y in adj_list[x]:
      if dist[y] == -1:
        dist[y] = dist[x] + 1
        q.append(y)

  for i in range(1, n+1):
    print(dist[i], end=' ')
  print()

find_cycle(1, 0)
calc_dist()


import os
from enum import Enum
from typing import Optional, Any

from .utils.project_models import LogEvent, StepMetadata, Status, LogLevel

# class mimicking Either behaviour to capture both success and failure
class Either:

    def __init__(self, left=None, right=None):
        self.left = left
        self.right = right

    @property
    def is_left(self):
        return self.left is not None

    @property
    def is_right(self) -> bool:
        return self.right is not None

def custom_print(message: Optional[Any] = None, exception=None,
                 step_id=None,
                 step_metadata: Optional[StepMetadata] = None,
                 step_status: Optional[Status] = None, level: LogLevel = LogLevel.INFO):
    if is_online_mode():
        # Custom print: Print all variables.
        if step_metadata is not None:
            log_event = LogEvent.from_step_metadata(step_metadata)
        elif step_status is not None:
            log_event = LogEvent.from_status(step_status, step_id)
        else:
            log_event = LogEvent.from_log(step_id, message, exception)

        print(log_event.to_json(), flush=True)
    else:
        # Regular print: Skip stepName.
        if exception is not None:
            print(message, exception)
        else:
            if message is not None:
                print(message)

def is_online_mode() -> bool:
    return os.environ.get("PRINT_MODE", "REGULAR") == "CUSTOM"

# If the item is a dictionary

def remove_null_items_recursively(item):
    # If the item is a dictionary
    if isinstance(item, dict):
        return {
            k: remove_null_items_recursively(v)
            for k, v in item.items() if v is not None
        }

    # If the item is a list
    elif isinstance(item, list):
        return [remove_null_items_recursively(v) for v in item if v is not None]

    # If the item is an enum
    elif isinstance(item, Enum):
        return item.value

    # If the item is neither a dictionary, a list, nor an enum
    else:
        return item


import bs4
import os.path
import pdfkit
import pytz
import requests
import tempfile
import urllib.parse
from bs4 import BeautifulSoup
from datetime import datetime, timezone, timedelta

from concierge.logger import get_logger
from concierge.scraper.common import Figure, template_path, template_loader, title_normalizer, render_option_us_letter

class AsahiEditorial:
    def __init__(self, url: str, article_id: str, article_date: datetime.date,
                 title: str, body: list, figure: Figure):
        self.logger = get_logger(__name__)
        self.type = 'editorial'
        self.lang = 'ja'
        self.url = url
        self.id = article_id
        self.date = article_date
        self.title = title.strip()
        self.filename = '{1}_{0}'.format(
            title_normalizer(self.title).replace('（社説）', '').replace(' ', '-').replace('　', '-'), self.date)
        self.body = body
        self.figure = figure
        self.template_path = template_path
        self.template = template_loader.get_template('asahi-editorial.html')
        self.temp_output = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
        self.temp_image = tempfile.NamedTemporaryFile()
        if self.figure.image:
            self.temp_image.write(self.figure.image)
        self.temp_html = self._render_html()

    def _render_html(self):
        return self.template.render(
            template_path=self.template_path,
            lang=self.lang,
            title=self.title,
            reference_url=self.url,
            article_id=self.id,
            date=self.date,
            article=self.body,
            figure_img=self.temp_image.name,
            figure_cap=self.figure.caption
        )

    def _render_us_letter(self):
        """
        Render US Letter PDF from html
        (for Fujitsu QAUDERNO)

        :return:
        """
        self.temp_output.write(pdfkit.from_string(self.temp_html, output_path=False, options=render_option_us_letter))
        self.temp_output.close()

    def develop(self):
        self.logger.info('develop editorial...')
        self._render_us_letter()
        return {'path': self.temp_output.name, 'category': 'asahi', 'filename': self.filename, 'file_ext': '.pdf'}

    def clear(self):
        self.temp_image.close()

def _extract_article_id(url: str) -> str:
    """

    :param url:
    :return:
    """
    _path = urllib.parse.urlparse(url).path
    editorial_id, extension = os.path.splitext(os.path.basename(_path))
    return editorial_id

class AsahiScraper:
    def __init__(self, pdf_format: str):
        self.logger = get_logger(__name__)
        self.pdf_format = pdf_format
        self.timezone = pytz.timezone('Asia/Tokyo')
        self.today = datetime.now(tz=self.timezone)
        self._result = {'editorial': []}
        self._asahi_news_url = 'https://www.asahi.com'
        self.asahi_editorial_view_url = self._asahi_news_url + '/articles/{0}.html?iref=pc_rensai_long_16_article'
        self.asahi_editorials_url = 'https://www.asahicom.jp/rensai/json/da16.json'

    def _fetch_editorial_list(self) -> list:
        """
        Fetch editorial list by current date.

        :return: List of editorial article path.
        """
        editorials = []
        r = requests.get(self.asahi_editorials_url, headers={'referer': self._asahi_news_url})
        for article in r.json()['items']:
            if datetime.strptime(article['release_date'], '%Y%m%d%H%M%S').date() >= self.today.date():
                editorials.append(article['id'])
        return editorials

    def _fetch_editorial_page(self, url: str) -> bs4.element.Tag:
        """

        :param url: Url for fetch.
        :return:
        """
        r = requests.get(url)
        parse = BeautifulSoup(r.content, features='html.parser').find('main')
        return parse

    def _extract_editorial_title(self, contents: bs4.element.Tag) -> str:
        """

        :param contents:
        :return:
        """
        headers = contents.find('h1')
        return headers.text

    def _extract_editorial_figure(self, contents: bs4.element.Tag) -> Figure:
        """

        :param contents:
        :return:
        """
        image = b''
        figure = contents.find('figure')
        caption = figure.find('figcaption').text
        image_url = figure.find('img').get('src')
        if image_url.startswith('//'):
            image_url = 'https:' + image_url
        try:
            r = requests.get(image_url)
            if r.ok:
                image = r.content
        except ConnectionError:
            pass
        return Figure(image=image, caption=caption)

    def _extract_editorial_body(self, contents: bs4.element.Tag) -> list:
        """

        :param contents:
        :return:
        """
        body = []
        content = contents.find_all('p')
        for p in content:
            if p.text and not p.text.endswith('分'):
                body.append(p.text)
        return body

    def _fetch_editorial(self, page_id: str) -> AsahiEditorial:
        """

        :param url:
        :return:
        """
        self.logger.info('fetch editorial...')
        url = self.asahi_editorial_view_url.format(page_id)
        article_id = _extract_article_id(url)
        contents = self._fetch_editorial_page(url)
        title = self._extract_editorial_title(contents)
        figure = self._extract_editorial_figure(contents)
        body = self._extract_editorial_body(contents)
        return AsahiEditorial(url=url, article_id=article_id, article_date=self.today.date(),
                              title=title, body=body, figure=figure)

    def _push_to_result(self, payload) -> None:
        """

        :param payload:
        :return:
        """
        self.logger.info('push to result article {0}'.format(payload.id))
        result = payload.develop()
        if result:
            self._result[payload.type].append(result)
        payload.clear()

    def download_editorials(self) -> dict:
        """

        :return:
        """
        self.logger.info('download editorials from Asahi...')
        articles = self._fetch_editorial_list()
        if articles:
            for a in articles:
                self._push_to_result(self._fetch_editorial(a))
        return self._result


# RNN Model 2 - LSTM
from keras.models import Model
from keras.layers import Input, LSTM

# Load SVG and other functions
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot

import os
os.environ["PATH"] += os.pathsep + 'C:/Users/Patrick/.conda/pkgs/graphviz-2.38.0-0/Library/bin/'

# Set unit, step, vector and data shape
units = 10
time_steps = 5
input_dim = 15
input_shape = (time_steps, input_dim)

# Create Recurrent Neural Network
x = Input(shape=input_shape, name='Input')
y = LSTM(units=units, activation='sigmoid', name='LSTM_1')(x)
model = Model(inputs=[x], outputs=[y])

# Use SVG to display
SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg'))

# Use Recurrent Neural Network model to ouput
y = LSTM(units=units, activation='sigmoid', return_sequences=True, name='LSTM_1')(x)
model = Model(inputs=[x], outputs=[y])

# Use SVG to display
SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg'))

# Create Recurrent Neural Network model and output status
x = Input(shape=input_shape, name='Input')
y, state_1, state_2 = LSTM(units=units, activation='sigmoid', return_state=True, name='LSTM_1')(x)
model = Model(inputs=[x], outputs=[y])

# Use SVG format to display
SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg'))

Basic for Deep Learning Book 1/RNN+Plots-LSTM.py


from typing import List, Union
from numbers import Number, Integral
import random
from math import ceil, log2
import datetime

from msdsl.generator.generator import CodeGenerator
from msdsl.expr.expr import ModelExpr, wrap_constant, ModelOperator, Constant, \
    ArithmeticOperator, ComparisonOperator, BitwiseOperator, Concatenate, Array, \
    TypeConversion, SIntToReal, UIntToSInt, BitwiseInv, ArithmeticShift, \
    BitwiseAccess, RealToSInt, SIntToUInt, BitwiseAnd, BitwiseOr, BitwiseXor, \
    ArithmeticRightShift, ArithmeticLeftShift, LessThan, LessThanOrEquals, \
    GreaterThan, GreaterThanOrEquals, EqualTo, NotEqualTo, Sum, Product, Min, \
    Max, CompressUInt, RandomInteger, MT19937, LCG
from msdsl.expr.table import Table, RealTable, UIntTable, SIntTable
from msdsl.expr.format import UIntFormat, SIntFormat, RealFormat, IntFormat
from msdsl.expr.signals import Signal, AnalogSignal, DigitalSignal, AnalogInput, AnalogOutput, DigitalOutput, \
    DigitalInput, DigitalParameter, RealParameter
from msdsl.generator.tree_op import tree_op
from msdsl.generator.svreal import compile_range_expr, compile_width_expr, compile_exponent_expr
from msdsl.expr.analyze import signal_names, signal_name
from msdsl.generator.case_statement import case_statment

BITWISE_OP = {
    BitwiseAnd: '&',
    BitwiseOr: '|',
    BitwiseXor: '^'
}

SHIFT_OP = {
    ArithmeticLeftShift: '<<<',
    ArithmeticRightShift: '>>>'
}

INT_COMP_OP = {
    LessThan: '<',
    LessThanOrEquals: '<=',
    GreaterThan: '>',
    GreaterThanOrEquals: '>=',
    EqualTo: '==',
    NotEqualTo: '!='
}

REAL_COMP_OP = {
    LessThan: 'LT_REAL',
    LessThanOrEquals: 'LE_REAL',
    GreaterThan: 'GT_REAL',
    GreaterThanOrEquals: 'GE_REAL',
    EqualTo: 'EQ_REAL',
    NotEqualTo: 'NE_REAL'
}

REAL_ARITH_OP = {
    Sum: 'ADD_REAL',
    Product: 'MUL_REAL',
    Min: 'MIN_REAL',
    Max: 'MAX_REAL'
}

INT_ARITH_OP = {
    Sum:     lambda a, b: f'({a}+{b})',
    Product: lambda a, b: f'({a}*{b})',
    Min:     lambda a, b: f'(({a} < {b}) ? {a} : {b})',
    Max:     lambda a, b: f'(({a} > {b}) ? {a} : {b})'
}

class VerilogGenerator(CodeGenerator):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.init_file()

    def make_section(self, label):
        self.comment(label)

    def expr_to_signal(self, expr: ModelExpr):
        # before starting, make sure that the expression is wrapped in case it is a number
        expr = wrap_constant(expr)

        if isinstance(expr, Signal):
            return expr
        elif isinstance(expr, Constant):
            return self.make_constant(expr)
        elif isinstance(expr, ArithmeticOperator):
            return self.make_arithmetic_operator(expr)
        elif isinstance(expr, CompressUInt):
            return self.make_compress_uint(expr)
        elif isinstance(expr, RandomInteger):
            return self.make_random_integer(expr)
        elif isinstance(expr, BitwiseInv):
            return self.make_bitwise_inv(expr)
        elif isinstance(expr, BitwiseOperator):
            return self.make_bitwise_operator(expr)
        elif isinstance(expr, ComparisonOperator):
            return self.make_comparison_operator(expr)
        elif isinstance(expr, Concatenate):
            return self.make_concatenation(expr)
        elif isinstance(expr, Array):
            return self.make_array(expr)
        elif isinstance(expr, ArithmeticShift):
            return self.make_arithmetic_shift(expr)
        elif isinstance(expr, BitwiseAccess):
            return self.make_bitwise_access(expr)
        elif isinstance(expr, TypeConversion):
            return self.make_type_conversion(expr)
        else:
            raise Exception(f'Unknown expression type: {expr.__class__.__name__}')

    def make_signal(self, signal: Signal):
        if isinstance(signal.format_, RealFormat):
            # compile the range, width, and exponent expressions
            range = compile_range_expr(signal.format_.range_)
            width = compile_width_expr(signal.format_.width)
            exponent = compile_exponent_expr(signal.format_.exponent)

            # call the appropriate macro
            if width is None:
                self.macro_call('MAKE_REAL', signal.name, range)
            elif exponent is None:
                self.macro_call('MAKE_GENERIC_REAL', signal.name, range, width)
            else:
                self.macro_call('MAKE_FORMAT_REAL', signal.name, range, width, exponent)
        elif isinstance(signal.format_, IntFormat):
            self.writeln(f'{self.int_type_str(signal.format_)} {signal.name};')
        else:
            raise Exception('Invalid signal type.')

    def make_probe(self, s: Signal):
        if isinstance(s, Signal) and isinstance(s.format_, RealFormat):
            self.macro_call('PROBE_ANALOG', s.name)
        elif isinstance(s, Signal) and isinstance(s.format_, IntFormat):
            self.macro_call('PROBE_DIGITAL', s.name, str(s.format_.width))
        else:
            raise Exception('Invalid signal type.')

    def make_assign(self, input_: Signal, output: Signal, check_format=True):
        if isinstance(input_.format_, RealFormat) and isinstance(output.format_, RealFormat):
            self.macro_call('ASSIGN_REAL', input_.name, output.name)
        elif ((not check_format) or
              ((isinstance(input_.format_, SIntFormat) and isinstance(output.format_, SIntFormat)) or
               (isinstance(input_.format_, UIntFormat) and isinstance(output.format_, UIntFormat)))):
            self.digital_assignment(signal=output, value=input_.name)
        else:
            raise Exception(f'Input and output formats do not match: {input_.name} with {input_.format_} vs. {output.name} with {output.format_}')

    def make_mem(self, next_: Signal, curr: Signal, init: Union[Number, DigitalParameter, RealParameter]=0,
                 clk: Signal=None, rst: Signal=None, ce: Signal = None, check_format=True):
        # set defaults
        clk_name = clk.name if clk is not None else '`CLK_MSDSL'

        # determine name of reset signal
        if rst is None:
            rst_name = '`RST_MSDSL'
        elif isinstance(rst, str):
            rst_name = rst
        else:
            rst_name = rst.name

        ce_name = ce.name if ce is not None else "1'b1"

        # create memory for real number
        if isinstance(next_.format_, RealFormat) and isinstance(curr.format_, RealFormat):
            # determine string expression for the initial value
            if isinstance(init, Number):
                init_str = str(init)
            elif isinstance(init, RealParameter):
                init_str = init.param_name
            elif isinstance(init, str):
                init_str = init
            else:
                raise Exception(f'Could not determine string representation for initial value {init}')

            # call the macro to create the memory
            self.macro_call('DFF_INTO_REAL', next_.name, curr.name, rst_name, clk_name, ce_name, init_str)
        # create memory for integer
        elif ((not check_format) or
              ((isinstance(next_.format_, SIntFormat) and isinstance(curr.format_, SIntFormat)) or
              (isinstance(next_.format_, UIntFormat) and isinstance(curr.format_, UIntFormat)))):
            # check that initial value is valid, if it's number. range checking for DigitalParameters is not
            # yet implemented
            if check_format:
                if isinstance(init, Number):
                    assert (curr.format_.min_val <= init <= curr.format_.max_val), \
                        (f'Initial value {init} does not fit in the range [{curr.format_.min_val}, '
                         f'{curr.format_.max_val}] of signal {curr.name}.')
                else:
                    print(f'Warning: could not validate range of init value {init}')

            # check that the widths of next_ and curr match
            if check_format:
                assert next_.format_.width == curr.format_.width, \
                    (f'The widths of {next_.name} ({next_.format_.width}) does not match '
                     f'the width of {curr.name} ({curr.format_.width}).')

            # determine string expression for the initial value
            if isinstance(init, Number):
                init_str = str(init)
            elif isinstance(init, DigitalParameter):
                init_str = init.name
            elif isinstance(init, str):
                init_str = init
            else:
                raise Exception(f'Could not determine string representation for initial value {init}')

            # call the macro to create the memory
            self.macro_call('MEM_INTO_DIGITAL', next_.name, curr.name, ce_name, clk_name,
                            rst_name, init_str, str(next_.format_.width))
        else:
            raise Exception(
                f'Next and current formats do not match: {next_.name} '
                f'with {next_.format_} vs. {curr.name} with {curr.format_}'
            )

    def make_sync_rom(self, signal: Signal, table: Table, addr: Signal,
                      clk: Signal=None, ce: Signal=None):
        # set defaults
        clk_name = clk.name if clk is not None else "`CLK_MSDSL"
        ce_name = ce.name if ce is not None else "1'b1"

        if isinstance(table, RealTable):
            self.macro_call('SYNC_ROM_INTO_REAL', addr.name, signal.name, clk_name,
                            ce_name, table.addr_bits, table.width, f'"{table.path.as_posix()}"',
                            table.exp)
        elif isinstance(table, SIntTable):
            self.macro_call('SYNC_ROM_INTO_SINT', addr.name, signal.name, clk_name,
                            ce_name, table.addr_bits, table.width, f'"{table.path.as_posix()}"')
        elif isinstance(table, UIntTable):
            self.macro_call('SYNC_ROM_INTO_UINT', addr.name, signal.name, clk_name,
                            ce_name, table.addr_bits, table.width, f'"{table.path.as_posix()}"')
        else:
            raise Exception(f'Unknown table type: {type(table)}')

    def make_sync_ram(self, signal: AnalogSignal, format_: RealFormat, addr: DigitalSignal,
                      clk: DigitalSignal=None, ce: DigitalSignal=None, we: DigitalSignal=None,
                      din: DigitalSignal=None):
        # set defaults
        din_name = din.name if din is not None else "'0"
        clk_name = clk.name if clk is not None else "`CLK_MSDSL"
        ce_name = ce.name if ce is not None else "1'b1"
        we_name = we.name if we is not None else "1'b0"

        # call the SVREAL macro
        self.macro_call('SYNC_RAM_INTO_REAL', addr.name, din_name, signal.name,
                        clk_name, ce_name, we_name, addr.format_.width, format_.width,
                        format_.exponent)

    def start_module(self, name: str, ios: List[Signal], real_params: List, digital_params: List=None):
        # set defaults
        if digital_params is None:
            digital_params = []

        # clear default nettype to make debugging easier
        self.default_nettype('none')
        self.writeln()

        # module name
        self.write(f'module {name}')

        # parameters
        parameters = []
        parameters += [f'parameter real {real_param.param_name}={real_param.default}' for real_param in real_params]
        for dig_param in digital_params:
            dig_param_str = 'parameter '
            if dig_param.signed:
                dig_param_str += 'signed '
            if dig_param.width > 1:
                dig_param_str += f'[{dig_param.width-1}:0] '
            dig_param_str += f'{dig_param.name}={dig_param.default}'
            parameters += [dig_param_str]
        parameters += [self.real_param_str(io) for io in ios if isinstance(io.format_, RealFormat)]
        if len(parameters) > 0:
            self.write(' #')
            self.comma_separated_lines(parameters)

        # ports
        ports = [self.port_str(io) for io in ios]
        if len(ports) > 0:
            self.write(' ')
            self.comma_separated_lines(ports)

        # end module definition and indent
        self.write(';' + self.line_ending)
        self.indent()

        # assign parameters to constants
        if len(real_params) > 0:
            self.comment('Assign real parameters to constant wires')
        for real_param in real_params:
            self.macro_call('MAKE_CONST_REAL', real_param.param_name, real_param.signal_name)

    def end_module(self):
        self.dedent()
        self.writeln('endmodule')
        self.writeln()
        self.default_nettype('wire')

    def make_constant(self, expr: Constant):
        output = Signal(name=next(self.namer), format_=expr.format_)

        if isinstance(expr.format_, RealFormat):
            # compile the range, width, and exponent expressions.
            if isinstance(expr.value, Integral):
                # avoid a synthesis corner-case:
                # https://forums.xilinx.com/t5/Synthesis/Possible-synthesis-bug-casting-integer-to-real-in-a-function/td-p/1140910
                const = str(float(expr.value))
            else:
                const = str(expr.value)
            range = compile_range_expr(expr.format_.range_)
            width = compile_width_expr(expr.format_.width)
            exponent = compile_exponent_expr(expr.format_.exponent)

            # call the appropriate macro
            if width is None:
                self.macro_call('MAKE_CONST_REAL', const, output.name)
            elif exponent is None:
                self.macro_call('MAKE_GENERIC_CONST_REAL', const, output.name, width)
            else:
                self.macro_call('MAKE_FORMAT_REAL', output.name, range, width, exponent)
                self.macro_call('ASSIGN_CONST_REAL', const, output.name)
        elif isinstance(expr.format_, IntFormat):
            self.make_signal(output)
            self.digital_assignment(output, expr.value)
        else:
            raise ValueError(f'Unknown expression format type: ' + expr.format_.__class__.__name__)

        return output

    def make_arithmetic_operator(self, expr: ArithmeticOperator):
        # first check for some special cases involving multiplication.  this is an attempt to get most multiplications
        # to consume only one DSP
        if isinstance(expr, Product) and (len(expr.operands) == 2) and isinstance(expr.format_, RealFormat):
            if isinstance(expr.operands[0], Constant) or isinstance(expr.operands[1], Constant):
                return self.make_constant_mul_signal(expr=expr)
            elif (isinstance(expr.operands[0], Array) and expr.operands[0].all_constants) or \
                 (isinstance(expr.operands[1], Array) and expr.operands[1].all_constants):
                return self.make_constant_array_mul_signal(expr=expr)

        # compile the inputs to signals
        inputs_ = [self.expr_to_signal(operand) for operand in expr.operands]

        # define the operator used to build up the expression.  it takes two signals as arguments and returns
        # a signal bound to the result
        def operator(a, b):
            # determine the parameters of the output signal
            name = next(self.namer)
            format_ = expr.function(a.format_, b.format_)

            # create the output signal
            c = Signal(name=name, format_=format_)

            # assign the result
            if isinstance(expr.format_, RealFormat):
                self.macro_call(REAL_ARITH_OP[type(expr)], a.name, b.name, c.name)
            elif isinstance(expr.format_, IntFormat):
                self.make_signal(c)
                value = INT_ARITH_OP[type(expr)](a.name, b.name)
                self.digital_assignment(signal=c, value=value)
            else:
                raise Exception(f'Unknown expression format type: {expr.format_.__class__.__name__}')

            # return the output signal
            return c

        # apply the operator in a tree-wise fashion
        output = tree_op(operands=inputs_, operator=operator)

        # return the resulting signal
        return output

    def make_constant_mul_signal(self, expr: Product):
        # figure out which operand is the constant and which is the signal (note: if both are constants, that is
        # not handled in a special fashion.  although that shouldn't usually occur due to the way that expressions
        # are built up.)
        if isinstance(expr.operands[0], Constant):
            constant = expr.operands[0]
            signal = self.expr_to_signal(expr.operands[1])
        elif isinstance(expr.operands[1], Constant):
            constant = expr.operands[1]
            signal = self.expr_to_signal(expr.operands[0])
        else:
            raise Exception(f'This expression does not represent a constant times a signal: {expr}.')

        # create the output signal
        output = Signal(name=next(self.namer), format_=expr.format_)

        # if multiplying by -1, use a special macro to reduce utilization
        if constant.value == -1:
            self.macro_call('NEGATE_REAL', signal.name, output.name)
        elif isinstance(constant.value, Integral):
            # avoid a synthesis corner-case:
            # https://forums.xilinx.com/t5/Synthesis/Possible-synthesis-bug-casting-integer-to-real-in-a-function/td-p/1140910
            self.macro_call('MUL_CONST_REAL', str(float(constant.value)), signal.name, output.name)
        else:
            self.macro_call('MUL_CONST_REAL', str(constant.value), signal.name, output.name)

        # return the output signal
        return output

    def make_constant_array_mul_signal(self, expr: Product):
        # figure out which operand is the constant array and which is the signal (note: if both are constant arrays,
        # that is not handled in a special fashion.  although that shouldn't usually occur due to the way that
        # expressions are built up.)
        if isinstance(expr.operands[0], Array) and expr.operands[0].all_constants:
            constant_array = expr.operands[0]
            signal = self.expr_to_signal(expr.operands[1])
        elif isinstance(expr.operands[1], Array) and expr.operands[1].all_constants:
            constant_array = expr.operands[1]
            signal = self.expr_to_signal(expr.operands[0])
        else:
            raise Exception(f'This expression does not represent a constant array times a signal: {expr}.')

        # create the output signal
        output = Signal(name=next(self.namer), format_=expr.format_)

        # create a short real variable to be assigned the selected value from the array
        array_name = next(self.namer)
        self.macro_call('MAKE_SHORT_REAL', array_name, compile_range_expr(constant_array.format_.range_))

        # perform the multiplication
        self.macro_call('MUL_REAL', array_name, signal.name, output.name)

        # compile the array address to a signal
        address = self.expr_to_signal(constant_array.address)

        # compute the valuess to go in the array
        values = [f'`FROM_REAL({element.value}, {array_name})' for element in constant_array.elements]

        # create the array itself
        case_statment(gen=self, sel=address.name, var=array_name, values=values, default=0)

        # return the output signal
        return output

    def make_compress_uint(self, expr: CompressUInt):
        # compile the input to a signal
        input_ = self.expr_to_signal(expr.operand)

        # determine the parameters of the output signal
        name = next(self.namer)
        format_ = expr.format_
        output = Signal(name=name, format_=format_)

        # assign the result
        self.macro_call('COMPRESS_UINT', input_.name, str(input_.format_.width), output.name)

        # return the resulting signal
        return output

    def make_random_integer(self, expr: RandomInteger):
        # validate input
        assert expr.format_.width == 32, 'Only width 32 is supported at this time.'

        # set defaults

        if expr.clk is None:
            clk = '`CLK_MSDSL'
        elif isinstance(expr.clk, str):
            clk = expr.clk
        else:
            clk = expr.clk.name

        if expr.rst is None:
            rst = '`RST_MSDSL'
        elif isinstance(expr.rst, str):
            rst = expr.rst
        else:
            rst = expr.rst.name

        if expr.cke is None:
            cke = "1'b1"
        elif isinstance(expr.cke, str):
            cke = expr.cke
        else:
            cke = expr.cke.name

        if expr.seed is None:
            seed = random.randint(0, (1<<expr.format_.width)-1)
            seed = self.hex_format(seed, expr.format_.width)
        elif isinstance(expr.seed, Integral):
            assert 0 <= expr.seed <= ((1<<expr.format_.width)-1), \
                'Seed is out of range.'
            seed = self.hex_format(expr.seed, expr.format_.width)
        elif isinstance(expr.seed, str):
            seed = expr.seed
        else:
            seed = expr.seed.name

        # determine the parameters of the output signal
        name = next(self.namer)
        output = Signal(name=name, format_=expr.format_)

        # assign the result
        if isinstance(expr, MT19937):
            self.macro_call('MT19937', clk, rst, cke, seed, output.name)
        elif isinstance(expr, LCG):
            self.macro_call('LCG_MSDSL', clk, rst, cke, seed, output.name)
        else:
            raise Exception(f'Unsupported expression: {expr}')

        # return the resulting signal
        return output

    def make_array(self, expr: Array):
        # make the output signal that will hold the results
        output = Signal(name=next(self.namer), format_=expr.format_)
        self.make_signal(output)

        # compile the array elements and address to signals
        elements = [self.expr_to_signal(element) for element in expr.elements]
        address  = self.expr_to_signal(expr.address)

        # extra step for analog signals -- they must be aligned to the output format
        if isinstance(expr.format_, RealFormat):
            # rename the elements list to indicate these values are not aligned to the output format
            non_aligned_elements = elements

            # created an "aligned" version of each signal
            elements = [Signal(name=next(self.namer), format_=expr.format_) for _ in range(len(expr))]
            for element, non_aligned_element in zip(elements, non_aligned_elements):
                self.make_signal(element)
                self.make_assign(input_=non_aligned_element, output=element)

        # now create the array
        case_statment(gen=self, sel=address.name, var=output.name, values=signal_names(elements), default=0)

        # and last return the output signal
        return output

    def make_bitwise_inv(self, expr: BitwiseInv):
        output = Signal(name=next(self.namer), format_=expr.format_)
        self.make_signal(output)

        input_ = self.expr_to_signal(expr.operand)
        value = f'~{input_.name}'
        self.digital_assignment(signal=output, value=value)

        return output

    def make_bitwise_operator(self, expr: BitwiseOperator):
        output = Signal(name=next(self.namer), format_=expr.format_)
        self.make_signal(output)

        inputs = [self.expr_to_signal(operand) for operand in expr.operands]
        value = BITWISE_OP[type(expr)].join(signal_names(inputs))
        self.digital_assignment(signal=output, value=value)

        return output

    def make_comparison_operator(self, expr: ComparisonOperator):
        output = Signal(name=next(self.namer), format_=expr.format_)

        lhs = self.expr_to_signal(expr.lhs)
        rhs = self.expr_to_signal(expr.rhs)

        if isinstance(lhs.format_, RealFormat) and isinstance(rhs.format_, RealFormat):
            self.macro_call(REAL_COMP_OP[type(expr)], lhs.name, rhs.name, output.name)
        elif isinstance(lhs.format_, IntFormat) and isinstance(rhs.format_, IntFormat):
            # make the output signal
            self.make_signal(output)

            # assign the output signal
            value = f"({lhs} {INT_COMP_OP[type(expr)]} {rhs}) ? 1'b1 : 1'b0"
            self.digital_assignment(signal=output, value=value)
        else:
            raise ValueError(f'Unknown format type combination for LHS and RHS: {lhs.format_.__class__.__name__} and {rhs.format_.__class__.__name__}.')

        return output

    def make_concatenation(self, expr: Concatenate):
        output = Signal(name=next(self.namer), format_=expr.format_)
        self.make_signal(output)

        inputs_ = [self.expr_to_signal(operand) for operand in expr.operands]
        value = '{' + ', '.join(signal_names(inputs_)) + '}'
        self.digital_assignment(signal=output, value=value)

        return output

    def make_arithmetic_shift(self, expr: ArithmeticShift):
        output = Signal(name=next(self.namer), format_=expr.format_)
        self.make_signal(output)

        input_ = self.expr_to_signal(expr.operand)
        value = f'{input_.name} {SHIFT_OP[type(expr)]} {expr.shift}'
        self.digital_assignment(signal=output, value=value)

        return output

    def make_bitwise_access(self, expr: BitwiseAccess):
        output = Signal(name=next(self.namer), format_=expr.format_)
        self.make_signal(output)

        input_ = self.expr_to_signal(expr.operand)
        value = f'{input_.name}[{expr.msb}:{expr.lsb}]'
        self.digital_assignment(signal=output, value=value)

        return output

    def make_type_conversion(self, expr: TypeConversion):
        # make the output signal
        output = Signal(name=next(self.namer), format_=expr.format_)

        # compile the input expression to a signal
        input_ = self.expr_to_signal(expr.operand)

        # handle the various cases
        if isinstance(expr, SIntToReal):
            self.macro_call('INT_TO_REAL', input_.name, str(input_.format_.width), output.name)
        elif isinstance(expr, RealToSInt):
            self.macro_call('REAL_TO_INT', input_.name, str(output.format_.width), output.name)
        elif isinstance(expr, UIntToSInt):
            # sanity check
            assert output.format_.width >= input_.format_.width+1, \
                f'Output SInt width ({output.format_.width}) is not at least one greater than the input UInt width ({input_.format_.width}).'

            # make the output signal
            self.make_signal(output)

            # construct string representation of new SInt
            num_zeros = output.format_.width - input_.format_.width
            value = f'{{{self.zero_const(num_zeros)}, {input_.name}}}'

            # make the assignment
            self.digital_assignment(signal=output, value=value, comment='UInt -> SInt')
        elif isinstance(expr, SIntToUInt):
            # sanity check
            assert output.format_.width >= input_.format_.width-1, \
                f'Output UInt width ({output.format_.width}) is not at least one less than the input SInt width ({input_.format_.width}).'

            # make the output signal
            self.make_signal(output)

            # then trim off the sign bit
            value = f'{input_.name}[{input_.format_.width-2}:0]'

            # pad with zeros if necessary
            num_zeros = output.format_.width - (input_.format_.width - 1)
            if num_zeros > 0:
                value = f'{{{self.zero_const(num_zeros)}, {value}}}'

            # make the assignment
            self.digital_assignment(signal=output, value=value, comment='SInt -> UInt')
        else:
            raise ValueError(f'Unknown type conversion: {expr.__class__.__name__}')

        return output

    def init_file(self):
        # print header
        self.comment(f'Model generated on {datetime.datetime.now()}')
        self.writeln()

        # set timescale
        self.writeln(f'`timescale 1ns/1ps')
        self.writeln()

        # include required libraries
        self.include('svreal.sv')
        self.include('msdsl.sv')
        self.writeln()

    def include(self, file):
        self.writeln(f'`include "{file}"')

    def default_nettype(self, type):
        self.writeln(f'`default_nettype {type}')

    def macro_call(self, macro_name, *args, comment=None):
        # format the basic macro call
        line = f"`{macro_name}({', '.join(str(arg) for arg in args)});"

        # add a comment if desired
        if comment is not None:
            line += f' // {comment}'

        # write the full line
        self.writeln(line)

    def digital_assignment(self, signal, value, comment=None):
        # format the basic assignment
        line = f'assign {signal.name} = {value};'

        # add a comment if desired
        if comment is not None:
            line += f' // {comment}'

        # write the full line
        self.writeln(line)

    def comment(self, content=''):
        self.writeln(f'// {content}')

    def comma_separated_lines(self, lines):
        self.write('(' + self.line_ending)
        self.write((',' + self.line_ending).join([self.tab_string + line for line in lines]))
        self.write(self.line_ending)
        self.write(')')

    @staticmethod
    def real_param_str(io):
        return f'`DECL_REAL({io.name})'

    @classmethod
    def port_str(cls, io):
        if isinstance(io, AnalogInput):
            return f'`INPUT_REAL({io.name})'
        elif isinstance(io, AnalogOutput):
            return f'`OUTPUT_REAL({io.name})'
        elif isinstance(io, DigitalInput):
            return f'input wire {cls.int_type_str(io.format_)} {io.name}'
        elif isinstance(io, DigitalOutput):
            return f'output wire {cls.int_type_str(io.format_)} {io.name}'
        else:
            raise Exception('Invalid type.')

    @classmethod
    def zero_const(cls, num_zeros):
        return str(num_zeros) + "'b" + '0'*num_zeros

    @classmethod
    def int_type_str(cls, format_: Union[SIntFormat, UIntFormat]):
        retval = 'logic'

        if isinstance(format_, UIntFormat):
            pass
        elif isinstance(format_, SIntFormat):
            retval += ' signed'
        else:
            raise Exception('Cannot determine if format is signed or unsigned.')

        if format_.width > 1:
            retval += f' [{format_.width-1}:0]'

        return retval

    @classmethod
    def hex_format(cls, val, width):
        str_width = ceil(width/4)
        return f"{width}'h{val:0{str_width}x}"


import torch
from collections import namedtuple

from absl import app

class CummuNoiseTorch:
    @torch.no_grad()
    def __init__(self, std, shapes, device, test_mode=False):
        """
        :param std: standard deviation of the noise
        :param shapes: shapes of the noise, which is basically shape of the gradients
        :param device: device for pytorch tensor
        :param test_mode: if in test mode, noise will be 1 in each node of the tree
        """
        assert std >= 0
        self.std = std
        self.shapes = shapes
        self.device = device
        self.step = 0
        self.binary = [0]
        self.noise_sum = [torch.zeros(shape).to(self.device) for shape in shapes]
        self.recorded = [[torch.zeros(shape).to(self.device) for shape in shapes]]
        self.test_mode = test_mode

    @torch.no_grad()
    def __call__(self):
        """
        :return: the noise to be added by DP-FTRL
        """
        if self.std <= 0 and not self.test_mode:
            return self.noise_sum

        self.step += 1

        idx = 0
        while idx < len(self.binary) and self.binary[idx] == 1:
            self.binary[idx] = 0
            for ns, re in zip(self.noise_sum, self.recorded[idx]):
                ns -= re
            idx += 1
        if idx >= len(self.binary):
            self.binary.append(0)
            self.recorded.append([torch.zeros(shape).to(self.device) for shape in self.shapes])

        for shape, ns, re in zip(self.shapes, self.noise_sum, self.recorded[idx]):
            if not self.test_mode:
                n = torch.normal(0, self.std, shape).to(self.device)
            else:
                n = torch.ones(shape).to(self.device)
            ns += n
            re.copy_(n)

        self.binary[idx] = 1
        return self.noise_sum

    @torch.no_grad()
    def proceed_until(self, step_target):
        """
        Proceed until the step_target-th step. This is for the binary tree completion trick.

        :return: the noise to be added by DP-FTRL
        """
        if self.step >= step_target:
            raise ValueError(f'Already reached {step_target}.')
        while self.step < step_target:
            noise_sum = self.__call__()
        return noise_sum

Element = namedtuple('Element', 'height value')

class CummuNoiseEffTorch:
    """
    The tree aggregation protocol with the trick in Honaker, "Efficient Use of Differentially Private Binary Trees", 2015
    """
    @torch.no_grad()
    def __init__(self, std, shapes, device):
        """
        :param std: standard deviation of the noise
        :param shapes: shapes of the noise, which is basically shape of the gradients
        :param device: device for pytorch tensor
        """
        self.std = std
        self.shapes = shapes
        self.device = device

        self.step = 0
        self.noise_sum = [torch.zeros(shape).to(self.device) for shape in shapes]
        self.stack = []

    @torch.no_grad()
    def get_noise(self):
        return [torch.normal(0, self.std, shape).to(self.device) for shape in self.shapes]

    @torch.no_grad()
    def push(self, elem):
        for i in range(len(self.shapes)):
            self.noise_sum[i] += elem.value[i] / (2.0 - 1 / 2 ** elem.height)
        self.stack.append(elem)

    @torch.no_grad()
    def pop(self):
        elem = self.stack.pop()
        for i in range(len(self.shapes)):
            self.noise_sum[i] -= elem.value[i] / (2.0 - 1 / 2 ** elem.height)

    @torch.no_grad()
    def __call__(self):
        """
        :return: the noise to be added by DP-FTRL
        """
        self.step += 1

        # add new element to the stack
        self.push(Element(0, self.get_noise()))

        # pop the stack
        while len(self.stack) >= 2 and self.stack[-1].height == self.stack[-2].height:
            # create new element
            left_value, right_value = self.stack[-2].value, self.stack[-1].value
            new_noise = self.get_noise()
            new_elem = Element(
                self.stack[-1].height + 1,
                [x + (y + z) / 2 for x, y, z in zip(new_noise, left_value, right_value)])

            # pop the stack, update sum
            self.pop()
            self.pop()

            # append to the stack, update sum
            self.push(new_elem)
        return self.noise_sum

    @torch.no_grad()
    def proceed_until(self, step_target):
        """
        Proceed until the step_target-th step. This is for the binary tree completion trick.

        :return: the noise to be added by DP-FTRL
        """
        if self.step >= step_target:
            raise ValueError(f'Already reached {step_target}.')
        while self.step < step_target:
            noise_sum = self.__call__()
        return noise_sum

def main(argv):
    # This is a small test. If we set the noise in each node as 1 (by setting
    # test_mode=True), we should be seeing the returned noise as the number of
    # 1s in the binary representations of i when cummu_noises is called i times.

    def countSetBits(n):
        count = 0
        while (n):
            n &= (n - 1)
            count += 1
        return count
    cummu_noises = CummuNoiseTorch(1.0, [(1,)], 'cuda', test_mode=True)
    for epoch in range(31):
        random_noise = cummu_noises()
        assert random_noise[0].cpu().numpy()[0] == countSetBits(epoch + 1)

if __name__ == '__main__':
    app.run(main)


#http://jecvay.com/2015/02/python3-web-bug-series5.html
#零基础自学用Python 3开发网络爬虫(五): 使用第三方模块快速抓取与解析
import requests
from bs4 import BeautifulSoup
response = requests.get("http://jecvay.com")
# print(response.text)
soup = BeautifulSoup(response.text, "html.parser")

print(soup.title.text)
# print(soup.body.text)

for x in soup.findAll("a"):
    print(x['href'])


import numpy as np
from enum import Enum

class SolutionCount(Enum):
    ZERO = 1
    ONE = 2
    INFINITY = 3

    def __str__(self):
        return self.name

# Gaussian is already taken :(
class ResultContainer:
    original_matrix: np.ndarray = None
    solution_count: SolutionCount = None
    triangle_matrix: np.ndarray = None
    triangles: np.ndarray = None
    determinant: float = None
    answer: np.ndarray = None
    answer_residual: np.ndarray = None  # English(невязка) == residual

def calculate_gauss_from_parameters(array: np.ndarray) -> ResultContainer:
    return _calculate(array)

def _calculate(array: np.ndarray) -> ResultContainer:
    result = ResultContainer()
    result.original_matrix = array
    result.solution_count = _kronecker_capelli(array)
    if result.solution_count != SolutionCount.ONE:
        return result
    result.triangle_matrix, result.triangles, switches = _forward_path(array.astype(np.float64))
    result.determinant = _calculate_determinant(result.triangle_matrix, switches)
    result.answer = _backward_path(result.triangle_matrix)
    result.answer_residual = _calculate_residual(result.original_matrix, result.answer)
    return result

def _kronecker_capelli(array: np.ndarray) -> SolutionCount:
    a_matrix = array[:, :-1]
    a_extended = array
    rank_a = np.linalg.matrix_rank(a_matrix)
    rank_a_ext = np.linalg.matrix_rank(a_extended)
    # rank(A) != rank(A|B)
    if rank_a != rank_a_ext:
        return SolutionCount.ZERO
    n = max(len(array), len(array[0]) - 1)
    # rank(A) == rank(A|B) < n
    if rank_a < n:
        return SolutionCount.INFINITY
    # rank(A) == rank(A|B) == n
    else:
        return SolutionCount.ONE

def _forward_path(array: np.ndarray) -> [np.ndarray, np.ndarray, int]:
    n = len(array)
    triangles = []
    switches = 0
    for i in range(n-1):
        max_val = abs(array[i][i])
        max_row = i
        for j in range(i + 1, n):
            if abs(array[j][i]) > max_val:
                max_val = abs(array[j][i])
                max_row = j
        if i != max_row:
            array[i], array[max_row] = list(array[max_row]), list(array[i])
            switches += 1

        a_ii = array[i][i]
        for j in range(i + 1, n):
            a_ji = array[j][i]
            r = - a_ji / a_ii
            for k in range(i, n + 1):
                array[j][k] += array[i][k] * r

        triangles.append(array.copy())

    return [array, triangles, switches]

def _calculate_determinant(triangle_matrix: np.ndarray, switches: int) -> float:
    n = len(triangle_matrix)
    determinant = (-1.0) ** switches
    for i in range(n):
        determinant *= triangle_matrix[i][i]
    return determinant

def _backward_path(array: np.ndarray) -> np.ndarray:
    n = len(array)
    x = list(0.0 for _ in range(n))
    for i in range(n - 1, -1, -1):
        a_ii = array[i][i]
        b = array[i][n]
        for j in range(i + 1, n):
            b -= array[i][j] * x[j]
        x[i] = b / a_ii
    return np.array(x)

def _calculate_residual(matrix: np.ndarray, answer: np.ndarray) -> np.ndarray:
    residuals = list()
    n = len(matrix)
    for i in range(n):
        residual = -matrix[i][n]
        for j in range(n):
            residual += matrix[i][j] * answer[j]
        residuals.append(residual)
    return np.array(residuals)


#/usr/bin/env python3
# coding: utf-8

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import argparse
import sys
import re

class PyplotConst (object):
	"""
	Constants for style-plotting with matplotlib.pyplot
	"""

	lineStyles = ['-', '--', '-.', ':', 'None', ' ', '']
	labelFont = {'fontname' : 'Linux Biolinum',
				 #~ 'weight' : 'bold',
				 'size'   : 14	}
	params = {'mathtext.default': 'regular' }

class Offsets (object):
	"""
	Defines the band offset of a single compound specified within the ALIGNMENTS file
	"""

	def __init__ (self, label, vbm, cbm):

		self.label = label
		"""
		Defines a label for the compound
		"""

		self.vbm = float(vbm)
		"""
		Defines the valence band maximum (ionization energy)
		"""

		self.cbm = float(cbm)
		"""
		Defines the conduction band minimum (electron affinity)
		"""

		self.vacum = 0
		"""
		Defines the vacuum level for this material
		"""

		self.color = 'black'
		"""
		Specifies the color for this band discontinuities
		"""

	def setVacuum (self, vac):
		"""
		Sets the vacuum potential for this material
		"""

		self.vacuum = float (vac)

	def setColor (self, color):
		"""
		Sets a color for this material representation
		"""

		self.color = color

def importAlignmentsList (fAlignments):
		'''
		The file ALIGNMENTS contains information on how to describe band offsets between the interfaces. Its format must be several lines organized as something like:

		Label   VBM    CBM   color

		Lines with --- and ... are band offsets dividers, representing dashed and dotted lines.

		For example:
		MoS$_2$ -5.795 -4.016 red
		--- red
		MoS$_2$ -5.819 -4.075 black
		SnS$_2$ -6.662 -5.120 black
		--- blue
		SnS$_2$ -6.586 -5.063 blue
		'''

		## Opens the ALIGNMENTS file
		try:
			with open(fAlignments,'r') as f:
				inputFile = f.read().strip()
				lines = [x for x in inputFile.split('\n') if x] # removes repeated \n ''

		except FileNotFoundError:
			print ('Alignments file not found. Please specify a valid filename.')
			sys.exit()

		offsetsList = []

		## Each line is a band offset or a ligature
		for eachLine in lines:
			## The first information should be the material
			offsetLabel = re.split(' +', eachLine.strip())[0]

			if offsetLabel in PyplotConst.lineStyles:
				## line format: LINE_STYLE COLOR
				try:
					color = re.split(' +', eachLine.strip())[1]
				except IndexError:
					print ('Invalid file formatting. Please correct it and try again.')
					sys.exit(1)

				## Creates an object which is only a ligature
				newOffset = Offsets (offsetLabel, 0, 0)
			else:
				## line format: LABEL VBM CBM COLOR
				try:
					vbm = re.split(' +', eachLine.strip())[1]
					cbm = re.split(' +', eachLine.strip())[2]
					color = re.split(' +', eachLine.strip())[3]
				except IndexError:
					print ('Invalid file formatting. Please correct it and try again.')
					sys.exit(1)

				## Creates an object which is a band offset
				newOffset = Offsets (offsetLabel, vbm, cbm)

			newOffset.setColor(color)
			offsetsList.append(newOffset)

		return offsetsList

def printOffsetsList (offsetsList, args):
		'''
		A list of band offsets is received and printed using matplotlib.pyplot.
		Lines joining different band offsets should not be at the end or beginning of the lists
		'''

		last_x = 0

		offset_x_length = 1
		offset_linewidth = 3
		ligature_x_length = 0.4
		ligature_linewidth = 1.5
		axes_margin = 0.25

		## Then, the offsets are plotted
		for i in range (len(offsetsList)):
			## Plot ligature
			if offsetsList[i].label in PyplotConst.lineStyles:

				## Reads the offsets from the last and the next one
				vbm_last = offsetsList[i-1].vbm
				cbm_last = offsetsList[i-1].cbm
				vbm_next = offsetsList[i+1].vbm
				cbm_next = offsetsList[i+1].cbm

				labelInterface = offsetsList[i-1].label + '/' + offsetsList[i+1].label

				if offsetsList[i].label == '-':
					## Plot line joining adjacent VBM
					plt.plot ([last_x, last_x], [vbm_last, vbm_next],
							  linestyle = offsetsList[i].label,
							  color = offsetsList[i].color,
							  linewidth = offset_linewidth)

					## Plot line joining adjacent CBM
					plt.plot ([last_x, last_x], [cbm_last, cbm_next],
							  linestyle = offsetsList[i].label,
							  color = offsetsList[i].color,
							  linewidth = offset_linewidth)

					## Label interface
					## label becomes aligned in the center if there is only one interface (two materials)
					if args.number == 2:
						plt.annotate (labelInterface, (last_x, (max(vbm_last, vbm_next)+ min(cbm_last, cbm_next))/2),
									  horizontalalignment='center', verticalalignment='center', **PyplotConst.labelFont)

					last_x = last_x

				else:
					## Plot line joining VBM with stylish lines
					plt.plot ([last_x, last_x + ligature_x_length], [vbm_last, vbm_next],
							  linestyle = offsetsList[i].label,
							  color = offsetsList[i].color,
							  linewidth = ligature_linewidth)

					## Plot line joining CBM with stylish lines
					plt.plot ([last_x, last_x + ligature_x_length], [cbm_last, cbm_next],
							  linestyle = offsetsList[i].label,
							  color = offsetsList[i].color,
							  linewidth = ligature_linewidth)

					last_x = last_x + ligature_x_length

			## Or plot offset
			else:

				vbm = offsetsList[i].vbm
				cbm = offsetsList[i].cbm

				## Plot VBM
				plt.plot ([last_x, last_x + offset_x_length], [vbm, vbm],
						  linestyle = '-',
						  color = offsetsList[i].color,
						  linewidth = offset_linewidth)

				plt.annotate (("% .2f" % offsetsList[i].vbm), (last_x + offset_x_length/2, vbm - 0.03), horizontalalignment='center', verticalalignment='top', **PyplotConst.labelFont)

				## Plot CBM
				plt.plot ([last_x, last_x + offset_x_length], [cbm, cbm],
						  linestyle = '-',
						  color = offsetsList[i].color,
						  linewidth = offset_linewidth)

				plt.annotate (("% .2f" % offsetsList[i].cbm), (last_x + offset_x_length/2, cbm + 0.02), horizontalalignment='center', verticalalignment='bottom', **PyplotConst.labelFont)

				plt.annotate (offsetsList[i].label, (last_x + offset_x_length/2, (vbm+cbm)/2),
									  horizontalalignment='center', verticalalignment='center', **PyplotConst.labelFont)

				last_x = last_x + offset_x_length

		## A set of pyplot configurations is passed
		plt.margins (axes_margin)
		plt.rcParams.update(PyplotConst.params)

		## Turn on the axis if requested
		if args.axis:
			plt.axis('on')
		else:
			plt.axis('off')

		plt.savefig (args.output)

		## Finally, the plot is shown
		if args.show:
			plt.show()

def parseArgs():
	"""
	Parse arguments from the command line. Uses the `argparse` package to
	establish all positional and optional arguments.
	"""
	parser = argparse.ArgumentParser(description='Easy script to create band alignments',
									epilog= "Written by Daniel S. Koda (feb. 2017).",
									prog="band_offsets.py")

	parser.add_argument('input_file', default='ALIGNMENTS', help="ALIGNMENTS input file")

	parser.add_argument('-o', '--output', default='offsets.png', help="output name for the generated files with its format. Default: offsets.png")

	parser.add_argument('-v', '--vacuum', type=float, default=0.0, help="vacuum dipole step (Default: 0.0)")

	parser.add_argument('-q', '--quiet', action='store_true', help="do not display text on the output window (default: False)")

	parser.add_argument('-x', '--axis', action='store_true', help="turn on the axis for the plot (default: False)")

	parser.add_argument('-s', '--show', action='store_true', help="show the band alignments before saving to file (default: False)")

	parser.add_argument('-n', '--number', type=int, default=2, help="number of materials composing the interface (Default: 2)")

	return parser.parse_args()

def printRunDescription (args):
	'''
	Print description of the options chosen and the crystals input.
	'''

	leftJustSpace = 20
	print ("input file:".ljust(leftJustSpace) + "%s" % args.input_file)
	print ("output file:".ljust(leftJustSpace) + "%s" % args.output)
	print ("vacuum step:".ljust(leftJustSpace) + "% .2f" % args.vacuum)

def main():
	'''
	Rotates a molecule in a POSCAR file by angleDegrees
	The refAtom is taken as reference to the rotation
	'''

	args = parseArgs()

	if not args.quiet:
		print ("*****************************")
		print (" vaspirin v2.0: band_offsets ")
		print ("*****************************")

		printRunDescription (args)

	## Open the input file and creates an offsets list
	offsetsList = importAlignmentsList (args.input_file)

	## Plots the artistic band offsets
	printOffsetsList (offsetsList, args)

if __name__ == "__main__":
	main ()


#!/usr/bin/env python

from widget import Widget
from collections import deque
import logging

logger = logging.getLogger('neoled')

class GraphWidget(Widget):
    def callback(self, bus, argument):
        self.called_bus = bus
        self.data.append(argument)
        self.data.popleft()

    def __init__(self, matrix, bus, offscreenCanvas, x=0, y=10, width=50, height=10, lineColor="0xFFFFFF",
                 borderColor="0x0A0A0A", bgColor="0x00000A", listen="event.key",
                 ranges=[(0, 3, "0x00FF00"), (3, 5, "0xFF8800"), (5, 10, "0xFF0000")], barWidth=2, stacked=True,
                 type=__name__):
        super(GraphWidget, self).__init__(matrix, bus, offscreenCanvas, x, y, width, height)
        self.lineColor = super(GraphWidget, self).color_from_hex(int(lineColor, 0))
        self.borderColor = super(GraphWidget, self).color_from_hex(int(borderColor, 0))
        self.bgColor = super(GraphWidget, self).color_from_hex(int(bgColor, 0))

        self.bus.subscribe(listen, self.callback)

        self.ranges = ranges
        self.bar_width = barWidth
        self.stacked = stacked
        self.data = deque([0] * (self.width // self.bar_width))
        self.scale = 1.0
        logger.info("Started Graph Widget at x: " + str(self.x) + " y: " + str(self.y))

    def Display(self):

        super(GraphWidget, self).drawBackground(self.bgColor)

        m = max(list(self.data))
        scale = 1.0

        if (m > 0):
            scale = self.height * 1.0 / m

        if scale < self.scale:
            self.scale = scale

        count = 0
        for d in list(self.data):

            for i in range(0, self.height, 1):
                color = super(GraphWidget, self).color_from_hex(0x000000)

                for r in self.ranges:
                    if self.stacked:
                        if (i < d * scale) and (r[0] <= i < r[1]):
                            color = super(GraphWidget, self).color_from_hex(int(r[2], 0))
                    else:
                        if (i < d * scale) and (r[0] <= d < r[1]):
                            color = super(GraphWidget, self).color_from_hex(int(r[2], 0))

                for w in range(self.bar_width):
                    # print "d: "+str(d)+"w:" + str(w) + " count:" + str(count) + " = " + str(self.x + w + (count * self.bar_width))
                    self.offscreenCanvas.SetPixel(self.x + w + (count * self.bar_width), self.y - i, color.red,
                                                  color.green, color.blue)

            count += 1

        super(GraphWidget, self).drawBorder(self.borderColor)


import typing

from aiogram.dispatcher.filters import Text
from aiogram.types import Message
from sqlalchemy import select, and_
from sqlalchemy.ext.asyncio import AsyncSession

from keyboards.default import kb_edit_master, kb_return_main_menu
from loader import dp
from misc import state_add_master
from model import database

@dp.message_handler(Text(equals="Удалить мастера"))
async def start_delete_master_handler(m: Message, session: AsyncSession):
    masters = await database.master_barber.MasterBarber.all(session)
    markup = kb_edit_master.get_master_btn(masters)
    markup.add(
        kb_return_main_menu.get_back()
    )
    await m.answer(
        text="Выберите мастера.\n<i>При нажатии на кнопку - мастер удалится</i>",
        reply_markup=markup
    )
    await state_add_master.StateDeleteMaster.start.set()

@dp.message_handler(state=state_add_master.StateDeleteMaster.start)
async def delete_master_handler(m: Message, session: AsyncSession):
    master_name = m.text

    stmt = select(
        database.master_barber.MasterBarber, database.schedule_barber.ScheduleBarber
    ).join(database.schedule_barber.ScheduleBarber,
           and_(database.schedule_barber.ScheduleBarber.master_id == database.master_barber.MasterBarber.id,
                database.schedule_barber.ScheduleBarber.status == database.schedule_barber.ScheduleBarber.typing_status.progress),
           isouter=True,
           ).where(
        database.master_barber.MasterBarber.master == master_name,
        database.master_barber.MasterBarber.delete_status == 0
    )

    response = await session.execute(stmt)
    master, schedule_master = response.all()[0]

    if schedule_master:
        return await m.answer("Вы не можете удалить мастера! У него есть действующие записи")

    master.delete_status = 1
    session.add(master)
    await session.commit()

    masters = await database.master_barber.MasterBarber.all(session)
    markup = kb_edit_master.get_master_btn(masters)
    markup.add(
        kb_return_main_menu.get_back()
    )
    await m.answer("Мастер успешно удален.", reply_markup=markup)


from typing import Iterable

class ConsumerIterator(Iterable):
    def __init__(self,generator):
        self.generator=generator

    def __iter__(self):
        return self

    def __next__(self):
        a = yield from self.generator
        return a

def producer():
    a = yield
    return a

class Mean:
    def compute(self,iterator):
        m=0
        i=0
        for v in iterator:
           v+=m
           i+=1
        self.result = m

def generate_producers():
    n = 3
    values = list(range(1,n+1))
    producers = [producer() for i in values]
    # initialize producers
    for c in producers:
        next(c)
    iterators = [ConsumerIterator(p) for p in producers]
    means = [Mean() for i in iterators]
    for m,i in zip(means,iterators):
        print(m.compute(i))
    for i in range(5):
        for i,c in enumerate(producers):
            c.send(values[i])
        for i,v in enumerate(values):
            values[i]*=2
    for m in means:
        print(m.result)

if __name__ == '__main__':
    generate_producers()


#!/bin/python3
import sys,socket
from datetime import datetime

#Define our target
if len(sys.argv) == 2:
	#print(sys.argv)#-------> argv is ['scanner.py', '192.168.1.0']

	target=socket.gethostbyname(sys.argv[1]) #translating host name to ipv4 not necessary we could 		#dirctly declare target=sys.argv[1] bt just taking extra step in case someone just had put a	 	 #hostname
else:
	print("Invalid amount of arguemnts")
	print("Syntax: python3 scanner.py <ip>")

#isnt a very good script coz if someone types a port which doesnt exist then this might run into ann error na

#creating a banner
print("-" *50)
print("Scanning Target " + target)
print("Time Started: "+ str(datetime.now()))
print("-" *50)

try:
	for port in range(50,85):
		s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)

		socket.setdefaulttimeout(1)
		result=s.connect_ex((target,port)) #connect_ex returns an error indicator if port is open result is going to be 0 if port is closed result back is going to trigger an error which is going to be 1
		if result == 0:
			print("Port {} is open".format(port))
		s.close()#close the connection

except KeyboardInterrupt:
	print("\n Exiting program")
	sys.exit()
except socket.gaierror:
	print("Hostname could not be resolved.")
	sys.exit()
except socket.error:
	print("coundt connect to server")
	sys.exit()

#python3 scanner.py <ip>


import cv2
import os
import numpy as np
import PIL
from PIL import Image
import tqdm
import time
import shutil

time_start =  time.time()
path = r'D:\project\image_similarity\image'
list_images = os.listdir(path)
list_sameImage = []
dict_image_load = {}
for image in tqdm.tqdm(list_images):
    path_image = os.path.join(path, image)
    img_1 = Image.open(path_image)
    image_read = np.array(img_1)
    dict_image_load[image] = image_read

print('finish load data')
for im1 in dict_image_load:
    for im2 in dict_image_load:
        try:
            if (dict_image_load[im1] == dict_image_load[im2]).all() and (im1 != im2):
                print('같은 이미지 pair',im1, im2)
                if [im1, im2] not in list_sameImage and [im2, im1] not in list_sameImage:
                    list_sameImage.append([im1, im2])
        except:
            print(im1, im2)

time_end =  time.time()
print(list_sameImage)
print(time_end - time_start)

os.mkdir(os.path.join(path, 'sameImage'))

for sameImage in list_sameImage:
    for sI in sameImage:
        filename = sI.split('\\')[-1]
        shutil.move(os.path.join(path, filename), os.path.join(os.path.join(path, 'sameImage'),filename))

f = open(os.path.join(path, 'list_sameImage.txt'), 'w', encoding = 'utf-8')

for sameImage in list_sameImage:

    f.write(str("['이미지','이미지']: "))
    f.write(str(sameImage))
    f.write(str('\n'))

f.close()


from django.conf.urls import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

import os
ROOT_DIR = os.path.dirname(__file__)

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'webtool.views.home', name='home'),
    # url(r'^webtool/', include('webtool.foo.urls')),
    url(r'^project/', include('project.urls')),
    url(r'^web-?log/', include('weblog.urls')),
    url(r'^files/', include('fileupload.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^$', 'views.index'),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^css/(?P<path>.*)$','django.views.static.serve',
     { 'document_root': os.path.join(ROOT_DIR, 'static/css')}),
    url(r'^js/(?P<path>.*)$','django.views.static.serve',
     { 'document_root': os.path.join(ROOT_DIR, 'static/js')}),
    url(r'^robots.txt$', 'views.robots'),
)


#!/usr/bin/env python

import click
import os
import random
import shutil

@click.command()
@click.option('--count', default=1, help='Number of file to sample.')
@click.option('--prefix', default="", help='Prefix of the files to consider')
@click.option('--printonly', default=False, type=click.BOOL, help='Print only instead of copying')
@click.option('--verbose', default=False, type=click.BOOL, help='Be verbose')
@click.argument('src', nargs=1, type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True))
@click.argument('dst', nargs=1, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True))
def cli(count, prefix, printonly, verbose, src, dst):
	"""Sample `count` files, copying them from `src` to `dst`"""

	def is_candidate(pathname, prefix=""):
		filename = os.path.basename(pathname)
		if prefix is not None and prefix != "":
			return os.path.isfile(pathname) and filename.startswith(prefix)
		return os.path.isfile(pathname)

	# Based on https://stackoverflow.com/questions/49280966/pulling-random-files-out-of-a-folder-for-sampling
	candidates = [f for f in os.listdir(src) if is_candidate(os.path.join(src, f), prefix)]
	shuffled = list(candidates)
	random.shuffle(shuffled)
	n_to_select = count
	if n_to_select > len(shuffled):
		n_to_select = len(shuffled)
	selected = shuffled[:n_to_select]
	for filename in selected:
		if printonly:
			print(filename)
			# click.echo(filename)
		else:
			if verbose:
				print("Copying... {}".format(filename))
			shutil.copy(os.path.join(src, filename), dst, follow_symlinks=True)

if __name__ == '__main__':
	cli()


import torch.nn as nn
from .model_utils import weights_init

class Generator(nn.Module):
    def __init__(self, ngpu, nz, ngf, nc):
        super(Generator, self).__init__()
        self.ngpu = ngpu
        self.main = nn.Sequential(
            nn.ConvTranspose2d(nz, ngf*8, 4, 1, 0, bias = False),
            nn.BatchNorm2d(ngf*8),
            nn.ReLU(True),
            nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf * 4),
            nn.ReLU(True),
            nn.ConvTranspose2d( ngf * 4, ngf * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf * 2),
            nn.ReLU(True),
            nn.ConvTranspose2d( ngf * 2, ngf, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf),
            nn.ReLU(True),
            nn.ConvTranspose2d(ngf, nc, 4, 2, 1, bias=False),
            nn.Tanh()
        )

    def forward(self, input):
        return self.main(input)

class Discriminator(nn.Module):
    def __init__(self, ngpu, ndf, nc):
        super(Discriminator, self).__init__()
        self.ngpu = ngpu
        self.main = nn.Sequential(
            # input is (nc) x 64 x 64
            nn.Conv2d(nc, ndf, 4, 2, 1, bias=False),
            nn.LeakyReLU(0.2, inplace=True),
            # state size. (ndf) x 32 x 32
            nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 2),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Dropout2d(p=0.2, inplace=False),
            # state size. (ndf*2) x 16 x 16
            nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 4),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Dropout2d(p=0.2, inplace=False),
            # state size. (ndf*4) x 8 x 8
            nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 8),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Dropout2d(p=0.2, inplace=False),
            # state size. (ndf*8) x 4 x 4
            nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False),
            nn.Sigmoid()
        )

    def forward(self, input):
        return self.main(input)

def init_models(ngpu, device, nz, ngf, ndf, nc = 3, verbose = False):
    # init generator
    netG = Generator(ngpu, nz, ngf, nc).to(device)
    # init discriminator
    netD = Discriminator(ngpu, ndf, nc).to(device)

    # parallelize
    if (device.type == 'cuda') and (ngpu > 1):
        netG = nn.DataParallel(netG, list(range(ngpu)))
        netD = nn.DataParallel(netD, list(range(ngpu)))

    netG.apply(weights_init)
    netD.apply(weights_init)

    if (verbose):
        print(netG)
        print(netD)

    return netG, netD


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('testimonials', '0002_auto_20151003_1248'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='testimonial',
            options={'ordering': ['item_order']},
        ),
        migrations.AddField(
            model_name='testimonial',
            name='item_order',
            field=models.PositiveIntegerField(default=0),
        ),
    ]


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from setuptools import setup, find_packages
import os
import subprocess

name = 'python-ipmi'
version_py = os.path.join(os.path.dirname(__file__), 'pyipmi', 'version.py')

def git_pep440_version():
    full = subprocess.check_output(
            ['git', 'describe', '--tags', '--always', '--dirty=.dirty'],
            stderr=subprocess.STDOUT).decode().strip()
    tag = subprocess.check_output(
            ['git', 'describe', '--tags', '--always', '--abbrev=0'],
            stderr=subprocess.STDOUT).decode().strip()
    tail = full[len(tag):]
    return tag + tail.replace('-', '.dev', 1).replace('-', '+', 1)

try:
    version = git_pep440_version()
    with open(version_py, 'w') as f:
        f.write('# This file was autogenerated by setup.py\n')
        f.write('__version__ = \'%s\'\n' % (version,))
except (OSError, subprocess.CalledProcessError, IOError):
    try:
        with open(version_py, 'r') as f:
            d = dict()
            exec(f.read(), d)
            version = d['__version__']
    except IOError:
        version = 'unknown'

with open('README.rst') as f:
    readme = f.read()

setup(name=name,
      version=version,
      description='Pure python IPMI library',
      long_description=readme,
      url='https://github.com/kontron/python-ipmi',
      download_url='https://github.com/kontron/python-ipmi/tarball/' + version,
      author='Michael Walle, Heiko Thiery',
      author_email='michael.walle@kontron.com, heiko.thiery@kontron.com',
      packages=find_packages(exclude=['tests*']),
      license='LGPLv2+',
      platforms=["any"],
      classifiers=[
          'Development Status :: 3 - Alpha',
          'Environment :: Console',
          'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
          'Natural Language :: English',
          'Operating System :: OS Independent',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.7',
          'Programming Language :: Python :: 3.8',
          'Programming Language :: Python :: 3.9',
          'Programming Language :: Python :: 3.10',
          'Programming Language :: Python :: 3.11',
          'Topic :: Software Development :: Libraries :: Python Modules',
      ],
      entry_points={
          'console_scripts': [
              'ipmitool.py = pyipmi.ipmitool:main',
          ]
      },
      test_suite='tests',
      tests_require=[
          'pytest',
          'mock',
      ],
      install_requires=[
          'future',
      ],
      )


from enum import Enum
from typing import NamedTuple, Tuple, Union

from quantlib.algorithms.qbase import QGranularity,         QGranularitySpecType,           resolve_qgranularityspec
from quantlib.algorithms.qbase import QRange,               QRangeSpecType,                 resolve_qrangespec
from quantlib.algorithms.qbase import QHParamsInitStrategy, QHParamsInitStrategySpecType,   resolve_qhparamsinitstrategyspec
from .qalgorithm               import QAlgorithm,           QAlgorithmSpecType,             resolve_qalgorithmspec
from quantlib.utils import quantlib_err_header

# -- CANONICAL DATA STRUCTURE -- #

class QDescription(NamedTuple):
    """Collect pieces of information required to assemble ``_QModule``s.

    ``_QModule``s are meant to be sub-classed. These sub-classes are grouped
    under namespaces identified by the acronym of the PTQ/QAT algorithm that
    they implement.

    Independently of the PTQ/QAT algorithm, each ``_QModule`` must describe
    how it quantises the ``torch.Tensor`` that it is responsible for. This
    description consists of three parts:
    * the quantisation granularity (i.e., which partitions of the
      ``torch.Tensor``'s components use different quantisers);
    * the integer range underlying the fake-quantised representation;
    * the strategy to initialise the quantisers' hyper-parameters.

    Then, each namespace of algorithm-specific ``_QModule``s should define a
    mapping from floating-point ``nn.Module``s to fake-quantised counterparts;
    optionally, each namespace could also define algorithm-specific arguments
    that should be passed to the constructors of its (algorithm-specific)
    ``_QModule`` sub-classes. These two pieces of information are aggregated
    into ``QAlgorithm`` objects.

    """
    qgranularity:         QGranularity
    qrange:               QRange
    qhparamsinitstrategy: QHParamsInitStrategy
    qalgorithm:           QAlgorithm

# -- CANONICALISAITON FLOW -- #

QDescriptionSpecType = Union[QDescription, Tuple[QGranularitySpecType, QRangeSpecType, QHParamsInitStrategySpecType, QAlgorithmSpecType]]

def resolve_qdescription_qdescriptionspectype(qdescriptionspec: QDescription) -> QDescription:
    return qdescriptionspec

def resolve_tuple_qdescriptionspectype(qdescriptionspec: Tuple[QGranularitySpecType, QRangeSpecType, QHParamsInitStrategySpecType, QAlgorithmSpecType]) -> QDescription:

    qgranularityspec, qrangespec, qhparamsinitstrategyspec, qalgorithmspec = qdescriptionspec
    qgranularity = resolve_qgranularityspec(qgranularityspec)
    qrange = resolve_qrangespec(qrangespec)
    qhparamsinitstrategy = resolve_qhparamsinitstrategyspec(qhparamsinitstrategyspec)
    qalgorithm = resolve_qalgorithmspec(qalgorithmspec)

    return QDescription(qgranularity=qgranularity,
                        qrange=qrange,
                        qhparamsinitstrategy=qhparamsinitstrategy,
                        qalgorithm=qalgorithm)

QDescriptionSpecSolvers = Enum('QDescriptionSpecSolvers',
                               [
                                   ('QDESCRIPTION', resolve_qdescription_qdescriptionspectype),
                                   ('TUPLE',        resolve_tuple_qdescriptionspectype),
                               ])

def resolve_qdescriptionspec(qdescriptionspec: QDescriptionSpecType) -> QDescription:

    qdescriptionspec_class = qdescriptionspec.__class__.__name__
    try:
        solver = getattr(QDescriptionSpecSolvers, qdescriptionspec_class.upper())  # when the values of an enumerated are functions, I can not access them in dictionary-style: https://stackoverflow.com/a/50211710
        qdescription = solver(qdescriptionspec)
        return qdescription
    except KeyError:
        raise TypeError(quantlib_err_header() + f"Unsupported QDescription specification type: {qdescriptionspec_class}.")