""" Keras models, using the functional interface

build() creates a VAE, encoder and generator
use vae.load_weights('myfile.h5') to load a pre-trained model
"""
from __future__ import division

import config  # incl. random seed

import numpy as np
import collections
from sklearn.decomposition import PCA
import scipy.stats
import keras
from keras.utils import to_categorical
from keras import optimizers, backend as K
from keras.layers import *
from keras.models import Model
from keras.preprocessing.image import ImageDataGenerator

from utils import utils, plot

########################################################################
### Model applications
########################################################################

def transfer_style(encoder, decoder, stylesA, stylesB, samples=[], amt=1.):
    # stylesA, stylesB, samples = lists of samples that can be encoded by `encoder`
    # encoder output must be a np array
    transformation = extract_transformation(encoder, stylesA, stylesB)
    latent_vectors = encoder.predict(samples)
    latent_vectors_ = apply_transformation(latent_vectors, transformation, amt)
    return decoder.predict(latent_vectors_)

def extract_transformation(encoder, stylesA, stylesB) -> np.array:
    # stylesA, stylesB = lists of samples that can be encoded by `encoder`
    # extract the linear latent transformation that corresponds with A -> B
    latent_vectors_A = encoder.predict(stylesA)
    latent_vectors_B = encoder.predict(stylesB)
    return latent_vectors_B.mean(axis=0) - latent_vectors_A.mean(axis=0)

def apply_transformation(vectors: np.array, transformation: np.array,
                         amt=1.) -> np.array:
    # np automatically maps the transformation to every instance with (+)
    return vectors + transformation * amt

def gen_latent(generator,
               batch_size=2,
               latent_dim=2,
               x_encoded=0.,
               latent_indices=(0, 1),
               n=10,
               m=4,
               min_x=0.05,
               max_x=0.95,
               min_y=0.05,
               max_y=0.95,
               assume_gaussion=True,
               plot_result=False,
               v=0):
    """ Original: keras.keras.examples.variational_autoencoder
    :x_encoded :: float | [ float ]

    to swap x,y set `latent_indices`` to (1,0)
    """
    if not isinstance(x_encoded, np.ndarray):
        x_encoded = np.repeat(x_encoded, latent_dim)
    if v: print(x_encoded.shape, x_encoded)
    x_decoded = generator.predict(x_encoded.reshape([1, latent_dim]))

    # Coordinate grid
    grid_x = np.linspace(min_x, max_x, n)
    grid_y = np.linspace(min_y, max_y, m)
    # linearly spaced coordinates on the unit square were transformed through
    # the inverse CDF (ppf) of the Gaussian to produce values of the latent
    # variables z, since the prior of the latent space is Gaussian
    if assume_gaussion:
        grid_x = scipy.stats.norm.ppf(grid_x)
        grid_y = scipy.stats.norm.ppf(grid_y)

    # Generation
    result = collections.defaultdict(dict)
    for yi in grid_x:
        for xi in grid_y:
            z_sample = x_encoded.copy()
            z_sample[np.array(latent_indices)] = (xi, yi)
            # z_sample = np.array([[yi, xi]])
            z_sample = np.tile(z_sample, batch_size).reshape(
                batch_size, latent_dim)
            x_decoded = generator.predict(z_sample, batch_size=batch_size)
            result[yi][xi] = x_decoded[0]

    if plot_result:
        plot.multi(result, v=v)
    return result

########################################################################
### Model construction
########################################################################
"""
If using a pre-trained model, make sure that the number of latent dims and the number of midi-notes are equal
"""

def build(input_shape=(160, 10, 1), latent_dim=10, epsilon_std=1.):
    timesteps, notes, _ = input_shape
    encoder_model, encoder_input, z_mean, z_log_var = encoder(
        input_shape, latent_dim)
    z_input = encoder_model(encoder_input)

    # this function must be defined locally in order for the model to be serializable
    # using the following inline lambda function will result in an error when .to_json() is called
    #   lambda args: sample(args, z_mean, z_log_var, latent_dim, epsilon_std)
    def sample(args):
        z_mean, z_log_var = args
        epsilon = K.random_normal(
            shape=(K.shape(z_mean)[0], latent_dim),
            mean=0.,
            stddev=epsilon_std)
        return z_mean + K.exp(z_log_var) * epsilon

    z_output = Lambda(sample)(z_input)
    decoder_model = decoder(z_output, latent_dim, input_shape)

    # VAE
    vae_input = Input(shape=(input_shape, ))
    vae_input = encoder_input
    vae_output = decoder_model(z_output)
    # vae_output = decoder_model(z_mean)
    vae = Model(vae_input, vae_output, name='vae-model-')

    vae_loss_ = vae_loss(
        vae_input,
        vae_output,
        z_mean,
        z_log_var,
        timesteps,
        notes,
        beta=0.75,
        gamma=0.05)
    vae.add_loss(vae_loss_)
    vae.compile(optimizer='adam')

    # Encoder (matrices -> latent vectors)
    encoder_ = Model(encoder_input, z_mean)
    # Generator (latent vectors -> matrices)
    # generator_input = Input((latent_dim, ))
    # generator_layers_ = utils.composition(decoders, generator_input)
    # generator = Model(generator_input, generator_layers_)
    return vae, encoder_, decoder_model

def build_manual(input_shape=(160, 10, 1), latent_dim=10, epsilon_std=1.):
    """
    this model is not serializable due to the lambda layers
    """
    timesteps, notes, _ = input_shape
    encoder_model, encoder_input, z_mean, z_log_var = encoder(
        input_shape, latent_dim)
    sample_ = lambda args: sample(args, z_mean, z_log_var, latent_dim, epsilon_std)
    z_input = encoder_model(encoder_input)
    z_output = Lambda(sample_)(z_input)
    decoders = list_decoders(input_shape)
    decoded = utils.composition(decoders, z_output, verbose=False)

    # VAE: Full model (to train)
    vae_input = encoder_input
    vae_output = decoded
    vae = Model(vae_input, vae_output)
    vae_loss_ = vae_loss(
        vae_input,
        vae_output,
        z_mean,
        z_log_var,
        timesteps,
        notes,
        beta=0.75,
        gamma=0.05)
    vae.add_loss(vae_loss_)
    vae.compile(optimizer='adam')

    # Encoder (matrices -> latent vectors)
    encoder_ = Model(encoder_input, z_mean)

    # Generator (latent vectors -> matrices)
    generator_input = Input((latent_dim, ))
    generator_layers_ = utils.composition(decoders, generator_input)
    generator = Model(generator_input, generator_layers_)

    return vae, encoder_, generator

# internal functions

def sample(args, z_mean, z_log_var, latent_dim=2, epsilon_std=1.):
    z_mean, z_log_var = args
    epsilon = K.random_normal(
        shape=(K.shape(z_mean)[0], latent_dim), mean=0., stddev=epsilon_std)
    return z_mean + K.exp(z_log_var) * epsilon

def encoder(input_shape, latent_dim=2):
    encoder_input = Input(shape=input_shape)
    nodes = np.prod(input_shape)
    timesteps, notes, channels = input_shape

    # Convolution
    h = encoder_input
    h = Reshape((timesteps, notes))(h)
    h = Conv1D(
        64, kernel_size=2, strides=1, activation='relu', padding='valid')(h)

    h = Bidirectional(LSTM(128))(h)

    # Z Mean, Variance
    z_mean = Dense(latent_dim, name='z_mean')(h)  # no activation='relu'
    z_log_var = Dense(latent_dim, name='z_log_var')(h)  # no activation='relu'

    encoder_output = [z_mean, z_log_var]
    encoder_model = Model(encoder_input, encoder_output, name='encoder_model-')
    return encoder_model, encoder_input, z_mean, z_log_var

def list_decoders(output_shape):
    # decoder_input = z_output
    # h = decoder_input
    # :output_shape = (timesteps, channels, channels) || (batches, filters, timesteps, channels)
    # keras offers just Conv2DTranspose and not Conv1DTranspose
    # - use 2D images during upsampling :: (timesteps, notes, channels) => (timesteps, notes, filters)
    # - use 1D images to optimize reconstruction :: (timesteps, filters) => (timesteps, notes)

    # image_data_format = 'channels_last'
    # goal shape: (timesteps, notes, channels)
    # start with the 'reverse': lots of small imgs => few large img

    timesteps, notes, channels = output_shape

    # keras.examples.variational_autoencoder_deconv.py
    decoders = []
    decoders += [Dense(256)]
    decoders += [LeakyReLU(alpha=0.3)]

    # add a bypass layer
    w = 256
    decoders += [Dense(w, activation='relu')]
    extra_decoders = []
    for _ in range(3):
        extra_decoders += [Dense(w, activation='elu')]

    extra_d = Lambda(lambda layer: utils.composition(extra_decoders, layer))
    decoders += [Lambda(lambda layer: Add()([layer, extra_d(layer)]))]

    decoders += [BatchNormalization(momentum=0.9)]
    n = 10  # 5

    decoders += [RepeatVector(n)]
    decoders += [Bidirectional(LSTM(128, return_sequences=True))]

    # Embedding decoder

    embedding_len = int(timesteps / n)
    filters = 250
    decoders += [TimeDistributed(Dense(filters, activation='relu'))]
    decoders += [
        TimeDistributed(Dense(notes * embedding_len, activation='sigmoid'))
    ]
    decoders += [Reshape((timesteps, notes, 1))]

    return decoders

def decoder(input_, latent_dim=10, output_shape=(40, 10, 1)):
    """ decoder_input = z_output
    decoder.predict :: z -> x
    the decoder models p(x|z), with an assumption over p(z)
    """
    # image_data_format = 'channels_last'
    timesteps, notes, channels = output_shape

    decoder_input = Input(shape=(latent_dim, ))

    h = decoder_input
    h = Dense(256)(h)
    h = LeakyReLU(alpha=0.3)(h)

    # add a bypassed layer
    w = 256
    h_bypass = Dense(w, activation='relu')(h)
    extra_decoders = []
    for _ in range(3):
        h_bypass = Dense(w, activation='elu')(h_bypass)

    h = Add()([h, h_bypass])
    h = BatchNormalization(momentum=0.9)(h)

    n = 10  # 5
    h = RepeatVector(n)(h)
    h = Bidirectional(LSTM(128, return_sequences=True))(h)

    # Embedding decoder

    embedding_len = int(timesteps / n)
    filters = 250
    h = TimeDistributed(Dense(filters, activation='relu'))(h)
    h = TimeDistributed(Dense(notes * embedding_len, activation='sigmoid'))(h)
    h = Reshape((timesteps, notes, 1))(h)

    return Model(decoder_input, h, name='decoder_model-')

def vae_loss(vae_input,
             vae_output,
             z_mean,
             z_log_var,
             timesteps=40,
             notes=9,
             beta=0.5,
             extra_loss_f=keras.losses.mean_absolute_error,
             gamma=0.5):
    """
    vae_input, vae_output == x == y :: np.ndarray
    z_mean, z_log_var :: np.ndarray
    beta = disentanglement amount
    gamma = multiplier for extra loss function
    """
    vae_input_ = K.flatten(vae_input)
    vae_output_ = K.flatten(vae_output)
    # bin. cross-entropy
    xent_loss = timesteps * notes * keras.metrics.binary_crossentropy(
        vae_input_, vae_output_)
    # Kullback-Leibler divergence
    kl_loss = -1. * K.sum(
        1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
    # (optional) kl_loss = max(kl_loss, free_bits)
    extra_loss = timesteps * notes * extra_loss_f(vae_input_, vae_output_)

    # change beta
    #     beta = ((1.0 - tf.pow(hparams.beta_rate, tf.to_float(self.global_step)))
    #             * hparams.max_beta)
    #     self.loss = tf.reduce_mean(r_loss) + beta * tf.reduce_mean(kl_cost)
    # y_true, y_pred, z_mean, z_log_var, timesteps=150, notes=3, beta=1.

    vae_loss = K.mean(xent_loss + beta * kl_loss + gamma * extra_loss)
    return vae_loss

class ImageDataGenerator(keras.preprocessing.image.ImageDataGenerator):
    def __init__(self,
                 x,
                 batch_size=32,
                 phase_mod=0.2,
                 whitening=False,
                 zca_epsilon=1e-6):
        keras.preprocessing.image.ImageDataGenerator.__init__(
            self,
            featurewise_center=False,  # set input mean to 0 over the dataset
            samplewise_center=False,  # set each sample mean to 0
            featurewise_std_normalization=False,  # divide inputs by dataset std
            samplewise_std_normalization=False,  # divide each input by its std
            zca_epsilon=zca_epsilon,
            zca_whitening=whitening,
            rotation_range=0,  # randomly rotate images in 0 to 180 degrees
            width_shift_range=0.,  # note-channel mod, but not shuffled
            height_shift_range=phase_mod,  # start_t, phase
            horizontal_flip=False,  # reverse
            vertical_flip=False)

        self.x = x
        self.fit(x)
        self.batch_size = batch_size

    def __len__(self):
        # n batches / epoch = #samples / batch_size
        # floor() to prevent occurences of samples multiple times
        return int(np.floor(self.x.shape[0] / self.batch_size))

    def shuffle_3rd_dim(self, x_batch):
        """ Shuffle the 3rd matrix dim with a bias for soft mutations
        :x_batch :: (samples, timesteps, 3rd_dim, channels)
        """
        z_batch = np.empty_like(x_batch)
        for batch_i in range(x_batch.shape[0]):
            indices = np.arange(x_batch.shape[2])
            np.random.shuffle(indices)
            for i, j in enumerate(indices):
                z_batch[batch_i, :, i] = x_batch[batch_i, :, j]

        return z_batch

    def shuffle_3rd_dim_soft(self,
                             x_batch,
                             rate=1,
                             intensity=0.5,
                             scale=0.5,
                             verbose=0):
        """ Shuffle the 3rd matrix dim with a bias for soft mutations
        :x_batch :: (samples, timesteps, 3rd_dim, channels)
        :rate = amount (%) of samples that are mutated
        :intensity = amount of mutations in a sample, relative to len(3rd_dim)
          in range [0, ..]
        :scale = intensity of mutations, relative to len(3rd_dim)
          standard_deviation = scale * len(3rd_dim)
        """
        # z_batch = np.empty_like(x_batch)
        z_batch = x_batch.copy()
        length = x_batch.shape[2]
        indices = np.arange(length)
        std = scale * length
        for batch_i in range(x_batch.shape[0]):
            if np.random.random() < rate:
                if verbose > 1: print('batch_i: %i' % batch_i)
                for i, j in enumerate(
                        self._shuffle_indices(indices, std, verbose)):
                    z_batch[batch_i, :, i] = x_batch[batch_i, :, j]

        return z_batch

    def _shuffle_indices(self, indices, intensity, std=1, verbose=0):
        indices = indices.copy()
        length = indices.shape[0]
        n_iterations = int(round(length * np.random.random()**(1 / intensity)))
        for _ in range(n_iterations):
            i1 = np.random.choice(indices)
            i2 = int((i1 + np.random.normal(0, std)) % length)
            if verbose > 0: print(i1, i2)
            indices[i1], indices[i2] = indices[i2], indices[i1]
        return indices


import torch
import torchvision.transforms as T
from torch.utils.data import Dataset
import cv2
from PIL import Image
import os
import sys

import numpy as np
import matplotlib.pyplot as plt

from sklearn.preprocessing import LabelEncoder

from  typing import List

class CustomDataset(Dataset):
    def __init__(self, data_path : str, resize : int = 227):
        super(CustomDataset, self).__init__()

        self.data_path = data_path
        self.resize    = resize
        self.transform = T.Compose([
                             T.ToPILImage(),
                             T.RandomResizedCrop(resize),
                             T.ColorJitter(brightness=0.3, saturation=0.3),
                             T.RandomHorizontalFlip(),
                             T.ToTensor(),
                         ])

        if not os.path.exists(data_path):
            print(f"Can't find : {data_path}")
            sys.exit(1)

        self.labels = os.listdir(data_path)

        # save classes
        with open('class.txt','w') as f:
            for l in self.labels:
                f.write(f"{l}\n")

        self.x = []
        self.y = []
        for label in self.labels:
             file_lists =  os.listdir(os.path.join(data_path, label))
             for fname in file_lists:
                 ## original
                 self.x.append(self.preprocessing(os.path.join(data_path, label, fname),
                                                  resize))
                 self.y.append(label)
                 ## data augmentation
                 for i in range(19):
                     self.x.append(self.preprocessing(os.path.join(data_path, label, fname),
                                                      resize, self.transform))
                     self.y.append(label)
        # encoding labels
        dataset_label_encoder = self.make_labels(self.labels)
        self.y = dataset_label_encoder.transform(self.y)

    @staticmethod
    def make_labels(labels: List):
        dataset_label_encoder = LabelEncoder()
        dataset_label_encoder.fit(labels)
        return dataset_label_encoder

    @staticmethod
    def preprocessing(fname : str, resize : int, transform : T = None):
        if not os.path.exists(fname):
            print(f"Can't find : {fname}")
            sys.exit(1)

        img = cv2.imread(fname)
        if transform is None:
            transform = T.Compose([
                            T.ToPILImage(),
                            T.Resize(resize),
                            T.ToTensor(),  # conver RGB pixel value in the [0,255] to [0.0, 1.0] range
                        ])
        img = transform(img)

        ## plot image
        #plt.imshow(img.permute(1,2,0))
        #plt.show(block=False)
        #plt.pause(1)
        return img

    def __len__(self):
        return len(self.x)

    def __getitem__(self, idx):
        return self.x[idx], torch.tensor(self.y[idx])


import qrcode
import csv
import os

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as ex:
        if ex.errno!=errno.EEXIST:
            raise

def make_codes():
    with open('short_urls.csv') as csvfile:
        reader = csv.reader(csvfile)
        for short_url,long_url,count in reader:
            path = long_url.replace('http://exoplanets.seti.org/','')
            img = qrcode.make(short_url)
            img.save('qrcodes/%s.png'%path)

if __name__ == "__main__":
    mkdir_p('qrcodes')
    make_codes()


# -*- coding: utf-8 -*-
import scrapy
from News_Data.items import NewsDataItem
import pandas as pd
from .parse_detail import ProcessContent
from urllib import parse
from scrapy.utils.project import get_project_settings
from gerapy_auto_extractor.extractors import *

settings = get_project_settings()

class NewsDataSpider(scrapy.Spider):
    name = 'News_Data'
    allowed_domains = []

    def start_requests(self):
        # 读取数据
        df = pd.read_excel("我的数据抓取列表.xlsx", sheet_name="数据列表")
        # 数据打乱
        df = df.sample(frac=1.0).reset_index(drop=True)

        for i in range(len(df)):
            data = df.iloc[i].to_dict()
            print(data)
            # 判断常规网页
            if data['动态加载url'] == '-' and data['url 参数'] == '-':
                yield scrapy.Request(
                    url=data["网址"], meta={'data': data, }, callback=self.parse_static
                )

    def parse_static(self, response):
        data = extract_list(response.text)
        for each in range(len(data)):
            item = NewsDataItem()
            item['title'] = data[each]["title"].strip()
            item['url'] = parse.urljoin(response.url, data[each]["url"])
            item['publishTime'] = ""
            item["new_type"] = response.meta["data"]["类别"]
            item['web_name'] = response.meta["data"]["网站名称"]
            item['channel_name'] = response.meta["data"]["网站频道"]

            print(item)

            yield scrapy.Request(item['url'], callback=self.parse_detail, meta={'item': item})

    # 具体内容在parse_detail.py中
    def parse_detail(self, response):
        item = ProcessContent(self, response)
        yield item

Crawler Project Template/Scrapy 新闻项目批量生产/News_Data/News_Data/spiders/News_Data.py


### Adapted from https://github.com/kundajelab/dragonn/blob/master/dragonn/plot.py
### Authored by Nathan Boley

import re

import matplotlib
matplotlib.use('pdf')
from matplotlib import pyplot as plt
from matplotlib.patches import PathPatch
from matplotlib.path import Path
import matplotlib.patches as mpatches

from shapely.wkt import loads as load_wkt
from shapely import affinity
import itertools

import numpy as np
from pkg_resources import resource_filename

##########################################################################
# copied from descartes
# https://pypi.python.org/pypi/descartes

class Polygon(object):
    # Adapt Shapely or GeoJSON/geo_interface polygons to a common interface

    def __init__(self, context):
        if hasattr(context, 'interiors'):
            self.context = context
        else:
            self.context = getattr(context, '__geo_interface__', context)

    @property
    def geom_type(self):
        return (
            getattr(self.context, 'geom_type', None) or self.context['type'])

    @property
    def exterior(self):
        return (
            getattr(self.context, 'exterior', None) or self.context['coordinates'][0])

    @property
    def interiors(self):
        value = getattr(self.context, 'interiors', None)
        if value is None:
            value = self.context['coordinates'][1:]
        return value

def PolygonPath(polygon):
    """Constructs a compound matplotlib path from a Shapely or GeoJSON-like
    geometric object"""
    this = Polygon(polygon)
    assert this.geom_type == 'Polygon'

    def coding(ob):
        # The codes will be all "LINETO" commands, except for "MOVETO"s at the
        # beginning of each subpath
        n = len(getattr(ob, 'coords', None) or ob)
        vals = np.ones(n, dtype=Path.code_type) * Path.LINETO
        vals[0] = Path.MOVETO
        return vals
    vertices = np.concatenate(
        [np.asarray(this.exterior)] + [np.asarray(r)
                                       for r in this.interiors])
    codes = np.concatenate(
        [coding(this.exterior)] + [coding(r)
                                   for r in this.interiors])
    return Path(vertices, codes)

def PolygonPatch(polygon, **kwargs):
    """Constructs a matplotlib patch from a geometric object
    The `polygon` may be a Shapely
    or GeoJSON-like object with or without holes.
    The `kwargs` are those supported by the matplotlib.patches.Polygon class
    constructor. Returns an instance of matplotlib.patches.PathPatch.
    Example (using Shapely Point and a matplotlib axes):
      >>> b = Point(0, 0).buffer(1.0)
      >>> patch = PolygonPatch(b, fc='blue', ec='blue', alpha=0.5)
      >>> axis.add_patch(patch)
    """
    return PathPatch(PolygonPath(polygon), **kwargs)

#
# END copied from descartes
#
##########################################################################

##########################################################################
# Initialize the polygon paths for A,C,G,T
#
# Geometry taken from JTS TestBuilder Monospace font with fixed precision model
# of 1000.0
#

A_data = """
MULTIPOLYGON (
((24.7631 57.3346, 34.3963 57.3346, 52.391 -1.422, 44.1555 -1.422, 39.8363
  13.8905, 19.2476 13.8905, 15.0039 -1.422, 6.781 -1.422, 24.7631 57.3346)),
((29.5608 50.3205, 21.1742 20.2623, 37.9474 20.2623, 29.5608 50.3205))
)
"""

C_data = """POLYGON((
52.391 2.5937, 48.5882 0.8417, 44.68 -0.4142, 40.5998 -1.17, 36.2814 -1.422,
32.8755 -1.2671, 29.6656 -0.8024, 26.6518 -0.0278, 23.834 1.0565,
21.2122 2.4507, 18.7865 4.1547, 16.5569 6.1686, 14.5233 8.4922,
12.7087 11.0966, 11.136 13.9527, 9.8053 17.0606, 8.7166 20.4201,
7.8698 24.0314, 7.2649 27.8943, 6.902 32.009, 6.781 36.3754, 6.9027 40.7209,
7.2678 44.8198, 7.8764 48.6722, 8.7283 52.278, 9.8236 55.6371,
11.1624 58.7497, 12.7446 61.6157, 14.5702 64.2351, 16.6133 66.5753,
18.8481 68.6034, 21.2745 70.3195, 23.8926 71.7235, 26.7023 72.8156,
29.7037 73.5956, 32.8967 74.0637, 36.2814 74.2197, 40.5998 73.9697,
44.68 73.2196, 48.5882 71.9696, 52.391 70.2196, 52.391 60.1101,
48.6468 62.739, 44.6331 64.657, 40.4709 65.8289, 36.2814 66.2196,
31.7716 65.7557, 29.7437 65.1758, 27.8672 64.3641, 26.1421 63.3203,
24.5684 62.0447, 23.146 60.5371, 21.875 58.7976, 19.7831 54.6129,
18.289 49.481, 17.3925 43.4019, 17.0936 36.3754, 17.3925 29.3763,
18.289 23.3166, 19.7831 18.1964, 21.875 14.0157, 23.146 12.2762,
24.5684 10.7686, 26.1421 9.4929, 27.8672 8.4492, 29.7437 7.6375,
31.7716 7.0576, 36.2814 6.5937, 40.5354 6.9844, 44.7034 8.1563,
48.6878 10.0743, 52.391 12.7032, 52.391 2.5937))"""

G_data = """POLYGON((
52.391 5.4974, 50.49 3.8964, 48.4724 2.502, 46.3383 1.3144, 44.0877 0.3334,
41.7314 -0.4346, 39.2805 -0.9832, 34.0946 -1.422, 30.9504 -1.2772,
27.9859 -0.843, 25.2009 -0.1191, 22.5956 0.8942, 20.1698 2.197,
17.9236 3.7894, 15.857 5.6713, 13.9699 7.8428, 12.285 10.2753,
10.8248 12.9404, 9.5892 15.8381, 8.5782 18.9685, 7.7919 22.3315,
7.2303 25.9271, 6.8933 29.7553, 6.781 33.8161, 6.8948 37.8674,
7.2362 41.6888, 7.8053 45.2803, 8.6019 48.6419, 9.6262 51.7737, 10.878 54.6755,
12.3575 57.3474, 14.0646 59.7895, 15.9743 61.9712, 18.0615 63.862,
20.3262 65.4618, 22.7685 66.7708, 25.3884 67.789, 28.1857 68.5162,
31.1606 68.9525, 34.3131 69.098, 38.5048 68.7957, 42.5144 67.8889,
46.3638 66.3703, 50.0748 64.2325, 50.0748 54.8075, 46.342 57.8466,
42.5144 59.9716, 38.5266 61.2226, 34.3131 61.6395, 30.1132 61.2053,
28.2228 60.6624, 26.4723 59.9024, 24.8614 58.9253, 23.3904 57.731,
22.0591 56.3195, 20.8675 54.691, 18.9046 50.7806, 17.5025 45.998,
16.6612 40.3432, 16.3808 33.8161, 16.6526 27.1962, 17.4679 21.4959,
18.8267 16.7151, 20.7291 12.8539, 21.8892 11.2595, 23.1951 9.8776,
24.6469 8.7084, 26.2446 7.7517, 27.9883 7.0076, 29.8778 6.4762, 34.0946 6.051,
36.9534 6.2276, 39.4407 6.7575, 41.6331 7.6625, 43.607 8.9644, 43.607 27.2172,
33.7304 27.2172, 33.7304 34.7776, 52.391 34.7776, 52.391 5.4974
))"""

T_data = """POLYGON((
6.781 58.3746, 52.391 58.3746, 52.391 51.5569, 33.6933 51.5569, 33.6933 -1.422,
25.5684 -1.422, 25.5684 51.5569, 6.781 51.5569, 6.781 58.3746
))"""

def standardize_polygons_str(data_str):
    """Given a POLYGON string, standardize the coordinates to a 1x1 grid.
    Input : data_str (taken from above)
    Output: tuple of polygon objects
    """
    # find all of the polygons in the letter (for instance an A
    # needs to be constructed from 2 polygons)
    path_strs = re.findall("\(\(([^\)]+?)\)\)", data_str.strip())

    # convert the data into a numpy array
    polygons_data = []
    for path_str in path_strs:
        data = np.array([
            tuple(map(float, x.split())) for x in path_str.strip().split(",")])
        polygons_data.append(data)

    # standardize the coordinates
    min_coords = np.vstack(data.min(0) for data in polygons_data).min(0)
    max_coords = np.vstack(data.max(0) for data in polygons_data).max(0)
    for data in polygons_data:
        data[:, ] -= min_coords
        data[:, ] /= (max_coords - min_coords)

    polygons = []
    for data in polygons_data:
        polygons.append(load_wkt(
            "POLYGON((%s))" % ",".join(" ".join(map(str, x)) for x in data)))

    return tuple(polygons)

letters_polygons = {}
letters_polygons['A'] = standardize_polygons_str(A_data)
letters_polygons['C'] = standardize_polygons_str(C_data)
letters_polygons['G'] = standardize_polygons_str(G_data)
letters_polygons['T'] = standardize_polygons_str(T_data)

colors = dict(zip(
    'ACGT', (('red', 'white'), ('blue',), ('orange',), ('green',))
))

def add_letter_to_axis(ax, let, x, y, height):
    """Add 'let' with position x,y and height height to matplotlib axis 'ax'.
    """
    for polygon, color in zip(letters_polygons[let], colors[let]):
        new_polygon = affinity.scale(
            polygon, yfact=height, origin=(0, 0, 0))
        new_polygon = affinity.translate(
            new_polygon, xoff=x, yoff=y)
        patch = PolygonPatch(
            new_polygon, edgecolor=color, facecolor=color)
        ax.add_patch(patch)
    return

def plot_bases_on_ax(letter_heights, ax):
    """
    Plot the N letters with heights taken from the Nx4 matrix letter_heights.
    Parameters
    ----------
    letter_heights: Nx4 array
    ax: axis to plot on
    """

    assert letter_heights.shape[1] == 4, letter_heights.shape
    for x_pos, heights in enumerate(letter_heights):
        letters_and_heights = sorted(zip(heights, 'ACGT'))
        y_pos_pos = 0.0
        y_neg_pos = 0.0
        for height, letter in letters_and_heights:
            if height > 0:
                add_letter_to_axis(ax, letter, 0.5 + x_pos, y_pos_pos, height)
                y_pos_pos += height
            elif height < 0:
                add_letter_to_axis(ax, letter, 0.5 + x_pos, y_neg_pos, height)
                y_neg_pos += height
    ax.set_xlim(0, letter_heights.shape[0] + 1)
    ax.set_xticks(np.arange(1, letter_heights.shape[0] + 1))
    ax.set_aspect(aspect='auto', adjustable='box')
    ax.autoscale_view()

def plot_bases(letter_heights, figsize=(12, 6), ylab='bits'):
    """
    Plot the N letters with heights taken from the Nx4 matrix letter_heights.
    Parameters
    ----------
    letter_heights: Nx4 array
    ylab: y axis label
    Returns
    -------
    pyplot figure
    """
    assert letter_heights.shape[1] == 4, letter_heights.shape

    fig = pyplot.figure(figsize=figsize)
    ax = fig.add_subplot(111)
    ax.set_xlabel('pos')
    ax.set_ylabel(ylab)
    plot_bases_on_ax(letter_heights, ax)

    return fig

def plot_pwm(letter_heights,
             figsize=(12, 6), ylab='bits', information_content=True):
    """
    Plots pwm. Displays information content by default.
    """
    if information_content:
        letter_heights = letter_heights * (
            2 + (letter_heights *
                 np.log2(letter_heights)).sum(axis=1))[:, np.newaxis]
    return plot_bases(letter_heights, figsize, ylab=ylab)

def add_letters_to_axis(ax, letter_heights):
    """
    Plots letter on user-specified axis.
    Parameters
    ----------
    ax : axis
    letter_heights: Nx4 array
    """
    assert letter_heights.shape[1] == 4

    x_range = [1, letter_heights.shape[0]]
    pos_heights = np.copy(letter_heights)
    pos_heights[letter_heights < 0] = 0
    neg_heights = np.copy(letter_heights)
    neg_heights[letter_heights > 0] = 0

    for x_pos, heights in enumerate(letter_heights):
        letters_and_heights = sorted(zip(heights, 'ACGT'))
        y_pos_pos = 0.0
        y_neg_pos = 0.0
        for height, letter in letters_and_heights:
            if height > 0:
                add_letter_to_axis(ax, letter, 0.5 + x_pos, y_pos_pos, height)
                y_pos_pos += height
            else:
                add_letter_to_axis(ax, letter, 0.5 + x_pos, y_neg_pos, height)
                y_neg_pos += height

    ax.set_xlim(x_range[0] - 1, x_range[1] + 1)
    ax.set_xticks(range(*x_range) + [x_range[-1]])
    ax.set_aspect(aspect='auto', adjustable='box')
    ax.autoscale_view()

def plot_delta_tracks(track_list, title_list, x_tick_interval=1,
                      track_height=3, axvline_loc=None,
                      plot_file=None):
    """
    track_list = list of tracks of size (seq_len, 4)
    title_list = list of labels corresponding to track_list
    """
    plt.clf()
    width = track_list[0].shape[1]
    fig_width = 20 + width/10
    fig = plt.figure(figsize=(fig_width, track_height*len(track_list)))
    yMax = np.max([arr.max() for arr in track_list]) + .01
    yMin = np.min([arr.min() for arr in track_list]) - .01
    for (i,array) in enumerate(track_list):
        ax = fig.add_subplot(len(track_list),1,i+1)
        if (array.shape[0]==4):
            letter_heights=array.T
            pos_heights = np.copy(letter_heights)
            pos_heights[letter_heights < 0] = 0
            neg_heights = np.copy(letter_heights)
            neg_heights[letter_heights > 0] = 0
            for x_pos, heights in enumerate(letter_heights):
                letters_and_heights = sorted(itertools.izip(heights, 'ACGT'))
                y_pos_pos = 0.0
                y_neg_pos = 0.0
                for height, letter in letters_and_heights:
                    if height > 0:
                        add_letter_to_axis(ax, letter, -0.5+x_pos,
                                           y_pos_pos, height)
                        y_pos_pos += height
                    else:
                        add_letter_to_axis(ax, letter, -0.5+x_pos,
                                           y_neg_pos, height)
                        y_neg_pos += height
            if (i==len(track_list)):
                ax.set_xlabel('pos')
            ax.set_aspect(aspect='auto', adjustable='box')
            ax.autoscale_view()
        elif (array.shape[0]==1):
            ax.plot(range(arr.shape[1]), array.squeeze(), 'k', lw=0.5)
        else:
           raise RuntimeError("Unsure how to deal with shape "+str(arr.shape))
        ax.axhline(0, linestyle='dashed', color='black')
        ax.set_title(title_list[i])
        ax.set_ylabel('Importance Score', fontsize=16)
        ax.set_xlim(-1, array.shape[1]+1)
        ax.set_ylim([yMin, yMax])
        if axvline_loc is not None:
            ax.axvline(axvline_loc)
        xticks_locations=range(-1,array.shape[1]+1)
        ax.set_xticks(xticks_locations[::x_tick_interval])
        ax_2=ax.twiny()
    fig.tight_layout()
    if plot_file is not None:
        plt.savefig(plot_file)
    # plt.show()
    return plt

def plot_dfim(map_array, figsize=(10, 3),
              cmap='plasma'):
    """
    Plot map
    """
    if len(map_array.shape) != 2:
        raise ValueError('Provide map array with shape 2 or unravel map')
    plt.figure(figsize=figsize)
    fig, ax = plt.subplots()
    if map_array.shape[0] == 1:
        im = ax.imshow(map_array, aspect=10, cmap=cmap)
    else:
        im = ax.imshow(map_array, aspect='auto', cmap=cmap)
    values = np.unique(map_array.ravel())
    legend_values=[np.min(values), np.mean(values), np.max(values)]
    colors = [ im.cmap(im.norm(value)) for value in legend_values]
    patches = [ mpatches.Patch(color=colors[i],
                               label="Level {l}".format(
                                l='%.4f'%legend_values[i]))
                   for i in range(len(legend_values))]
    plt.legend(handles=patches, bbox_to_anchor=(1.05, 1),
               loc=2, borderaxespad=0. )
    # plt.show()
    return plt


import argparse
import logging
import os
import sys
import gc
import cv2

import numpy as np
from PIL import Image
import torch
import torch.nn as nn

from utils import ThyroidDataset

from torch import optim
from torch.autograd import Variable
from tqdm import tqdm

from utils.ThyroidDataset import BasicDataset
from utils.eval_demo import eval_demo
from utils.loss import MultipleLoss

from network import densenet

import torchvision.models as models
from tensorboardX import SummaryWriter

from torch.utils.data import DataLoader

from torch.backends import cudnn

from mylogger import Logger

cudnn.benchmark = False
cudnn.deterministic = True
data_dir = "/home/kongming/dagengren/data/Thyroid"
dir_checkpoint = 'checkpoints_src/'
model_dir = "../pretrained_model/model.pth"

class GradCam:
    def __init__(self, model, use_cuda):
        self.model = model
        self.model.eval()
        self.cuda = use_cuda
        if self.cuda:
            self.model = model.cuda()

    def __call__(self, input, classifier_index, index=None):
        # if self.cuda:
        #     features, output = self.extractor(input.cuda(), vector.cuda(), mask_nodule.cuda(), mask_margin.cuda(), aspect.cuda(), classifier_index)#self.forward(input.cuda())
        # else:
        #     features, output = self.extractor(input, vector, mask_nodule, mask_margin, aspect, classifier_index)#self.forward(input)
        #

        input = torch.from_numpy(input.transpose(2, 0, 1)).unsqueeze(0)
        if self.cuda:
            input = input.float().cuda()

        mask_pred, outputs, features = self.model(input)
        output = outputs[classifier_index+1]

        if index == None:
            index = np.argmax(output.cpu().data.numpy())
        pre = np.argmax(output.cpu().data.numpy())

        one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)
        one_hot[0][index] = 1
        one_hot = Variable(torch.from_numpy(one_hot), requires_grad=True)
        if self.cuda:
            one_hot = torch.sum(one_hot.cuda() * output)
        else:
            one_hot = torch.sum(one_hot * output)

        self.model.zero_grad()
        one_hot.backward(retain_graph=True)
        grads_val = self.model.gradients.cpu().data.numpy()

        target = features
        target = target.cpu().data.numpy()[0, :]

        weights = np.mean(grads_val, axis=(2, 3))[0, :]
        cam = np.zeros(target.shape[1:], dtype=np.float32)

        for i, w in enumerate(weights):
            cam += w * target[i, :, :]

        cam = np.maximum(cam, 0)
        cam = cv2.resize(cam, (224, 224))
        cam = cam - np.min(cam)
        cam = cam / np.max(cam)

        return cam, pre, output, mask_pred

def show_cam_on_image(img, mask, size, rootPath):
    heatmap = cv2.applyColorMap(np.uint8(255*mask), cv2.COLORMAP_JET)
    heatmap = np.float32(heatmap) / 255
    heatmap = cv2.resize(heatmap, (size[0], size[1]))
    cam = heatmap + np.float32(img)/255
    cam = cam / np.max(cam)
    result = np.uint8(255 * cam)
    print(rootPath+"cam.jpg")
    cv2.imwrite(rootPath+"cam.jpg", result)

def test_single(net, device, fname):
    img_pth = "./demo/demo_img/{}".format(fname)
    vis_pth = "./demo/visualization/{}".format(fname.split(".")[0])
    try:
        os.mkdir(vis_pth)
    except OSError:
        pass
    img = Image.open(img_pth)
    orig_size = img.size
    np_img = np.array(img.resize((224,224)))/255
    #print(np_img.shape)
    mask_np = eval_demo(net, np_img, device, log.logger)
    print("Start generate segmentation...")
    mask_img = Image.fromarray(mask_np.astype('uint8')).convert('RGB').resize(orig_size)
    mask_img.save(os.path.join(vis_pth, fname.split(".")[0]+"_seg.jpg"))
    print("Finished.")
    print("Start generate heatmap for attributes...")
    attribute_name = ["Echogenic Foci", "Composition", "Margin", "Echogenicity"]
    target_name = [["none", "exists"], ["solid", "non-solid"],
                   ["ill-defined", "well-defind"], ["hypoechogenic", "hyper-isoechogenic"]]
    typeList = [2, 2, 2, 2]
    grad_cam = GradCam(model=net, use_cuda=True)
    for class_index in range(0, len(typeList)):
        for target_index in range(0, typeList[class_index]):
            cam_mask, output_index, output, mask_pred = grad_cam(np_img, class_index, target_index)
            rootPath = os.path.join(vis_pth, fname.split(".")[0])
            rootPath += "_" + attribute_name[class_index] + '_' + target_name[class_index][target_index] + '_'
            show_cam_on_image(img, cam_mask, orig_size, rootPath)
    print("finished.")

    #log.logger.info('Validation Dice Coeff: {}'.format(val_score))

def get_args():
    parser = argparse.ArgumentParser(description='Train the UNet on images and target masks',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-f', '--filename', dest='fname', type=str, default="demo1.jpg",
                        help='The file in ./demo/demo_img/')

    return parser.parse_args()

if __name__ == '__main__':

    log = Logger("log_demo.txt")

    args = get_args()
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    seed = 1
    torch.manual_seed(seed)
    if device=="cuda":
        torch.cuda.manual_seed(seed)

    log.logger.info(f'Using device {device}')

    log.logger.info(f'Start initialize model...')

    net_pretrained=models.densenet201(pretrained=True)
    net = densenet.densenet201(num_classes=1)

    net_dict = net.state_dict()

    for k in net_pretrained.features.state_dict():
        k_S = k.split(".")
        block_num=int(k_S[0][-1])
        if block_num==5:
            k_net = "block_4."
            for i in k_S:
                k_net=k_net+i+"."
            k_net = k_net[:-1]
        else:
            #k_S[0]="block_{}".format(block_num)
            k_net = "block_{}.".format(block_num)
            for i in k_S:
                k_net=k_net+i+"."
            k_net = k_net[:-1]

        #print(k)
        #print(k_net)

        v = net_pretrained.features.state_dict()[k]
        net_dict[k_net]=v

    net.load_state_dict(net_dict)
    log.logger.info(f'Model loaded from {model_dir}')
    net.load_state_dict(
        torch.load(model_dir, map_location=device)
    )

    log.logger.info(f'Model initialized')
    net.to(device=device)

    print("Filename: {}".format(args.fname))
    try:
        test_single(net, device, args.fname)
    except KeyboardInterrupt:
        try:
            sys.exit(0)
        except SystemExit:
            os._exit(0)


__author__ = "V. Armando Sole - ESRF"
__contact__ = "sole@esrf.fr"
__license__ = "MIT"
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
import unittest
import os
import sys
import numpy
import gc
import shutil
import tempfile
try:
    import h5py
    HAS_H5PY = True
except Exception:
    HAS_H5PY = None

DEBUG = 0

class testMcaStackExport(unittest.TestCase):
    def setUp(self):
        """
        Get the data directory
        """
        self._importSuccess = False
        self._outputDir = None
        self._h5File = None
        try:
            from PyMca5 import PyMcaDataDir
            self._importSuccess = True
            self.dataDir = PyMcaDataDir.PYMCA_DATA_DIR
        except Exception:
            self.dataDir = None

    def tearDown(self):
        gc.collect()
        if self._h5File is not None:
            fileName = self._h5File
            if os.path.exists(fileName):
                os.remove(fileName)
        if self._outputDir is not None:
            shutil.rmtree(self._outputDir, ignore_errors=True)
            if os.path.exists(self._outputDir):
                raise IOError("Directory <%s> not deleted" % self._outputDir)

    @unittest.skipIf(not HAS_H5PY, "skipped h5py missing")
    def testSingleStackExport(self):
        from PyMca5 import PyMcaDataDir
        from PyMca5.PyMcaIO import specfilewrapper as specfile
        from PyMca5.PyMcaIO import ConfigDict
        from PyMca5.PyMcaCore import DataObject
        from PyMca5.PyMcaCore import StackBase
        from PyMca5.PyMcaCore import McaStackExport
        spe = os.path.join(self.dataDir, "Steel.spe")
        cfg = os.path.join(self.dataDir, "Steel.cfg")
        sf = specfile.Specfile(spe)
        self.assertTrue(len(sf) == 1, "File %s cannot be read" % spe)
        self.assertTrue(sf[0].nbmca() == 1,
                        "Spe file should contain MCA data")

        y = counts = sf[0].mca(1)
        x = channels = numpy.arange(y.size).astype(numpy.float64)
        sf = None
        configuration = ConfigDict.ConfigDict()
        configuration.read(cfg)
        calibration = configuration["detector"]["zero"], \
                      configuration["detector"]["gain"], 0.0
        initialTime = configuration["concentrations"]["time"]
        # create the data
        nRows = 5
        nColumns = 10
        nTimes = 3
        data = numpy.zeros((nRows, nColumns, counts.size), dtype = numpy.float64)
        live_time = numpy.zeros((nRows * nColumns), dtype=numpy.float64)
        xpos = 10 + numpy.zeros((nRows * nColumns), dtype=numpy.float64)
        ypos = 100 + numpy.zeros((nRows * nColumns), dtype=numpy.float64)
        mcaIndex = 0
        for i in range(nRows):
            for j in range(nColumns):
                data[i, j] = counts
                live_time[i * nColumns + j] = initialTime * \
                                              (1 + mcaIndex % nTimes)
                xpos[mcaIndex] += j
                ypos[mcaIndex] += i
                mcaIndex += 1

        # create the stack data object
        stack = DataObject.DataObject()
        stack.data = data
        stack.info = {}
        stack.info["McaCalib"] = calibration
        stack.info["McaLiveTime"] = live_time
        stack.x = [channels]
        stack.info["positioners"] = {"x": xpos,
                                     "y": ypos}

        tmpDir = tempfile.gettempdir()
        self._h5File = os.path.join(tmpDir, "SteelStack.h5")
        if os.path.exists(self._h5File):
            os.remove(self._h5File)
        McaStackExport.exportStackList(stack, self._h5File)

        # read back the stack
        from PyMca5.PyMcaIO import HDF5Stack1D
        stackRead = HDF5Stack1D.HDF5Stack1D([self._h5File],
                                            {"y":"/measurement/detector_00"})

        # let's play
        sb = StackBase.StackBase()
        sb.setStack(stackRead)

        # positioners
        data = stackRead.info["positioners"]["x"]
        self.assertTrue(numpy.allclose(data, xpos),
                "Incorrect readout of x positions")
        data = stackRead.info["positioners"]["y"]
        self.assertTrue(numpy.allclose(data, ypos),
                "Incorrect readout of y positions")

        # calibration and live time
        x, y, legend, info = sb.getStackOriginalCurve()
        readCalib = info["McaCalib"]
        readLiveTime = info["McaLiveTime"]
        self.assertTrue(abs(readCalib[0] - calibration[0]) < 1.0e-10,
                "Calibration zero. Expected %f got %f" % \
                             (calibration[0], readCalib[0]))
        self.assertTrue(abs(readCalib[1] - calibration[1]) < 1.0e-10,
                "Calibration gain. Expected %f got %f" % \
                             (calibration[1], readCalib[0]))
        self.assertTrue(abs(readCalib[2] - calibration[2]) < 1.0e-10,
                "Calibration 2nd order. Expected %f got %f" % \
                             (calibration[2], readCalib[2]))
        self.assertTrue(abs(live_time.sum() - readLiveTime) < 1.0e-5,
                "Incorrect sum of live time data")

    @unittest.skipIf(not HAS_H5PY, "skipped h5py missing")
    def testSingleArrayExport(self):
        from PyMca5.PyMcaCore import StackBase
        from PyMca5.PyMcaCore import McaStackExport
        tmpDir = tempfile.gettempdir()
        self._h5File = os.path.join(tmpDir, "Array.h5")
        data = numpy.arange(3*1024).reshape(3, 1024)
        McaStackExport.exportStackList([data], self._h5File)
        # read back the stack
        from PyMca5.PyMcaIO import HDF5Stack1D
        stackRead = HDF5Stack1D.HDF5Stack1D([self._h5File],
                                            {"y":"/measurement/detector_00"})
        # let's play
        sb = StackBase.StackBase()
        sb.setStack(stackRead)

        # check the data
        self.assertTrue(numpy.allclose(data, stackRead.data),
                "Incorrect data readout")

def getSuite(auto=True):
    testSuite = unittest.TestSuite()
    if auto:
        testSuite.addTest(unittest.TestLoader().loadTestsFromTestCase(\
            testMcaStackExport))
    else:
        # use a predefined order
        testSuite.addTest(testMcaStackExport("testSingleStackExport"))
        testSuite.addTest(testMcaStackExport("testSingleArrayExport"))
    return testSuite

def test(auto=False):
    return unittest.TextTestRunner(verbosity=2).run(getSuite(auto=auto))

if __name__ == '__main__':
    if len(sys.argv) > 1:
        auto = False
    else:
        auto = True
    result = test(auto)
    sys.exit(not result.wasSuccessful())

{'PyMcaDataDir': 'PyMca5.PyMcaDataDir', 'specfile': 'PyMca5.PyMcaIO.specfilewrapper', 'ConfigDict': 'PyMca5.PyMcaIO.ConfigDict', 'DataObject': 'PyMca5.PyMcaCore.DataObject', 'StackBase': 'PyMca5.PyMcaCore.StackBase', 'McaStackExport': 'PyMca5.PyMcaCore.McaStackExport', 'HDF5Stack1D': 'PyMca5.PyMcaIO.HDF5Stack1D'}


"""
Definitions of basic 'edits' (Actions) to transform a product into synthons and reactants
"""
from abc import ABCMeta, abstractmethod
from typing import Optional, Tuple

from rdkit import Chem
from rdkit.Chem import Mol, rdchem

from utils.chem import attach_lg, fix_Hs_Charge, get_atom_Chiral, get_bond_stereo

MAX_BONDS = {'C': 4, 'N': 3, 'O': 2, 'Br': 1, 'Cl': 1, 'F': 1, 'I': 1}

class ReactionAction(metaclass=ABCMeta):
    def __init__(self, atom_map1: int, atom_map2: int, action_vocab: str):
        self.atom_map1 = atom_map1
        self.atom_map2 = atom_map2
        self.action_vocab = action_vocab

    @abstractmethod
    def get_tuple(self) -> Tuple[str, ...]:
        raise NotImplementedError('Abstract method')

    @abstractmethod
    def apply(self, mol: Mol) -> Mol:
        raise NotImplementedError('Abstract method')

class AtomEditAction(ReactionAction):
    def __init__(self, atom_map1: int, num_explicit_hs: int, chiral_tag: int, action_vocab: str):
        super(AtomEditAction, self).__init__(atom_map1, -1, action_vocab)
        self.num_explicit_hs = num_explicit_hs
        self.chiral_tag = chiral_tag

    @property
    def feat_vals(self) -> Tuple[int, int]:
        return self.num_explicit_hs, self.chiral_tag

    def get_tuple(self) -> Tuple[str, Tuple[int, int]]:
        return self.action_vocab, self.feat_vals

    def apply(self, mol: Mol) -> Mol:
        new_mol = Chem.RWMol(mol)
        amap_idx = {atom.GetAtomMapNum(): atom.GetIdx() for atom in new_mol.GetAtoms()
                    if atom.GetAtomMapNum() != 0}
        atom_idx = amap_idx[self.atom_map1]
        atom = new_mol.GetAtomWithIdx(atom_idx)
        atom.SetNumExplicitHs(self.num_explicit_hs)
        a_chiral = rdchem.ChiralType.values[self.chiral_tag]
        atom.SetChiralTag(a_chiral)
        pred_mol = new_mol.GetMol()
        return pred_mol

    def __str__(self):
        return f'Edit Atom {self.atom_map1}: Num explicit Hs={self.num_explicit_hs}, Chiral_tag={self.chiral_tag}'

class BondEditAction(ReactionAction):
    def __init__(self, atom_map1: int, atom_map2: int,
                 bond_type: Optional[int], bond_stereo: Optional[int],
                 action_vocab: str):
        super(BondEditAction, self).__init__(
            atom_map1, atom_map2, action_vocab)
        self.bond_type = bond_type
        self.bond_stereo = bond_stereo

    def get_tuple(self) -> Tuple[str, Tuple[Optional[int], Optional[int]]]:
        return self.action_vocab, (self.bond_type, self.bond_stereo)

    def apply(self, mol: Mol) -> Mol:
        new_mol = Chem.RWMol(mol)
        amap_idx = {atom.GetAtomMapNum(): atom.GetIdx() for atom in new_mol.GetAtoms()
                    if atom.GetAtomMapNum() != 0}
        atom1 = new_mol.GetAtomWithIdx(amap_idx[self.atom_map1])
        atom2 = new_mol.GetAtomWithIdx(amap_idx[self.atom_map2])

        if self.bond_type is None:  # delete bond
            bond = new_mol.GetBondBetweenAtoms(atom1.GetIdx(), atom2.GetIdx())
            new_mol.RemoveBond(atom1.GetIdx(), atom2.GetIdx())
            pred_mol = new_mol.GetMol()

        else:
            b_type = rdchem.BondType.values[self.bond_type]
            b_stereo = rdchem.BondStereo.values[self.bond_stereo]

            bond = new_mol.GetBondBetweenAtoms(atom1.GetIdx(), atom2.GetIdx())
            b1 = bond.GetBondTypeAsDouble()
            if bond is None:  # add new bond
                pass
            else:  # change an existing bond
                bond.SetBondType(b_type)
                bond.SetStereo(b_stereo)
                b2 = bond.GetBondTypeAsDouble()

                val = b1 - b2
                if val > 0:
                    atom1.SetNumExplicitHs(int(atom1.GetNumExplicitHs() + val))
                    atom2.SetNumExplicitHs(int(atom2.GetNumExplicitHs() + val))

                elif val < 0:
                    atom1.SetNumExplicitHs(
                        int(max(0, atom1.GetNumExplicitHs() + val)))
                    atom2.SetNumExplicitHs(
                        int(max(0, atom2.GetNumExplicitHs() + val)))

                if atom1.GetSymbol() == 'S' and atom2.GetSymbol() == 'O':
                    if b1 == 1.0 and b2 == 2.0 and atom2.GetFormalCharge() == -1:
                        atom2.SetFormalCharge(0)
                    if b1 == 2.0 and b2 == 1.0 and atom2.GetFormalCharge() == 0:
                        atom1.SetNumExplicitHs(0)

                elif atom2.GetSymbol() == 'S' and atom1.GetSymbol() == 'O':
                    if b1 == 1.0 and b2 == 2.0 and atom1.GetFormalCharge() == -1:
                        atom1.SetFormalCharge(0)
                    if b1 == 2.0 and b2 == 1.0 and atom1.GetFormalCharge() == 0:
                        atom2.SetNumExplicitHs(0)

                elif atom1.GetSymbol() == 'C' and atom2.GetSymbol() == 'N':
                    if b1 == 3.0 and b2 == 1.0 and atom1.GetFormalCharge() == -1 and atom2.GetFormalCharge() == 1:
                        atom1.SetFormalCharge(0)
                        atom2.SetFormalCharge(0)
                        atom1.SetNumExplicitHs(1)
                        atom2.SetNumExplicitHs(1)

                elif atom2.GetSymbol() == 'C' and atom1.GetSymbol() == 'N':
                    if b1 == 3.0 and b2 == 1.0 and atom2.GetFormalCharge() == -1 and atom1.GetFormalCharge() == 1:
                        atom1.SetFormalCharge(0)
                        atom2.SetFormalCharge(0)
                        atom1.SetNumExplicitHs(1)
                        atom2.SetNumExplicitHs(1)

            pred_mol = new_mol.GetMol()
            # fix explicit Hs and charge
            pred_mol = fix_Hs_Charge(pred_mol)

        return pred_mol

    def __str__(self):
        if self.bond_type is None:
            return f'Delete bond {self.atom_map1, self.atom_map2}'
        bond_feat = f'Bond type={self.bond_type}, Bond Stereo={self.bond_stereo}'
        return f'{self.action_vocab} {self.atom_map1, self.atom_map2}: {bond_feat}'

class AddGroupAction(ReactionAction):
    def __init__(self, atom_map1: int, leaving_group: str, action_vocab: str):
        super(AddGroupAction, self).__init__(atom_map1, -1, action_vocab)
        self.leaving_group = leaving_group

    def get_tuple(self) -> Tuple[str, str]:
        return self.action_vocab, self.leaving_group

    def apply(self, mol: Mol) -> Mol:
        lg_mol = Chem.MolFromSmiles(self.leaving_group)
        Chem.Kekulize(lg_mol)
        try:
            pred_mol = attach_lg(main_mol=mol, lg_mol=lg_mol,
                                 attach_atom_map=self.atom_map1)
        except Exception as e:
            print('fail to attach lg')
            pred_mol = mol
        # fix explicit Hs and charge
        pred_mol = fix_Hs_Charge(pred_mol)
        return pred_mol

    def __str__(self):
        return f'Attaching {self.leaving_group} to atom {self.atom_map1}'

class Termination(ReactionAction):
    def __init__(self, action_vocab: str):
        super(Termination, self).__init__(-1, -1, action_vocab=action_vocab)

    def get_tuple(self) -> Tuple[str]:
        return self.action_vocab

    def apply(self, mol: Mol) -> Mol:

        atom_chiral = get_atom_Chiral(mol)
        bond_stereo = get_bond_stereo(mol)
        if all(int(bt) == 0 for bt in bond_stereo.values()) and all(int(chiral) == 0 for chiral in atom_chiral.values()):
            mol = Chem.MolFromSmiles(
                Chem.MolToSmiles(mol, isomericSmiles=False))
            return mol

        mol = Chem.MolFromSmiles(Chem.MolToSmiles(mol))

        # Dealing with the inconsistency between molecular mol chirality and atomic chiral tag
        for atom in mol.GetAtoms():
            amap_num = atom.GetAtomMapNum()
            atom.SetChiralTag(atom_chiral[amap_num])

        # Dealing with the inconsistency between molecular mol stereo and bond stereo
        amap_idx = {atom.GetAtomMapNum(): atom.GetIdx()
                    for atom in mol.GetAtoms()}
        for bond in mol.GetBonds():
            a1, a2 = bond.GetBeginAtom().GetAtomMapNum(), bond.GetEndAtom().GetAtomMapNum()
            atom1 = mol.GetAtomWithIdx(amap_idx[a1])
            atom2 = mol.GetAtomWithIdx(amap_idx[a2])
            bond_atoms = sorted([a1, a2])
            st = bond_stereo[tuple(bond_atoms)]

            a1_max_neigh = None
            a2_max_neigh = None
            bond.SetStereo(st)

            if int(st) != 0 and len(list(bond.GetStereoAtoms())) < 2:
                if len([a.GetAtomicNum() for a in atom1.GetNeighbors() if a.GetIdx() != atom2.GetIdx()]) == 0 or len(
                        [a.GetAtomicNum() for a in atom2.GetNeighbors() if a.GetIdx() != atom1.GetIdx()]) == 0:
                    continue
                else:
                    a1_max_neigh_num = max(
                        [a.GetAtomicNum() for a in atom1.GetNeighbors() if a.GetIdx() != atom2.GetIdx()])
                    a2_max_neigh_num = max(
                        [a.GetAtomicNum() for a in atom2.GetNeighbors() if a.GetIdx() != atom1.GetIdx()])

                    for a in atom1.GetNeighbors():
                        if a.GetAtomicNum() == a1_max_neigh_num and a.GetIdx() != atom2.GetIdx():
                            a1_max_neigh = a.GetIdx()
                    for a in atom2.GetNeighbors():
                        if a.GetAtomicNum() == a2_max_neigh_num and a.GetIdx() != atom1.GetIdx():
                            a2_max_neigh = a.GetIdx()

                    if all(a.GetAtomicNum() == a1_max_neigh_num for a in atom1.GetNeighbors() if
                           a.GetIdx() != atom2.GetIdx()) and len(
                            [a.GetAtomicNum() for a in atom1.GetNeighbors() if a.GetIdx() != atom2.GetIdx()]) == 2:
                        a11_max_neigh_num = 0
                        for a in atom1.GetNeighbors():
                            if a.GetIdx() != atom2.GetIdx():
                                if len([a1.GetAtomicNum() for a1 in a.GetNeighbors() if
                                        a1.GetIdx() != atom1.GetIdx()]) == 0:
                                    continue
                                else:
                                    a11_max_neigh_num = max(a11_max_neigh_num,
                                                            max([a1.GetAtomicNum() for a1 in a.GetNeighbors() if
                                                                 a1.GetIdx() != atom1.GetIdx()]))
                        for a in atom1.GetNeighbors():
                            if a.GetIdx() != atom2.GetIdx():
                                for a1 in a.GetNeighbors():
                                    if a1.GetAtomicNum() == a11_max_neigh_num and a1.GetIdx() != atom1.GetIdx():
                                        a1_max_neigh = a.GetIdx()

                    if all(a.GetAtomicNum() == a2_max_neigh_num for a in atom2.GetNeighbors() if
                           a.GetIdx() != atom1.GetIdx()) and len(
                            [a.GetAtomicNum() for a in atom2.GetNeighbors() if a.GetIdx() != atom1.GetIdx()]) == 2:
                        a12_max_neigh_num = 0
                        for a in atom2.GetNeighbors():
                            if a.GetIdx() != atom1.GetIdx():
                                if len([a2.GetAtomicNum() for a2 in a.GetNeighbors() if
                                        a2.GetIdx() != atom2.GetIdx()]) == 0:
                                    continue
                                else:
                                    a12_max_neigh_num = max(a12_max_neigh_num,
                                                            max([a2.GetAtomicNum() for a2 in a.GetNeighbors() if
                                                                 a2.GetIdx() != atom2.GetIdx()]))
                        for a in atom2.GetNeighbors():
                            if a.GetIdx() != atom1.GetIdx():
                                for a2 in a.GetNeighbors():
                                    if a2.GetAtomicNum() == a12_max_neigh_num and a2.GetIdx() != atom2.GetIdx():
                                        a2_max_neigh = a.GetIdx()

                if a1_max_neigh is not None and a2_max_neigh is not None:
                    try:
                        bond.SetStereoAtoms(a1_max_neigh, a2_max_neigh)
                    except:
                        bond.SetStereoAtoms(a2_max_neigh, a1_max_neigh)

        return mol

    def __str__(self):
        return 'Terminate'


"""
Given a function write a decorator which retries
for a certain period of time and breaks when the output is received
"""

from datetime import datetime
from datetime import timedelta
retry_time_period_secs = 10

def retry_time_decorator(fn):
    def inner_fn(*args, **kwargs):
        retry_end_time = datetime.now() + timedelta(seconds=retry_time_period_secs)
        while datetime.now() < retry_end_time:
            if fn(*args, **kwargs):
                print("Found the Value")
                break
            else:
                continue
        print("End of While Loop")
        return fn(*args, **kwargs)
    return inner_fn

@retry_time_decorator
def sample_fn_retry_time():
    """ to check for retry for a certain period of time"""
    return False

sample_fn_retry_time()


from django.urls import path
from . import views

app_name = 'blog'

urlpatterns = [
    path('', views.blogs, name='blogs'),
    path('<slug:slug>', views.blog_detail, name='detail'),
    path('tag/<slug:tag>', views.blog_tag, name='tag'),
    path('category/<slug:category>', views.blog_category, name='category'),
    path('search/', views.search, name='search'),
]


import sys
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtPrintSupport import QPrintDialog, QPrinter
from ui.ui import Ui_Form
from ui.mouse_event import GraphicsScene
import cv2
import numpy as np
from options.test_options import TestOptions
# from model import Model
import os
import time
import torch
import pix2pixHD_model as pix
from models import create_model
COUNT=0
class Ex(QWidget, Ui_Form):
    def __init__(self, model):
        super().__init__()

        self.setupUi(self)
        self.show()
        self.model = model

        self.output_img = None

        self.mat_img = None

        self.ld_mask = None
        self.ld_sk = None

        self.modes = [0,0,0]
        self.mouse_clicked = False
        self.scene = GraphicsScene(self.modes)
        self.graphicsView.setScene(self.scene)
        self.graphicsView.setAlignment(Qt.AlignTop | Qt.AlignLeft)
        self.graphicsView.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        self.graphicsView.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)

        self.result_scene = QGraphicsScene()
        self.graphicsView_2.setScene(self.result_scene)
        self.graphicsView_2.setAlignment(Qt.AlignTop | Qt.AlignLeft)
        self.graphicsView_2.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        self.graphicsView_2.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)

        self.dlg = QColorDialog(self.graphicsView)
        self.color = None

    def mode_select(self, mode):
        for i in range(len(self.modes)):
            self.modes[i] = 0
        self.modes[mode] = 1

    def open(self):
        fileName, _ = QFileDialog.getOpenFileName(self, "Open File",
                QDir.currentPath())
        if fileName:
            image = QPixmap(fileName)
            # print(fileName)
            file_label='test_label/'+fileName.split('/')[-1]
            # print(file_label)
            file_label=file_label.split('.')[0]+'.png'
            self.mat_label=cv2.imread(file_label)
            # print(self.mat_label)

            self.mat_label=cv2.cvtColor(self.mat_label,cv2.COLOR_BGR2GRAY)
            mat_img = cv2.imread(fileName)

            mat_img=cv2.cvtColor(mat_img,cv2.COLOR_BGR2RGB)
            if image.isNull():
                QMessageBox.information(self, "Image Viewer",
                        "Cannot load %s." % fileName)
                return

            # redbrush = QBrush(Qt.red)
            # blackpen = QPen(Qt.black)
            # blackpen.setWidth(5)
            self.image = image.scaled(self.graphicsView.size(), Qt.IgnoreAspectRatio)
            mat_img = mat_img/127.5 - 1
            self.mat_img = np.expand_dims(mat_img,axis=0)
            # cv2.imshow('a',self.mat_img.squeeze())
            # cv2.waitKey(0)
            self.scene.reset()
            if len(self.scene.items())>0:
                self.scene.reset_items()
            self.scene.addPixmap(self.image)
            if len(self.result_scene.items())>0:
                self.result_scene.removeItem(self.result_scene.items()[-1])
            self.result_scene.addPixmap(self.image)

    def mask_mode(self):
        self.mode_select(0)

    def sketch_mode(self):
        self.mode_select(1)

    def stroke_mode(self):
        if not self.color:
            self.color_change_mode()
        self.scene.get_stk_color(self.color)
        self.mode_select(2)

    def color_change_mode(self):
        self.dlg.exec_()
        self.color = self.dlg.currentColor().name()
        self.pushButton_4.setStyleSheet("background-color: %s;" % self.color)
        self.scene.get_stk_color(self.color)

    def complete(self):
        global COUNT
        sketch = self.make_sketch(self.scene.sketch_points)
        sketch=torch.FloatTensor(sketch)
        stroke = self.make_stroke(self.scene.stroke_points)
        stroke=torch.FloatTensor(stroke)
        mask = self.make_mask(self.scene.mask_points)
        mask=torch.FloatTensor(mask)
        self.mat_img=torch.FloatTensor(self.mat_img)
        if not type(self.ld_mask)==type(None):
            ld_mask = np.expand_dims(self.ld_mask[:,:,0:1],axis=0)
            ld_mask[ld_mask>0] = 1
            ld_mask[ld_mask<1] = 0
            mask = mask+ld_mask
            mask[mask>0] = 1
            mask[mask<1] = 0
            mask = np.asarray(mask,dtype=np.uint8)

        if not type(self.ld_sk)==type(None):
            sketch = sketch+self.ld_sk
            sketch[sketch>0]=1

        noise = self.make_noise()
        noise=torch.FloatTensor(noise)
        self.mat_label=torch.FloatTensor(self.mat_label)
        self.mat_label=self.mat_label.reshape(1,512,320,1)

        start_t = time.time()
        result = self.model.inference(self.mat_label.permute(0,3,1,2).cuda()
                                      ,self.mat_img.permute(0,3,1,2).cuda()
                                      ,sketch.permute(0,3,1,2).cuda()
                                      ,stroke.permute(0,3,1,2).cuda()
                                      ,mask.permute(0,3,1,2).cuda()
                                      ,noise.permute(0,3,1,2).cuda())
        result=result*mask.permute(0,3,1,2).cuda()+self.mat_img.permute(0,3,1,2).cuda()*(1-mask.permute(0,3,1,2).cuda())
        end_t = time.time()
        print('inference time : {}'.format(end_t-start_t))
        result = (result+1)*127.5
        result=result.permute(0,2,3,1)
        result=result.cpu().numpy()
        result = np.asarray(result[0,:,:,:],dtype=np.uint8)

        self.output_img = result
        qim = QImage(result.data, result.shape[1], result.shape[0], result.strides[0], QImage.Format_RGB888)
        self.result_scene.removeItem(self.result_scene.items()[-1])
        self.result_scene.addPixmap(QPixmap.fromImage(qim))
        COUNT+=1

    def make_noise(self):
        noise = np.zeros([512, 320, 1],dtype=np.uint8)
        noise = cv2.randn(noise, 0, 255)
        noise = np.asarray(noise/255,dtype=np.uint8)
        noise = np.expand_dims(noise,axis=0)
        return noise

    def make_mask(self, pts):
        if len(pts)>0:
            mask = np.zeros((512,320,3))
            for pt in pts:
                cv2.line(mask,pt['prev'],pt['curr'],(255,255,255),12)
            mask = np.asarray(mask[:,:,0]/255,dtype=np.uint8)
            mask = np.expand_dims(mask,axis=2)
            mask = np.expand_dims(mask,axis=0)
        else:
            mask = np.zeros((512,320,3))
            mask = np.asarray(mask[:,:,0]/255,dtype=np.uint8)
            mask = np.expand_dims(mask,axis=2)
            mask = np.expand_dims(mask,axis=0)
        return mask

    def make_sketch(self, pts):
        if len(pts)>0:
            sketch = np.zeros((512,320,3))
            # sketch = 255*sketch
            for pt in pts:
                cv2.line(sketch,pt['prev'],pt['curr'],(255,255,255),1)
            sketch = np.asarray(sketch[:,:,0]/255,dtype=np.uint8)
            sketch = np.expand_dims(sketch,axis=2)
            sketch = np.expand_dims(sketch,axis=0)
        else:
            sketch = np.zeros((512,320,3))
            # sketch = 255*sketch
            sketch = np.asarray(sketch[:,:,0]/255,dtype=np.uint8)
            sketch = np.expand_dims(sketch,axis=2)
            sketch = np.expand_dims(sketch,axis=0)
        return sketch

    def make_stroke(self, pts):
        if len(pts)>0:
            stroke = np.zeros((512,320,3))
            for pt in pts:
                c = pt['color'].lstrip('#')
                color = tuple(int(c[i:i+2], 16) for i in (0, 2 ,4))
                cv2.line(stroke,pt['prev'],pt['curr'],color,4)
            stroke = stroke/127.5 - 1
            stroke = np.expand_dims(stroke,axis=0)
        else:
            stroke = np.zeros((512,320,3))
            stroke = stroke/127.5 - 1
            stroke = np.expand_dims(stroke,axis=0)
        return stroke

    def arrange(self):
        image = np.asarray((self.mat_img[0]+1)*127.5,dtype=np.uint8)
        image=cv2.cvtColor(image,cv2.COLOR_RGB2BGR)
        if len(self.scene.mask_points)>0:
            for pt in self.scene.mask_points:
                cv2.line(image,pt['prev'],pt['curr'],(255,255,255),12)
        if len(self.scene.stroke_points)>0:
            for pt in self.scene.stroke_points:
                c = pt['color'].lstrip('#')
                color = tuple(int(c[i:i+2], 16) for i in (0, 2 ,4))
                color = (color[2],color[1],color[0])
                cv2.line(image,pt['prev'],pt['curr'],color,4)
        if len(self.scene.sketch_points)>0:
            for pt in self.scene.sketch_points:
                cv2.line(image,pt['prev'],pt['curr'],(0,0,0),1)
        cv2.imwrite('tmp.jpg',image)
        image = QPixmap('tmp.jpg')
        self.scene.history.append(3)
        self.scene.addPixmap(image)

    def save_img(self):
        if type(self.output_img):
            fileName, _ = QFileDialog.getSaveFileName(self, "Save File",
                    QDir.currentPath())
            cv2.imwrite(fileName+'.jpg',self.output_img)

    def undo(self):
        self.scene.undo()

    def clear(self):
        self.scene.reset_items()
        self.scene.reset()
        if type(self.image):
            self.scene.addPixmap(self.image)

if __name__ == '__main__':
    opt = TestOptions().parse(save=False)
    model = create_model(opt)
    app = QApplication(sys.argv)
    ex = Ex(model)
    sys.exit(app.exec_())


import flask
from flask import request
import numpy as np
import pickle

scaler = pickle.load(open('model/scaler.pkl', 'rb'))
model = pickle.load(open('model/model_fp2.pkl', 'rb'))

app = flask.Flask(__name__, template_folder='templates')

@app.route('/')
def main():
    return(flask.render_template('main.html'))
if __name__ == '__main__':
    app.run(debug=True)

@app.route('/predict',methods=['POST'])
def predict():
    '''
    For rendering results on HTML GUI
    '''
    Lokasi = int(request.form['Lokasi'])
    Hari_Ini_Hujan = int(request.form['Hari_Ini_Hujan'])
    Kelembaban_3Sore = float(request.form['Kelembaban_3Sore'])
    Kecepatan_Angin = float(request.form['Kecepatan_Angin'])
    Tekanan_Udara_9Pagi = float(request.form['Tekanan_Udara_9Pagi'])
    predict_list = [[Lokasi, Hari_Ini_Hujan, Kelembaban_3Sore, Kecepatan_Angin,Tekanan_Udara_9Pagi]]
    predict = scaler.transform(predict_list)
    prediction = model.predict(predict)
    output = {0: 'Tidak Hujan', 1: 'Hujan'}
    return flask.render_template('main.html', prediction_text='Prediksi Cuaca Hari Esok yaitu {}'.format(output[prediction[0]]))


import pandas as pd
import matplotlib as mpl
mpl.rcParams['pdf.fonttype'] = 42
mpl.rcParams['ps.fonttype'] = 42
mpl.rc('image',cmap='plasma')

#from glob import glob
#import numpy as np
import matplotlib.pyplot as plt

#import pathlib
import os
import ntpath

fpath = ['..','darknet','videos'] # folder path to csv file
fname = '20190111GOPR9027qtr-yolov3-tiny-litter_10000-th0p1-nms1p0.csv' # name of csv file

def fps_plot(fpathList,fname,platform='unknown'):
    resultName = os.sep.join(fpathList + [platform + '-' + fname[:-4]])

    filePath = os.sep.join(fpathList + [fname])
    df = pd.read_csv(filePath,sep=',',engine='python')
    df['loopDuration'] = df['loopDuration'] * 1000 # convert loop duration to ms

    f = plt.figure()
    ax = f.gca()

    df.plot(x='Time',y=['loopDuration','InferenceTime'],cmap= mpl.cm.plasma,ax=ax)
    ax.set_ylabel('Time in milliseconds',fontsize=16,fontweight='bold')
    ax.set_xlabel('Time in seconds',fontsize=16,fontweight='bold')
    ax.tick_params(axis='both',which='major',labelsize=16)
    ax.legend(fontsize=16,loc='upper center', ncol=2,bbox_to_anchor=(0.5,1.2))
    ax.set_xlim([0, df['Time'].max() * 1.1])
    ax.set_ylim([0, df['loopDuration'].max() * 1.1])

    summaryCols = ['loopDuration','InferenceTime']
    meanSTD = pd.DataFrame(index=['Mean','Standard Deviation'],columns=summaryCols)
    meanSTD.loc['Mean',['loopDuration','InferenceTime']] = df[summaryCols].mean()
    meanSTD.loc['Standard Deviation',summaryCols] = df[summaryCols].std()
    f.savefig(resultName + '.pdf',bbox_inches='tight')
    meanSTD.to_csv(resultName + '.csv')
#    return df,meanSTD
# fpath_fname_platform = [[['..','darknet','videos','20190529-elite8470p'],[csvlist],elite8470p]]
def get_basenames(pathlist):
    basenames = []
    for p in list(pathlist):
        basenames.append(ntpath.basename(p))
    return basenames
def generate_plots(fpath_fname_platform):
    '''
    fpath_fname_platform = [[fpathlist1,fnamelist1,platform1],...]
    fpathlist1 = ['pdir','fdir']
    fnamelist1 = [fname1, fname2, fname3, ...]
    platform1 = platformname
    '''
    for f in fpath_fname_platform:
        fpath,fnames,platform = f
        for fn in fnames:
            fps_plot(fpath,fn,platform=platform)


import torch
import torch.nn as nn
import torch.nn.functional as F

from geneva.definitions.res_blocks import ResDownBlock

class ImageEncoder(nn.Module):
    def __init__(self, cfg):
        """Encodes Image to 16x16 features maps of depth 256
        , return the 16x16 features as well as the global sum
        pooled features(shape=512)"""
        super().__init__()
        self.encode_image = cfg.use_fg

        if self.encode_image:
            if cfg.img_encoder_type == 'res_blocks':
                self.image_encoder = nn.Sequential(
                    # 3 x 128 x 128
                    ResDownBlock(3, 64, downsample=True,
                                 use_spectral_norm=False),
                    # 64 x 64 x 64
                    nn.BatchNorm2d(64),
                    ResDownBlock(64, 128, downsample=True,
                                 use_spectral_norm=False),
                    # 128 x 32 x 32
                    nn.BatchNorm2d(128),
                    ResDownBlock(128, cfg.image_feat_dim,
                                 downsample=True,
                                 use_spectral_norm=False),
                    nn.BatchNorm2d(cfg.image_feat_dim),
                    # 256 x 16 x 16
                )
            elif cfg.img_encoder_type == 'conv':
                self.image_encoder = nn.Sequential(
                    nn.Conv2d(3, 64, 4, 2, 1, bias=False),
                    nn.ReLU(),
                    nn.BatchNorm2d(64),
                    nn.Conv2d(64, 128, 4, 2, 1, bias=False),
                    nn.ReLU(),
                    nn.BatchNorm2d(128),
                    nn.Conv2d(128, cfg.image_feat_dim, 4, 2, 1,
                              bias=False),
                    nn.BatchNorm2d(cfg.image_feat_dim),
                )

            self.object_detector = nn.Linear(cfg.image_feat_dim,
                                             cfg.num_objects)

        self.cfg = cfg

    def forward(self, img):
        if not self.encode_image:
            return None, None, None

        image_features = self.image_encoder(img)
        pooled_features = torch.sum(image_features, dim=(2, 3))

        object_detections = F.sigmoid(self.object_detector(pooled_features))
        return image_features, pooled_features, object_detections


import streamlit as st
import msal_streamlit_authentication

login_token = msal_authentication(
    auth={
        "clientId": "aaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee",
        "authority": "https://login.microsoftonline.com/aaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee",
        "redirectUri": "/",
        "postLogoutRedirectUri": "/"
    }, # Corresponds to the 'auth' configuration for an MSAL Instance
    cache={
        "cacheLocation": "sessionStorage",
        "storeAuthStateInCookie": False
    }, # Corresponds to the 'cache' configuration for an MSAL Instance
    login_request={
        "scopes": ["aaaaaaa-bbbb-cccc-dddd-eeeeeeeeeee/.default"]
    }, # Optional
    logout_request={}, # Optional
    login_button_text="Login", # Optional, defaults to "Login"
    logout_button_text="Logout", # Optional, defaults to "Logout"
    key=1 # Optional if only a single instance is needed
)
st.write("Recevied login token:", login_token)


from __future__ import (
    absolute_import,
    print_function,
    unicode_literals,
    division,
    )

import sys
import functools
import operator
import pandas as pd

import qalsadi.analex
import pyarabic.araby as araby

sys.path.append('../')
sys.path.append('../asmai')
sys.path.append('../support')

import asmai.anasem as asm
import sylajone.anasyn
import sylajone.syn_const as syc

def functools_reduce(a):
    return functools.reduce(operator.concat, a)

def test2():
    import pprint
    text  =  u"يعبد الله منذ أن تطلع الشمس"
    result  =  []
    analex  =  qalsadi.analex.Analex()
    anasyn  =  sylajone.anasyn.SyntaxAnalyzer()
    anasem  =  asm.SemanticAnalyzer()
    result  =  analex.check_text(text)
    result, synodelist  =  anasyn.analyze(result)
    # semantic result
    result  =  anasem.analyze(result)

    # the result contains objets
    df = pd.DataFrame(anasem.decode(result))
    print(df.head())
    df.to_csv("output/test.csv", encoding="utf8", sep="\t")
    #~ anasem.pprint(result)
def test():
    import pprint
    text  =  u"يعبد الله منذ أن تطلع الشمس"
    result  =  []
    anasem  =  asm.SemanticAnalyzer()
    result  =  anasem.analyze_text(text)

    # the result contains objets
    sem_result = anasem.display_sem(result)
    pprint.pprint(sem_result)

def test_pprint():
    import pprint
    text  =  u"يعبد الله منذ أن تطلع الشمس"
    result  =  []
    anasem  =  asm.SemanticAnalyzer()
    result  =  anasem.analyze_text(text)

    # the result contains objets
    anasem.pprint(result)

def test_pd():
    import pprint
    text  =  u"يعبد الله منذ أن تطلع الشمس"
    result  =  []
    anasem  =  asm.SemanticAnalyzer()
    result  =  anasem.analyze_text(text)
    # the result contains objets
    df = pd.DataFrame(anasem.decode(result))
    print(df.head())
    df.to_csv("output/test.csv", encoding="utf8", sep="\t")

def test(display="pprint"):
    import pprint
    text  =  u"يعبد الله منذ أن تطلع الشمس"
    result  =  []
    anasem  =  asm.SemanticAnalyzer()
    result  =  anasem.analyze_text(text)

    if display == "pandas":
        # the result contains objets
        df = pd.DataFrame(anasem.decode(result))
        print(df.head())
        df.to_csv("output/test.csv", encoding="utf8", sep="\t")
    elif display == "pprint":
        # the result contains objets
        anasem.pprint(result)
    elif display == "only":
        # the result contains objets
        sem_result = anasem.display_sem(result)
        pprint.pprint(sem_result)
    elif display == "all":
        # the result contains objets
        sem_result = anasem.display_sem(result, all=True)
        pprint.pprint(sem_result)
    else:
        pprint.pprint(result)

if __name__  ==  '__main__':
    #~ test2()
    #~ display_format = "all"
    #~ display_format = "pandas"
    display_format = "only"
    #~ display_format = "pprint"
    test(display_format)


#!/usr/bin/env python
try:
    from setuptools import setup
    args = {}
except ImportError:
    from distutils.core import setup
    print("""\
*** WARNING: setuptools is not found.  Using distutils...
""")

from setuptools import setup
try:
    from pypandoc import convert
    read_md = lambda f: convert(f, 'rst')
except ImportError:
    print("warning: pypandoc module not found, could not convert Markdown to RST")
    read_md = lambda f: open(f, 'r').read()

from os import path
setup(name='aflow',
      version='0.0.11',
      description='Python API for searching AFLOW database.',
      long_description= "" if not path.isfile("README.md") else read_md('README.md'),
      author='Conrad W Rosenbrock',
      author_email='rosenbrockc@gmail.com',
      url='https://github.com/rosenbrockc/aflow',
      license='MIT',
      setup_requires=['pytest-runner',],
      tests_require=['pytest', 'python-coveralls'],
      install_requires=[
          "termcolor",
          "numpy",
          "six",
          "jinja2",
          "beautifulsoup4",
          "ase"
      ],
      packages=['aflow'],
      scripts=[],
      package_data={'aflow': ['templates/*']},
      include_package_data=True,
      classifiers=[
          'Development Status :: 4 - Beta',
          'Intended Audience :: Science/Research',
          'Natural Language :: English',
          'Operating System :: MacOS',
          'Operating System :: Microsoft :: Windows',
          'Programming Language :: Python',
          'Programming Language :: Python :: 2',
          'Programming Language :: Python :: 2.7',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.5',
      ],
     )


import os
import re
from typing import Set, Sequence

from git import Repo

def all_file_names_which_have_contained_the_lines_in_multiple_files(file_paths: Sequence[str], repo: Repo) -> Set[str]:
    """
    Parses the git log for all lines in multiple files, to determine all the file paths in which these lines
    have existed.

    Useful for tracking renames in a repo.

    :param file_paths: Relative paths to file within repo
    :param repo:
    :return:
    """
    all_names = set()
    for in_file in file_paths:
        all_names.update(
            all_file_names_which_have_contained_the_lines_in_a_file(in_file, repo)
        )
    return all_names

def all_file_names_which_have_contained_the_lines_in_a_file(file_path: str, repo: Repo) -> Set[str]:
    """
    Parses the git log for all lines in a file, to determine all the file paths in which these lines
    have existed.

    Useful for tracking renames in a repo.

    :param file_path: Relative path to file within repo
    :param repo:
    :return:
    """
    full_path = os.path.join(repo.working_tree_dir, file_path)

    if os.path.isdir(full_path):
        # Cannot detect changes directly on a directory
        return set()
    try:
        log = full_git_history_for_contents_of_file(full_path, repo)
    except EmptyFileException:
        return set()

    unique_matches = get_filenames_from_git_log(log)
    return unique_matches

def full_git_history_for_contents_of_file(file_path: str, repo: Repo) -> str:
    """
    Runs git log on all of the lines in a file
    """
    num_lines = file_length(file_path)
    if num_lines == 0:
        raise EmptyFileException('could not track history of lines in an empty file')
    file_search_str = f'1,{num_lines}:{file_path}'
    log = repo.git.log('--format=oneline', '--compact-summary', '-L', file_search_str)
    return log

def get_filenames_from_git_log(git_log: str) -> Set[str]:
    """

    :param git_log:
    :return:
    """
    pattern = re.compile(r'--- a\/(.+)\n\+\+\+ b\/(.+)')
    match_tuples = re.findall(pattern, git_log)
    unique_matches = {file for match_tup in match_tuples for file in match_tup}
    return unique_matches

def file_length(file_path: str) -> int:
    """
    Returns the number of lines in a file
    """
    i = -1
    with open(file_path) as f:
        for i, l in enumerate(f):
            pass
    return i + 1

class EmptyFileException(Exception):
    pass


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('wad_Budget', '0003_remove_budget_internalsudsobject'),
    ]

    operations = [
        migrations.AddField(
            model_name='budget',
            name='internalbooleansync',
            field=models.BooleanField(default=False),
        ),
        migrations.AlterField(
            model_name='budget',
            name='budgetamount',
            field=models.BigIntegerField(default=20000000),
        ),
        migrations.AlterField(
            model_name='budget',
            name='budgetdeliverymethod',
            field=models.CharField(default=b'ACCELERATED', max_length=20, choices=[(b'STANDARD', b'Standard'), (b'ACCELERATED', b'Accelerated'), (b'TESTING', b'Testing')]),
        ),
        migrations.AlterField(
            model_name='budget',
            name='budgetname',
            field=models.CharField(help_text=b'Budget name', unique=True, max_length=255),
        ),
        migrations.AlterField(
            model_name='budget',
            name='budgetstatus',
            field=models.CharField(default=b'ENABLED', max_length=20, choices=[(b'ENABLED', b'Enabled'), (b'PAUSED', b'Paused'), (b'REMOVED', b'Removed'), (b'TESTING', b'Testing')]),
        ),
    ]


import json
dict_a={}
list_a=[]
dict_b={}

dict_b={"msg":"good","star":3}
list_a.append(dict_b.copy())	#add dict to list   .copy() creates a shallow copy
dict_b["msg"]="bad"	#pass by ref gives output as all dict over writes
list_a.append(dict_b)
#print(list_a)

dict_a={"product":"some","price":"12","comment":list_a}	#jaso format
#print(dict_a)
#print(dict_a["comment"][0]["msg"]) #print comment at 0th value at list at mgs
#print(len(dict_b["comment"]))
#for x in dict_a["comment"]:
#	print x

print(dict_a)
data=json.dumps(dict_a); #receive jason string
print(data)
data1=json.loads(data)
print(data1["product"])


import requests
from bs4 import BeautifulSoup
import argparse
import threading

session = requests.Session()  # Initialize a session to use for requests

# Global list to hold found parameters
found_params = []

# Thread lock to handle concurrent writes to the found_params list
param_lock = threading.Lock()

def check_param(target_url, param, method):
    if method.upper() == "GET":
        response = session.get(target_url, params={param: 'test'})
    else:  # POST
        response = session.post(target_url, data={param: 'test'})

    if 'test' in response.text:
        with param_lock:
            found_params.append(param)

def brute_force_params(target_url, wordlist, method):
    threads = []

    with open(wordlist, 'r') as wlist:
        for param in wlist:
            param = param.strip()
            t = threading.Thread(target=check_param, args=(target_url, param, method))
            threads.append(t)
            t.start()

    # Wait for all threads to finish
    for t in threads:
        t.join()

    return found_params

def find_old_parameters(target_url):
    base_url = "http://web.archive.org/cdx/search/cdx"
    params = {
        'url': target_url,
        'output': 'text',
        'fl': 'original',
        'limit': '100'  # fetch last 100 URLs, you can adjust this as needed
    }

    response = requests.get(base_url, params=params)

    if response.status_code == 200:
        urls = response.text.split("\n")
        for url in urls:
            # Extract the parameters
            if "?" in url:
                params_part = url.split('?')[1]
                params = params_part.split('&')
                for param in params:
                    param_name = param.split('=')[0]
                    print(f"Found old parameter: {param_name}")

def analyze_js(target_url):
    response = requests.get(target_url)
    soup = BeautifulSoup(response.text, 'html.parser')

    # Find all linked JS files
    script_tags = soup.find_all('script', src=True)
    for tag in script_tags:
        js_link = tag['src']

        # Ensure the JS link is a full URL
        if not js_link.startswith(('http', 'https')):
            if js_link.startswith('/'):
                js_link = target_url.rstrip('/') + js_link
            else:
                js_link = target_url + '/' + js_link

        js_response = requests.get(js_link)
        js_content = js_response.text

        # A naive approach to extract possible parameters: look for variables or object keys
        # This can be expanded with a proper JS parser or AST analyzer for better results
        possible_params = set()  # using a set to avoid duplicates
        for line in js_content.split('\n'):
            if 'var ' in line:  # basic way to check for variable declaration
                possible_param = line.split('var ')[1].split('=')[0].strip()
                possible_params.add(possible_param)

        # Print found parameters
        for param in possible_params:
            print(f"Possible parameter found in JS: {param}")

def main():
    parser = argparse.ArgumentParser(description='ParamHunter: Discover hidden parameters in web applications')
    parser.add_argument('-u', '--url', help='Target URL', required=True)
    parser.add_argument('-w', '--wordlist', help='Path to wordlist', default='default_wordlist.txt')
    parser.add_argument('-t', '--type', help='Request type (GET or POST)', default='GET')
    parser.add_argument('-a', '--archives', help='Search archives for old parameters', action='store_true')
    parser.add_argument('-j', '--javascript', help='Analyze JS for parameters', action='store_true')
    args = parser.parse_args()

    if args.archives:
        find_old_parameters(args.url)

    if args.javascript:
        analyze_js(args.url)

    brute_force_params(args.url, args.wordlist, args.type)
    print(f"Found parameters: {', '.join(found_params)}")

if __name__ == "__main__":
    main()