from subprocess import *
import yaml
import io
import logging
import os
import os.path

CONFIG_FILE='/opt/config.yaml'
SYSLOG_FIFO='/var/log/syslog-fifo'

def main():
  global config

  logging.getLogger().setLevel(logging.INFO)
  logging.info('Reading config: %s' % CONFIG_FILE)
  with open(CONFIG_FILE, 'r') as f:
    config = yaml.load(f)

  spawn_rsyslogd()
  base_postfix_config()
  configure_postscreen()
  configure_sasl()
  configure_virtual_domains()
  spawn_postsrsd()
  spawn_postfix()

  # Wait for something to exit.  As soon as some child process exits we exit
  os.wait()

def spawn_rsyslogd():
  logging.info('Starting rsyslogd')
  if os.path.exists(SYSLOG_FIFO):
    call(['rm', SYSLOG_FIFO])
  if os.path.exists('/var/run/rsyslogd.pid'):
    call(['rm', '/var/run/rsyslogd.pid'])
  check_call(['mkfifo', SYSLOG_FIFO])
  with open('/etc/rsyslog.conf', 'w') as f:
    f.write(r"""
$ModLoad imuxsock
$template TraditionalFormatWithPRI,"%timegenerated:::date-rfc3339% %pri-text% %HOSTNAME% %syslogtag%%msg:::drop-last-lf%\n"
$WorkDirectory /var/spool/rsyslog
*.* |/var/log/syslog-fifo;TraditionalFormatWithPRI
""".strip())
  Popen(['cat', SYSLOG_FIFO])
  Popen(['rsyslogd', '-n'])

def base_postfix_config():
  # Set up the spool directory so it hass all of the right sub-dirs.
  check_call(['postfix', 'post-install', 'create-missing'])

  # Turn off all the chrooting in postfix. This only works for postfix v2.11+.
  # We grab that from wheezy-backports.
  check_call(['postconf', '-F', '*/*/chroot = n'])

  # Set up a virtual map (placeholder for now)
  check_call(['postconf', '-e', 'myhostname=%s' % config['ptr_hostname']])
  check_call(['postconf', '-e', 'local_recipient_maps='])
  check_call(['postconf', '-e', 'mydomain=localdomain'])
  check_call(['postconf', '-e', 'mydestination='])
  check_call(['postconf', '-e', ('message_size_limit = %d' % (50*1024*1024,))])

  # Enable TLS for smtpd
  check_call(['postconf', '-e', 'smtpd_tls_security_level=may'])
  check_call(['postconf', '-e', 'smtpd_tls_auth_only=yes'])

  # Configure the submission service (587)
  check_call(['postconf', '-Me', 'submission/inet = submission inet n - n - - smtpd'])
  check_call(['postconf', '-Pe', 'submission/inet/syslog_name=postfix/submission'])
  check_call(['postconf', '-Pe', 'submission/inet/smtpd_tls_security_level=encrypt'])
  check_call(['postconf', '-Pe', 'submission/inet/smtpd_sasl_auth_enable=yes'])
  check_call(['postconf', '-Pe', 'submission/inet/smtpd_reject_unlisted_recipient=no'])
  check_call(['postconf', '-Pe', 'submission/inet/smtpd_recipient_restrictions='])
  check_call(['postconf', '-Pe', 'submission/inet/smtpd_relay_restrictions=permit_sasl_authenticated,reject'])
  check_call(['postconf', '-Pe', 'submission/inet/milter_macro_daemon_name=ORIGINATING'])

def configure_postscreen():
  if not config['postscreen']['enable']:
    return

  check_call(['postconf', '-MX', 'smtp/inet'])
  check_call(['postconf', '-Me', 'smtpd/pass = smtpd pass - - n - - smtpd'])
  check_call(['postconf', '-Me', 'smtp/inet = smtp inet n - n - 1 postscreen'])
  check_call(['postconf', '-Me', 'tlsproxy/unix = tlsproxy unix - - n - 0 tlsproxy'])
  check_call(['postconf', '-Me', 'dnsblog/unix = dnsblog unix - - n - 0 dnsblog'])

  # Before-220 tests
  check_call(['postconf', '-e', 'postscreen_dnsbl_action = enforce'])
  check_call(['postconf', '-e', 'postscreen_dnsbl_threshold = 2'])
  check_call(['postconf', '-e', 'postscreen_dnsbl_sites = zen.spamhaus.org*2 bl.spamcop.net*1 b.barracudacentral.org*1 swl.spamhaus.org*-3'])

  check_call(['postconf', '-e', 'postscreen_greet_action = enforce'])

  if config['postscreen']['enable_slow_checks']:
    # After-220 tests -- these can cause significant delays so are disabled for now.
    check_call(['postconf', '-e', 'postscreen_bare_newline_action = enforce'])
    check_call(['postconf', '-e', 'postscreen_bare_newline_enable = yes'])
    check_call(['postconf', '-e', 'postscreen_non_smtp_command_enable = yes'])
    check_call(['postconf', '-e', 'postscreen_pipelining_enable = yes'])

def configure_sasl():
  check_call(['postconf', '-e', 'smtpd_sasl_type = cyrus'])
  check_call(['postconf', '-e', 'smtpd_sasl_auth_enable = yes'])
  check_call(['postconf', '-e', 'broken_sasl_auth_clients = yes'])
  check_call(['adduser', 'postfix', 'sasl'])
  with open('/etc/postfix/sasl/smtpd.conf', 'w') as f:
    f.write("""
pwcheck_method: auxprop
auxprop_plugin: sasldb
mech_list: PLAIN
""".strip())

  for (domain, domain_info) in config['virtual_domains'].items():
    for account in domain_info['accounts']:
      logging.info("Creating SASL account for %s@%s" % (account['name'], domain))
      cmd = ['saslpasswd2', '-p', '-c', '-u', domain, account['name']]
      p = Popen(cmd, stdin=PIPE)
      p.communicate(input=account['password'])
      retcode = p.poll()
      if retcode:
        raise CalledProcessError(retcode, cmd)

def get_forward_list(account):
  forward = account['forward']
  if isinstance(forward, basestring):
    forward = [forward]
  return forward

def configure_virtual_domains():
  domains = config['virtual_domains'].keys()
  check_call(['postconf', '-e', 'virtual_alias_domains=%s' % ' '.join(domains)])
  check_call(['postconf', '-e', 'virtual_alias_maps=hash:/etc/postfix/virtual regexp:/etc/postfix/virtual.regexp'])
  with open('/etc/postfix/virtual', 'w') as f:
    for (domain, domain_info) in config['virtual_domains'].items():
      for account in domain_info['accounts']:
        def forward(alias):
          forward = ' '.join(get_forward_list(account))
          logging.info("Forwarding %s@%s to %s" % (alias, domain, forward))
          f.write('%s@%s %s\n' % (alias, domain, account['forward']))
        forward(account['name'])
        for alias in account.get('aliases', []):
          forward(alias)
      f.write('\n')
  check_call(['postmap', '/etc/postfix/virtual'])

  with open('/etc/postfix/virtual.regexp', 'w') as f:
    for (domain, domain_info) in config['virtual_domains'].items():
      for account in domain_info['accounts']:
        if account.get('dot_plus_rewrite', True):
          name = account['name']
          forwards = ' '.join(["%s+$3@%s" % tuple(forward.split('@')) for forward in get_forward_list(account)])
          logging.info("Forwarding %s.*@%s to %s" % (name, domain, forwards))
          f.write('/^%s((\\+|\\.)([-a-zA-Z0-9_]+))?@%s$/ %s\n' % (name, domain, forwards))

  ai_fn = '/etc/postfix/access_inbound'
  check_call(['postconf', '-e', 'smtpd_sender_restrictions=check_recipient_access hash:%s' % ai_fn])
  with open(ai_fn, 'w') as f:
    for (domain, domain_info) in config['virtual_domains'].items():
      for blackhole in domain_info.get('blackholes', []):
        bh_email = "%s@%s" % (blackhole, domain)
        logging.info("Blackholeing %s" % bh_email)
        f.write('%s REJECT\n' % bh_email)
  check_call(['postmap', ai_fn])

def spawn_postsrsd():
  if not config['srs']['enable']:
    return

  check_call(['postconf', '-e', 'sender_canonical_maps = tcp:127.0.0.1:10001'])
  check_call(['postconf', '-e', 'sender_canonical_classes = envelope_sender'])
  check_call(['postconf', '-e', 'recipient_canonical_maps = tcp:127.0.0.1:10002'])
  check_call(['postconf', '-e', 'recipient_canonical_classes = envelope_recipient,header_recipient'])

  with open('/opt/srs.secret', 'w') as f:
    f.write(config['srs']['srs_secret'])

  exclude_domains=",".join(config['virtual_domains'].keys())

  cmd = ['postsrsd-1.2/build/postsrsd', '-s/opt/srs.secret', '-upostfix',
         '-d%s' % config['srs']['srs_domain'],
         '-X%s' % exclude_domains]
  logging.info("Running postsrsd: %s", repr(cmd))
  Popen(cmd)

def spawn_postfix():
  # Start up the postfix master in the foreground.  Normally this happens with:
  #   /etc/init.d/postfix start ->
  #   /usr/sbin/postfix ->
  #   /usr/lib/postfix/postfix-script ->
  #   /usr/lib/postfix/master
  #
  # The only thing I can figure that *really* matters here is that we change the
  # working directory to the spool directory.  I might be missing some env
  # variables :/
  os.chdir('/var/spool/postfix')
  cmd = ['/usr/lib/postfix/master', '-d']
  logging.info("Running postfix master: %s", repr(cmd))
  Popen(cmd)

if __name__ == "__main__":
  main()


import numpy as np
import pandas as pd

import sklearn
from sklearn.model_selection import KFold

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Convolution2D, MaxPooling2D, BatchNormalization
from tensorflow.keras.models import Sequential
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.random import set_seed

from typing import Tuple, List
from tqdm import tqdm
import os

#Seeding random state to 13 for reproducibility
seed = 13
np.random.seed(seed)
set_seed(seed)

#Declare Callbacks: stop training if accuracy doesn't rise 1% within 3 epochs
callbacks = [
    keras.callbacks.EarlyStopping(
        monitor = "val_loss",
        patience = 6,
        verbose = 1,
        restore_best_weights = True,
        min_delta = 0.01
    )
]

#Helper Function: Return the paths to all jpg files found within a directory
def getImageDirs(root: str = "data"):
    imageDirs = []
    for subDirectory, directory, files in os.walk(root):
        for file in files:
            if file[-4:] == ".jpg":
                path = os.path.join(subDirectory, file)
                imageDirs.append(path)
    return imageDirs

#Helper Function: Return the class weights given a list of classes
def getClassWeightsFromLabels(labels: List[int]):
    weights = sklearn.utils.class_weight.compute_class_weight(class_weight="balanced", classes=np.unique(labels), y=labels)
    return {0: weights[0], 1: weights[1]}

#Helper Function: Return the img paths and classes in seperate lists given a txt file from the LABELS folder
def getDirsAndClasses(root: str, file: str) -> Tuple[List[str], List[int]]:
    imageDirs = []
    classes = []
    line = ""
    with open(root + file, "r") as f:
        for line in tqdm(f):
            imageDir, clazz = line.split()
            imageDirs.append(imageDir)
            classes.append(int(clazz))
    return imageDirs, classes

def makeDataset(pathFromCwdToDataRoot = "Data") -> pd.DataFrame:
    #Get the Train Dataset using split from the LABELS folder
    root = f'{os.getcwd()}/{pathFromCwdToDataRoot}/CNR-EXT-150x150'
    imageDirs, classes = getDirsAndClasses(root, "/LABELS/train.txt")
    patchesRoot = root + "/PATCHES/"
    train = pd.DataFrame([
                {
                    "image": patchesRoot + filename,
                    "class": "free" if clazz == 0 else "busy"
                }
                for filename, clazz in tqdm(zip(imageDirs, classes))
        ])
    #Now Get Test
    imageDirs, classes = getDirsAndClasses(root, "/LABELS/test.txt")
    test = pd.DataFrame([
                {
                    "image": patchesRoot + filename,
                    "class": "free" if clazz == 0 else "busy"
                }
                for filename, clazz in tqdm(zip(imageDirs, classes))
        ])

    # Combine them together to get the full dataset
    dataset = train.append(test)

    return dataset

# Helper Function: Create the k-fold splits and calculate the class weights
def makeFolds(dataset: pd.DataFrame, n_folds: int = 1, batch_size: int = 128, target_size: Tuple[int] = (150, 150)):
    # Choose number of folds (1=normal experiment)
    # Make the k folds
    kFCV_sets=[]

    busy_samples=dataset.loc[dataset["class"] == "busy"]
    free_samples=dataset.loc[dataset["class"] == "free"]

    # If we don't want to perform k-folding
    if n_folds == 1:
        busy = [i for i in range(int(len(busy_train)*0.9))],[i for i in range(int(len(busy_train)*0.9), len(busy_train))]
        free = [i for i in range(int(len(free_train)*0.9))],[i for i in range(int(len(free_train)*0.9), len(free_train))]
    # We do want to perform k-folding
    else:
        busy_kf = sklearn.model_selection.KFold(n_splits = n_folds)
        free_kf = sklearn.model_selection.KFold(n_splits = n_folds)

    # Iterate through the folds and make the data generators for the models to use
    for k in range(n_folds):
        if n_folds != 1:
            busy = next(busy_kf.split(busy_samples), None)
            free = next(busy_kf.split(free_samples), None)
        busy_train, free_train = busy_samples.iloc[busy[0]], free_samples.iloc[free[0]]
        busy_train, busy_val = busy_train[:int(len(busy_train)*0.9)], busy_train[int(len(busy_train)*0.9):]
        free_train, free_val = free_train[:int(len(free_train)*0.9)], free_train[int(len(free_train)*0.9):]

        train = busy_train.append(free_train)
        val = busy_val.append(free_val)
        test = busy_samples.iloc[busy[1]].append(free_samples.iloc[free[1]])

        #Declare data generators and preprocessing
        train_datagen = ImageDataGenerator(
            #Augment data with random flips, normalize each sample's input
            vertical_flip = True,
            horizontal_flip = True,
            rescale = 1.0 / 255.0,
            samplewise_std_normalization = True
        )
        train_generator = train_datagen.flow_from_dataframe(
            directory = None, #none since the df has absolute paths
            dataframe = train,
            x_col = "image",
            y_col = "class",
            validate_filenames = False, #faster for huge datasets
            target_size = target_size,
            color_mode = "rgb",
            batch_size = batch_size,
            class_mode = "binary",
            shuffle = True
        )

        test_datagen = ImageDataGenerator(
            samplewise_std_normalization = True
        )
        test_generator = test_datagen.flow_from_dataframe(
            directory = None,
            dataframe = test,
            x_col = "image",
            y_col = "class",
            validate_filenames = False,
            target_size = target_size,
            color_mode = "rgb",
            batch_size = batch_size,
            class_mode = "binary",
            shuffle = True
        )
        val_generator = test_datagen.flow_from_dataframe(
            directory = None,
            dataframe = val,
            x_col = "image",
            y_col = "class",
            validate_filenames = False,
            target_size = target_size,
            color_mode = "rgb",
            batch_size = batch_size,
            class_mode = "binary",
            shuffle = True
        )

        print()

        kFCV_sets.append([train_generator, test_generator, val_generator])

    #Extract Class Weights (Weights will be the same for all folds)
    classes = list(train["class"])
    weights_dict = getClassWeightsFromLabels(classes)

    return kFCV_sets, weights_dict

#Helper Function: Create a TF/Keras model
def makeModel(inputShape: Tuple[int], modelName:str = '') -> keras.Model:
    model = None

    if modelName == "AlexNet":
        model = makeAlexNet(inputShape=inputShape, modelName=modelName)
    elif modelName in ["InceptionResNetV2","MobileNetV2","ResNet50V2","DenseNet121","DenseNet201","NASNetLarge"]:
        model = makePrebuiltModel(inputShape=inputShape, modelName=modelName)
    elif modelName == "SimpleNet":
        model = makeSimpleNet(inputShape=inputShape, modelName=modelName)
    elif modelName == "SimpleDenseNet":
        model = makeSimpleDenseNet(inputShape=inputShape, modelName=modelName)
    elif modelName == "SimpleResNet":
        model = makeSimpleResNet(inputShape=inputShape, modelName=modelName)
    elif modelName == "mAlexNet":
        model = makemAlexNet(inputShape=inputShape, modelName=modelName)
    elif modelName == "mDenseNet":
        model = makemDenseNet(inputShape=inputShape, modelName=modelName)
    else:
        raise("Unknown model name given. Available names are AlexNet, InceptionResNetV2, MobileNetV2, ResNet50V2, DenseNet121, DenseNet201, NASNetLarge, SimpleNet, SimpleDenseNet, SimpleResNet, mAlexNet, and mDenseNet.")

    return model

##################################################################################################

"""
All of the following functions define the various models that can be used.
Each has the same signature:
------------------------ Params ------------------------
    inputShape: Tuple[int]      The shape of the images
    modelName: str              The name of the model
--------------------------------------------------------
"""

#Helper Function: Create an AlexNet model
def makeAlexNet(inputShape: Tuple[int], modelName: str ='') -> keras.Model:
    #Build Model
    AlexNet = Sequential(name=modelName)

    AlexNet.add(tf.keras.layers.experimental.preprocessing.Resizing(height=224, width=224))

    #1st Convolutional Layer
    AlexNet.add(Convolution2D(filters=96, input_shape=inputShape, kernel_size=(11,11), strides=(4,4), padding='same'))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))
    AlexNet.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    #2nd Convolutional Layer
    AlexNet.add(Convolution2D(filters=256, kernel_size=(5, 5), strides=(1,1), padding='same'))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))
    AlexNet.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    #3rd Convolutional Layer
    AlexNet.add(Convolution2D(filters=384, kernel_size=(3,3), strides=(1,1), padding='same'))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))

    #4th Convolutional Layer
    AlexNet.add(Convolution2D(filters=384, kernel_size=(3,3), strides=(1,1), padding='same'))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))

    #5th Convolutional Layer
    AlexNet.add(Convolution2D(filters=256, kernel_size=(3,3), strides=(1,1), padding='same'))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))
    AlexNet.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    #Passing it to a Fully Connected layer
    AlexNet.add(Flatten())
    # 1st Fully Connected Layer
    AlexNet.add(Dense(4096, input_shape=(32,32,3,)))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))
    # Add Dropout to prevent overfitting
    AlexNet.add(Dropout(0.4))

    #2nd Fully Connected Layer
    AlexNet.add(Dense(4096))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))
    #Add Dropout
    AlexNet.add(Dropout(0.4))

    #3rd Fully Connected Layer
    AlexNet.add(Dense(1000))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('relu'))
    #Add Dropout
    AlexNet.add(Dropout(0.4))

    #Output Layer
    AlexNet.add(Dense(1))
    AlexNet.add(BatchNormalization())
    AlexNet.add(Activation('sigmoid'))

    return AlexNet

#Helper Function: Create a Keras prebuilt model
def makePrebuiltModel(inputShape: Tuple[int], modelName:str ='') -> keras.Model:
    """
    Load model by inputing the name to modelName
    Options are "InceptionResNetV2", "MobileNetV2", "ResNet50V2", "DenseNet121", "DenseNet201", and "NASNetLarge"
    """
    input = keras.Input(shape=inputShape, name="Input")
    x=None
    if modelName=="InceptionResNetV2":
        baseModel = keras.applications.InceptionResNetV2(include_top=False, weights="imagenet", input_shape=inputShape)(input)
    elif modelName=="MobileNetV2":
        baseModel = keras.applications.MobileNetV2(include_top=False, weights="imagenet", input_shape=inputShape)(input)
    elif modelName=="ResNet50V2":
        baseModel = keras.applications.ResNet50V2(include_top=False, weights="imagenet", input_shape=inputShape)(input)
    elif modelName=="DenseNet121":
        baseModel = keras.applications.DenseNet121(include_top=False, weights="imagenet", input_shape=inputShape)(input)
    elif modelName=="DenseNet201":
        baseModel = keras.applications.DenseNet201(include_top=False, weights="imagenet", input_shape=inputShape)(input)
    elif modelName=="NASNetLarge":
        x = tf.keras.layers.experimental.preprocessing.Resizing(height=331, width=331)(input)
        baseModel = keras.applications.NASNetLarge(include_top=False, weights="imagenet", input_shape=(331,331,3))(x)
    else:
        raise("Model Name Not In Recognized Keras Models")

    baseModel.trainable = False
    x = layers.Flatten()(baseModel)
    x = layers.Dense(128, activation="relu")(x)
    x = layers.Dropout(0.3)(x)
    x = layers.Dense(128, activation="relu")(x)
    x = layers.Dropout(0.3)(x)

    if x != None:
        output=layers.Dense(1, activation="sigmoid")(x)
        return keras.Model(inputs=input, outputs=output, name=modelName)
    else:
        raise("Model Name Not Recognized")

#Helper Function: Create a very simple model as a baseline
def makeSimpleNet(inputShape: Tuple[int], modelName:str ='') -> keras.Model:
    input = keras.Input(shape=inputShape, name="Input")
    x = layers.AveragePooling2D(pool_size=(50, 50))(input)
    x = layers.Flatten()(x)
    x = layers.Dense(64, activation="relu")(x)
    x = layers.Dropout(0.3)(x)
    x = layers.Dense(64, activation="relu")(x)
    x = layers.Dropout(0.3)(x)
    output=layers.Dense(1, activation="sigmoid")(x)
    return keras.Model(inputs=input, outputs=output, name=modelName)

#Helper Function: Create a mini AlexNet
def makemAlexNet(inputShape: Tuple[int], modelName:str ='') -> keras.Model:
    inputs = keras.Input(shape=inputShape, name="Input")

    x = tf.keras.layers.experimental.preprocessing.Resizing(height=224, width=224)(inputs)
    x = layers.Conv2D(filters=16, kernel_size=11, strides=4, activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=3, strides=2)(x)
    x = layers.Conv2D(filters=20, kernel_size=5, strides=1, activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=3, strides=2)(x)
    x = layers.Conv2D(filters=30, kernel_size=3, strides=1, activation="relu")(x)
    x = layers.MaxPooling2D(pool_size=3, strides=2)(x)
    x = layers.Dense(units=48, activation="relu")(x)
    x = layers.Flatten()(x)
    outputs = layers.Dense(units=1, activation="sigmoid")(x)

    model = keras.Model(inputs=inputs, outputs=outputs, name=modelName)
    return model

#Helper Function: Create a mini DenseNet
def makemDenseNet(inputShape: Tuple[int], modelName:str ='') -> keras.Model:
    def makeDenseBlock(groupCount: int, inputs):
        blockConcats = []
        x = layers.BatchNormalization()(inputs)
        x = layers.Conv2D(filters=16, kernel_size=(1, 1), activation="relu", padding="same")(x)
        x = layers.Conv2D(filters=16, kernel_size=(3, 3), activation="relu", padding="same")(x)
        blockConcats.append(x)
        for count in range(groupCount):
            x = layers.Concatenate()(blockConcats) if len(blockConcats) > 1 else x
            x = layers.BatchNormalization()(x)
            x = layers.Conv2D(filters=16, kernel_size=(1, 1), activation="relu", padding="same")(x)
            x = layers.Conv2D(filters=16, kernel_size=(3, 3), activation="relu", padding="same")(x)
            blockConcats.append(x)
        x = layers.BatchNormalization()(x)
        x = layers.Conv2D(filters=16, kernel_size=(1, 1), activation="relu", padding="same")(x)
        x = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
        return x

    inputs = keras.Input(shape=inputShape, name="Input")
    x = layers.Conv2D(filters=16, kernel_size=(7, 7), strides=(5, 5), activation="relu")(inputs)
    x = layers.MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(x)

    x = makeDenseBlock(groupCount=2, inputs=x)

    x = layers.GlobalAveragePooling2D()(x)
    outputs = layers.Dense(units=1, activation="sigmoid")(x)

    return keras.Model(inputs=inputs, outputs=outputs, name=modelName)

#Helper Function: Create a simple DenseNet
def makeSimpleDenseNet(inputShape: Tuple[int], modelName:str ='') -> keras.Model:
    def makeDenseBlock(groupCount: int, inputs):
        blockConcats = []
        x = layers.BatchNormalization()(inputs)
        x = layers.Conv2D(filters=64, kernel_size=(1, 1), activation="relu", padding="same")(x)
        x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="same")(x)
        blockConcats.append(x)
        for count in range(groupCount):
            x = layers.Concatenate()(blockConcats) if len(blockConcats) > 1 else x
            x = layers.BatchNormalization()(x)
            x = layers.Conv2D(filters=64, kernel_size=(1, 1), activation="relu", padding="same")(x)
            x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="same")(x)
            blockConcats.append(x)
        x = layers.BatchNormalization()(x)
        x = layers.Conv2D(filters=64, kernel_size=(1, 1), activation="relu", padding="same")(x)
        x = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
        return x

    inputs = keras.Input(shape=inputShape, name="Input")
    x = layers.Conv2D(filters=32, kernel_size=(7, 7), strides=(2, 2), activation="relu")(inputs)
    x = layers.MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(x)

    x = makeDenseBlock(groupCount=6, inputs=x)

    x = layers.GlobalAveragePooling2D()(x)
    outputs = layers.Dense(units=1, activation="sigmoid")(x)

    return keras.Model(inputs=inputs, outputs=outputs, name=modelName)

#Helper Function: Create a simple ResNet
def makeSimpleResNet(inputShape: Tuple[int], modelName:str ='') -> keras.Model:
    """
    Source: https://www.tensorflow.org/guide/keras/functional#a_toy_resnet_model
    """
    inputs = keras.Input(shape=inputShape, name="Input")
    x = layers.Conv2D(filters=32, kernel_size=(3, 3), activation="relu")(inputs)
    x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu")(x)
    block_1_output = layers.MaxPooling2D(pool_size=(3, 3), strides=(3, 3))(x)

    x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="same")(block_1_output)
    x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="same")(x)
    block_2_output = layers.add([x, block_1_output])

    x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="same")(block_2_output)
    x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="same")(x)
    block_3_output = layers.add([x, block_2_output])

    x = layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu")(block_3_output)
    x = layers.GlobalAveragePooling2D()(x)
    x = layers.Dense(units=256, activation="relu")(x)
    x = layers.Dense(units=256, activation="relu")(x)
    x = layers.Dense(units=256, activation="relu")(x)
    x = layers.Dropout(0.5)(x)
    outputs = layers.Dense(units=1, activation="sigmoid")(x)

    return keras.Model(inputs=inputs, outputs=outputs, name=modelName)


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 28 02:48:14 2017

@author: Will
"""

from RootsTool import  Point3d, RootAttributes, Skeleton, MetaNode3d, MetaEdge3d, MetaGraph

from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *

from PyQt5.QtWidgets import QMainWindow
from PyQt5 import QtWidgets
from SkeletonViewer import *

import sys
from RootsUI import Ui_RootsUI
from ConnectionTabWidget import Ui_ConnectionTabWidget
from VisualizationTabWidget import Ui_VisualizationTabWidget
from BreakTabWidget import Ui_BreakTabWidget
from SplitTabWidget import Ui_SplitTabWidget

import test_glviewer as tgl
from GLObjects import MetaGraphGL, Colorization

from MetaGraphThread import MetaGraphThread
from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QThread
import types

#class RootsGUI(Ui_RootsUI):
#    def __init__(self, dialog):
#        Ui_RootsUI.__init__(self)
#        self.setupUi(dialog)
#        self.dlg = dialog
#        self.LoadFile.clicked.connect(self.PickFile)
##        layout = self.GLWidgetHolder.layout
#        layout = QVBoxLayout(self.GLWidgetHolder)
#        layout.addStretch(1)
#        self.GLView = tgl.GLWidget()
##        self.GLView = SkeletonViewer()
#        layout.addChildWidget(self.GLView)
#
#        self.GLWidgetHolder.setLayout(layout)
#        self.GLWidgetHolder.show()
#        #self.GLView = SkeletonViewer(self.GLWidgetHolder)
#
#    def PickFile(self):
#        options = QFileDialog.Options()
#
#        options |= QFileDialog.DontUseNativeDialog
#        self.loadFileName = QFileDialog.getOpenFileName(self.dlg, 'Open File', "")
#        qDebug(str(self.loadFileName[0]))
#        self.LoadSkeleton(str(self.loadFileName[0]))
#
#    def LoadSkeleton(self, filename):
#        self.skeleton = Skeleton(filename)
#        self.GLView.setSkeleton(self.skeleton)

class ConnectionTabWidget(Ui_ConnectionTabWidget):
    def __init__(self, widget=None):
        Ui_ConnectionTabWidget.__init__(self)
        self.setupUi(widget)

class VisualizationTabWidget(Ui_VisualizationTabWidget, QObject):

    lowColorChanged = pyqtSignal(object, object, object)
    highColorChanged = pyqtSignal(object, object, object)

    viewSkeleton = pyqtSignal(bool)
    viewGraph = pyqtSignal(bool)
    viewBoth = pyqtSignal(bool)

    @pyqtSlot(int)
    def edgeColorizationChanged(self, optionId : int):
        if optionId == 0: #thickness
            self.graph.colorizeThickness()
            pass
        elif optionId == 1: #width
            self.graph.colorizeWidth()
            pass
        elif optionId == 2: #thickness / width
            self.graph.colorizeRatio()
            pass
        elif optionId ==3 : #component
            self.graph.colorizeComponents()
            pass

    @pyqtSlot(int)
    def nodeColorizationChanged(self, optionId : int):
        if optionId == 0: #thickness
            self.graph.nodeColorization = Colorization.THICKNESS
            pass
        elif optionId == 1: #width
            self.graph.nodeColorization = Colorization.WIDTH
            pass
        elif optionId == 2: #degree
            self.graph.nodeColorization = Colorization.DEGREE
            pass
        elif optionId ==3 : #component
            self.graph.nodeColorization = Colorization.COMPONENTS
            pass

    @pyqtSlot(int)
    def geometryVisualizationChanged(self, optionId : int):
        if optionId == 0: #skeleton
            self.viewSkeleton.emit(True)
            pass
        elif optionId == 1: #graph
            self.viewGraph.emit(True)
            pass
        elif optionId == 2: #both
            self.viewBoth.emit(True)
            pass

    @pyqtSlot(bool)
    def enteringSkelView(self, val : bool):
        self.geometryVisualization.currentIndexChanged.disconnect(self.geometryVisualizationChanged)
        self.geometryVisualization.setCurrentIndex(0)
        self.geometryVisualization.currentIndexChanged.connect(self.geometryVisualizationChanged)

    @pyqtSlot(bool)
    def enteringGraphView(self, val : bool):
        self.geometryVisualization.currentIndexChanged.disconnect(self.geometryVisualizationChanged)
        self.geometryVisualization.setCurrentIndex(1)
        self.geometryVisualization.currentIndexChanged.connect(self.geometryVisualizationChanged)

    @pyqtSlot(bool)
    def enteringBothView(self, val : bool):
        self.geometryVisualization.currentIndexChanged.disconnect(self.geometryVisualizationChanged)
        self.geometryVisualization.setCurrentIndex(2)
        self.geometryVisualization.currentIndexChanged.connect(self.geometryVisualizationChanged)

    def __init__(self, widget, graphObject : MetaGraphGL, viewSkeletonButton, viewGraphButton, viewBothButton):
        Ui_VisualizationTabWidget.__init__(self)
        QObject.__init__(self)
        self.setupUi(widget)
        self.widget = widget
        self.graph = graphObject

        self.lowColor = QColor(Qt.blue)
        self.highColor = QColor(Qt.red)

        self.lowColorButton.setAutoFillBackground(True)
        self.highColorButton.setAutoFillBackground(True)

        self.setLowColor(self.lowColor)
        self.setHighColor(self.highColor)

        self.lowColorButton.clicked.connect(self.pickLowColor)
        self.highColorButton.clicked.connect(self.pickHighColor)

        self.edgeColorizationOptions = {}
        self.edgeColorizationOptions[0] = "by Thickness"
        self.edgeColorizationOptions[1] = "by Width"
        self.edgeColorizationOptions[2] = "by Thickness/Width"
        self.edgeColorizationOptions[3] = "by Component"

        self.nodeColorizationOptions = {}
        self.nodeColorizationOptions[0] = "by Thickness"
        self.nodeColorizationOptions[1] = "by Width"
        self.nodeColorizationOptions[2] = "by Degree"
        self.nodeColorizationOptions[3] = "by Component"

        self.geometryVisualizationOptions = {}
        self.geometryVisualizationOptions[0] = "View Skeleton"
        self.geometryVisualizationOptions[1] = "View MetaGraph"
        self.geometryVisualizationOptions[2] = "View Both"

        for key in self.edgeColorizationOptions:
            self.edgeColorization.addItem(self.edgeColorizationOptions[key])

        for key in self.nodeColorizationOptions:
            self.nodeColorization.addItem(self.nodeColorizationOptions[key])

        for key in self.geometryVisualizationOptions:
            self.geometryVisualization.addItem(self.geometryVisualizationOptions[key])

        self.edgeColorization.currentIndexChanged.connect(self.edgeColorizationChanged)
        self.nodeColorization.currentIndexChanged.connect(self.nodeColorizationChanged)
        self.geometryVisualization.currentIndexChanged.connect(self.geometryVisualizationChanged)

        self.viewGraph.connect(viewGraphButton.trigger)
        graphObject.enteringGraphView.connect(self.enteringGraphView)

        self.viewSkeleton.connect(viewSkeletonButton.trigger)
        graphObject.enteringSkeletonView.connect(self.enteringSkelView)

        self.viewBoth.connect(viewBothButton.trigger)
        graphObject.enteringBothView.connect(self.enteringBothView)

        self.lowColorChanged.connect(graphObject.lowColorChanged)
        self.highColorChanged.connect(graphObject.highColorChanged)

        self.lowColorChanged.emit(self.lowColor.redF(), self.lowColor.greenF(), self.lowColor.blueF())
        self.highColorChanged.emit(self.highColor.redF(), self.highColor.greenF(), self.highColor.blueF())

        self.graph.colorizeThickness()
        self.viewSkeleton.emit(True)

    @pyqtSlot(bool)
    def pickLowColor(self, someBool):
        pickedColor = QColorDialog.getColor(self.lowColor, self.widget)
        self.setLowColor(pickedColor)

    @pyqtSlot(bool)
    def pickHighColor(self, someBool):
        pickedColor = QColorDialog.getColor(self.highColor, self.widget)
        self.setHighColor(pickedColor)

    @pyqtSlot(QColor)
    def setLowColor(self, lowColor : QColor):
        self.lowColor = lowColor

        lowColorHSL = self.lowColor.toHsl()
        lowLightness = lowColorHSL.lightnessF()
        lowTextColor = QColor(Qt.black)
        if lowLightness < 0.1791:
            lowTextColor = QColor(Qt.white)

        self.lowColorButton.setStyleSheet( "background-color: " + self.lowColor.name() + "; color:" + lowTextColor.name())
        self.lowColorButton.update()
        self.lowColorChanged.emit(self.lowColor.redF(), self.lowColor.greenF(), self.lowColor.blueF())

    @pyqtSlot(QColor)
    def setHighColor(self, highColor : QColor):
        self.highColor = highColor

        highColorHSL = self.highColor.toHsl()
        highLightness = highColorHSL.lightnessF()
        highTextColor = QColor(Qt.black)
        if highLightness < 0.1791:
            highTextColor = QColor(Qt.white)

        self.highColorButton.setStyleSheet( "background-color: " + self.highColor.name() + "; color:" + highTextColor.name())
        self.highColorButton.update()

        self.highColorChanged.emit(self.highColor.redF(), self.highColor.greenF(), self.highColor.blueF())

class BreakTabWidget(Ui_BreakTabWidget, QObject):
    def __init__(self, widget=None):
        Ui_BreakTabWidget.__init__(self)
        QObject.__init__(self)
        self.setupUi(widget)
        self.widget = widget

class SplitTabWidget(Ui_SplitTabWidget, QObject):
    def __init__(self, widget=None):
        Ui_SplitTabWidget.__init__(self)
        QObject.__init__(self)
        self.setupUi(widget)

        self.widget = widget

class RootsTabbedProgram(QMainWindow):

    @pyqtSlot()
    def notifyConfirmed(self):
        print('Notify confirmed')

    @pyqtSlot()
    def terminateConfirmed(self):
        print('Terminate confirmed')

    @pyqtSlot()
    def mainPrint(self, toPrint : object):
        print(toPrint)

    @pyqtSlot()
    def acceptPressed(self):
        print('accept pressed')

    def __init__(self, parent = None):
        super(RootsTabbedProgram, self).__init__(parent)

        #self.metaThread = MetaGraphThread()
        #self.metaThread.wasNotified.connect(self.notifyConfirmed)
        #self.metaThread.wasTerminated.connect(self.terminateConfirmed)
        #self.metaThread.start()

        self.currentMode = -2

        self.glwidget = tgl.GLWidget(self)
        self.dockedWidget = None
        self.__setUI()

    def __setUI(self, title="RootsEditor"):
        self.mainMenu = self.menuBar()

        self.mainMenu.setNativeMenuBar(False)
        self.fileMenu = self.mainMenu.addMenu('File')

        loadButton = QAction('Load rootfile', self)
        loadButton.setShortcut('Ctrl+L')
        loadButton.setShortcutContext(Qt.ApplicationShortcut)
        loadButton.setStatusTip('Load rootfile or skeleton')
        loadButton.triggered.connect(self.loadFile)
        self.fileMenu.addAction(loadButton)

        exitButton = QAction('Exit', self)
        exitButton.setShortcut('Ctrl+E')
        exitButton.setShortcutContext(Qt.ApplicationShortcut)
        exitButton.setStatusTip('Exit RootsEditor')
        exitButton.triggered.connect(self.close)
        self.fileMenu.addAction(exitButton)

        self.modeMenu = self.mainMenu.addMenu('Mode')

        connectionModeButton = QAction('Connection', self)
        connectionModeButton.setShortcut('Ctrl+C')
        connectionModeButton.setShortcutContext(Qt.ApplicationShortcut)
        connectionModeButton.setStatusTip('Connect broken components')
        connectionModeButton.triggered.connect(self.enterConnectionMode)
        self.modeMenu.addAction(connectionModeButton)

        breakModeButton = QAction('Break', self)
        breakModeButton.setShortcut('Ctrl+B')
        breakModeButton.setShortcutContext(Qt.ApplicationShortcut)
        breakModeButton.setStatusTip('Break invalid edges')
        breakModeButton.triggered.connect(self.enterBreakMode)
        self.modeMenu.addAction(breakModeButton)

        splitModeButton = QAction('Split', self)
        splitModeButton.setShortcut('Ctrl+X')
        splitModeButton.setShortcutContext(Qt.ApplicationShortcut)
        splitModeButton.setStatusTip('Split edges between two branches that have merged')
        splitModeButton.triggered.connect(self.enterSplitMode)
        self.modeMenu.addAction(splitModeButton)

        self.viewMenu = self.mainMenu.addMenu('View Mode')

        viewGraphButton = QAction('MetaGraph', self)
        viewGraphButton.setShortcut('Ctrl+G')
        viewGraphButton.setShortcutContext(Qt.ApplicationShortcut)
        viewGraphButton.setStatusTip('View the metagpraph only')
        viewGraphButton.triggered.connect(self.glwidget.metaGL.enterGraphView)
        self.viewMenu.addAction(viewGraphButton)

        viewSkeletonButton = QAction('Skeleton', self)
        viewSkeletonButton.setShortcut('Ctrl+R')
        viewSkeletonButton.setShortcutContext(Qt.ApplicationShortcut)
        viewSkeletonButton.setStatusTip('View the skeleton only')
        viewSkeletonButton.triggered.connect(self.glwidget.metaGL.enterSkeletonView)
        self.viewMenu.addAction(viewSkeletonButton)

        viewBothButton = QAction('Both', self)
        viewBothButton.setShortcut('Ctrl+D')
        viewBothButton.setShortcutContext(Qt.ApplicationShortcut)
        viewBothButton.setStatusTip('View metagraph and skeleton simultaneously')
        viewBothButton.triggered.connect(self.glwidget.metaGL.enterBothView)
        self.viewMenu.addAction(viewBothButton)

        recenterButton = QAction('Recenter', self)
        recenterButton.setShortcut('Ctrl+F')
        recenterButton.setShortcutContext(Qt.ApplicationShortcut)
        recenterButton.setStatusTip('Recenter view on skeleton')
        recenterButton.triggered.connect(self.glwidget.recenter)
        self.viewMenu.addAction(recenterButton)

        acceptShortcut = QAction('Accept Operation', self)
        acceptShortcut.setShortcut('Ctrl+A')
        acceptShortcut.setShortcutContext(Qt.ApplicationShortcut)
        acceptShortcut.triggered.connect(self.glwidget.metaGL.connectionOptions.acceptConnection)
        acceptShortcut.triggered.connect(self.glwidget.metaGL.breakOptions.acceptBreak)
        acceptShortcut.triggered.connect(self.glwidget.metaGL.splitOptions.acceptSplit)
        acceptShortcut.triggered.connect(self.acceptPressed)
        self.addAction(acceptShortcut)

        centralWidget = QtWidgets.QWidget()

        self.setCentralWidget(centralWidget)
        centralLayout = QtWidgets.QGridLayout()
        centralLayout.addWidget(self.glwidget, 1, 1)
        centralWidget.setLayout(centralLayout)
        centralWidget.setSizePolicy(QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding))
        self.glwidget.setSizePolicy(QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding))
        self.glwidget.setMinimumSize(800, 500)

        #notifyButton = QAction('Notify', self)
        #notifyButton.setShortcut('Ctrl+N')
        #notifyButton.triggered.connect(self.metaThread.notify)
        #self.modeMenu.addAction(notifyButton)

        #terminateButton = QAction('Terminate', self)
        #terminateButton.setShortcut('Ctrl+T')
        #terminateButton.triggered.connect(self.metaThread.terminate)
        #self.modeMenu.addAction(terminateButton)

        w = 1300
        h = 800
        self.resize(w, h)
        self.installEventFilter(self)
        self.setFocusPolicy(Qt.StrongFocus)
        self.setWindowTitle('Skeleton Viewer')
        self.dockedWidget = None
        self.viewWidget = None

        dock = QDockWidget('Visualization Tab', self)
        dock.setAllowedAreas(QtCore.Qt.BottomDockWidgetArea)
        dockWidget = QWidget()
        VisualizationTab = VisualizationTabWidget(dockWidget, self.glwidget.metaGL, viewSkeletonButton, viewGraphButton, viewBothButton)

        self.VisualizationTab = VisualizationTab

        dock.setWidget(dockWidget)
        dock.setTitleBarWidget(QWidget())

        self.addDockWidget(QtCore.Qt.BottomDockWidgetArea, dock)
        self.visualizationDock = dock

    def enterConnectionMode(self):
        if self.currentMode == 0 or self.currentMode == -2:
            return
        self.currentMode = 0
        self.closeDockWidget()
        dock = QDockWidget('Connection Tab', self)
        dock.setAllowedAreas(QtCore.Qt.RightDockWidgetArea)
        dockWidget = QWidget()
        ConnectionTab = ConnectionTabWidget(dockWidget)

        dock.setWidget(dockWidget)
        dock.setTitleBarWidget(QWidget())
        self.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)
        self.glwidget.enterConnectionMode(ConnectionTab)
        self.dockedWidget = dock

    def enterBreakMode(self):
        if self.currentMode == 1 or self.currentMode == -2:
            return
        self.closeDockWidget()
        self.currentMode = 1

        self.closeDockWidget()
        dock = QDockWidget('Break Tab', self)
        dock.setAllowedAreas(QtCore.Qt.RightDockWidgetArea)
        dockWidget = QWidget()
        BreakTab = BreakTabWidget(dockWidget)

        dock.setWidget(dockWidget)
        dock.setTitleBarWidget(QWidget())
        self.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)
        self.glwidget.enterBreakMode(BreakTab)
        self.dockedWidget = dock

    def enterSplitMode(self):
        if self.currentMode == 2 or self.currentMode == -2:
            return
        self.closeDockWidget()
        self.currentMode = 2

        self.closeDockWidget()
        dock = QDockWidget('Connection Tab', self)
        dock.setAllowedAreas(QtCore.Qt.RightDockWidgetArea)
        dockWidget = QWidget()
        SplitTab = SplitTabWidget(dockWidget)

        dock.setWidget(dockWidget)
        dock.setTitleBarWidget(QWidget())
        self.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)
        self.glwidget.enterSplitMode(SplitTab)
        self.dockedWidget = dock

    def eventFilter(self, obj, event):
        return False

    def closeDockWidget(self):
        print('closing dock widget')
        if self.dockedWidget != None:
            print('dock widget is not none')
            self.dockedWidget.hide()
            self.dockedWidget.destroy()

    def loadFile(self):
        options = QFileDialog.Options()

        options |= QFileDialog.DontUseNativeDialog
        self.loadFileName = QFileDialog.getOpenFileName(self, 'Open File', "")
#        qDebug(str(self.loadFileName[0]))

        if self.loadFileName[0] != "":
            self.glwidget.loadFileEvent(str(self.loadFileName[0]))
            self.currentMode = -1
            #self.metaThread.loadFileEvent(str(self.loadFileName[0]))

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = RootsTabbedProgram()
    window.show()
#    dialog = QDialog()
#    prog = RootsGUI(dialog)

#    dialog.show()
    sys.exit(app.exec_())


import requests
from bs4 import BeautifulSoup

url = 'http://photo.poco.cn/lastphoto-htx-id-5748659-p-0.xhtml#'

wb_data = requests.get(url)

soup = BeautifulSoup(wb_data.text,'lxml')
imgs = soup.select('div.img-item > div.img-box.photo')
print(imgs)


import sounddevice
from scipy.io.wavfile import write

fs = 44100
second = 1

print(sounddevice.query_devices())
sounddevice.default.device = 1

for i in range(2):
    print("recorde")
    record_voice = sounddevice.rec( int( second * fs ) , samplerate = fs , channels = 1 )
    sounddevice.wait()
    write("out" + str(i) + ".wav", fs , record_voice )


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 17 00:29:53 2020

@author: gao
"""

import numpy as np
import numpy_financial as npf

#%%
def cal_interets(principal, rate, n, interest_type='simple'):
    '''
    计算本息和
    ---------
    Params:
    ------
    principal: -> float 本金
    rate: -> float 利率
    n: -> float 年份
    interest_type: -> str 单利(simple)|复利(compound)|连续复利(continuous)

    Returns:
    -------
    future value : -> float 终值
    '''
    if interest_type == 'simple':
        final = principal * (1 + rate * n)
    elif interest_type == 'compound':
        final = principal * (1 + rate) ** n
    elif interest_type == 'continuous':
        final = principal * np.exp(rate * n)
    else:
        raise Exception('Invalid Type!')
    return final

#%%
# 计算终值
npf.fv(0.1, 4, 0, -1000)

# 计算现值
npf.pv(0.1, 4, 0, -1500)

# 计算净现值
rate, cashflows = 0.08, [-40_000, 5_000, 8_000, 12_000, 30_000]
npf.npv(rate, cashflows).round(5)

#%%
npf.pv(0.06, 3, -100, 0)
npf.pv(0.06, 3, -100, 0, when='begin')

# 验证
npf.pv(0.06, 3, 100, 0, when='begin') / npf.pv(0.06, 3, 100, 0) == 1.06

npf.fv(0.06, 3, -100, 0)
npf.fv(0.06, 3, -100, 0, when='begin')

# 验证
npf.fv(0.06, 3, 100, 0, when='begin') / npf.fv(0.06, 3, 100, 0) == 1.06

#%%
round(npf.irr([-100, 39, 59, 55, 20]), 5)
round(npf.irr([-100, 0, 0, 74]), 5)
round(npf.irr([-100, 100, 0, -7]), 5)
round(npf.irr([-100, 100, 0, 7]), 5)
round(npf.irr([-5, 10.5, 1, -8, 1]), 5)


import pycurl
from io import BytesIO

def curlRequest(s):
    buffer = BytesIO()
    print(type(s))
    print(s)
    s = s.replace(" ", "%20")
    s = s.replace("'", "%E2%80%99")
    s = s.replace("\t", "")
    request = "http://localhost:2222/rest/annotate?text=" + s.encode("ascii", errors="ignore").decode()
    # request = "https://api.dbpedia-spotlight.org/en/annotate?text=" + b;
    print(request)
    crl = pycurl.Curl()
    crl.setopt(crl.URL, request)
    crl.setopt(pycurl.HTTPHEADER, ['accept: application/json'])
    crl.setopt(pycurl.CUSTOMREQUEST, "GET")
    crl.setopt(crl.WRITEDATA, buffer)
    crl.perform()
    crl.close()
    result = buffer.getvalue().decode('utf-8')
    print(result)
    print(type(result))
    dict = {}
    try:
        result = result[result.index("[")+1:result.index("]"):1].split("},")
        key = ""
        value = ""
        for single_result in result:
            items = single_result.split("\",")
            for item in items:
                if (item.startswith("\"@surfaceForm")):
                    key = item[item.index(":") + 2:]
                    print(key)
                if (item.startswith("{\"@URI")):
                    value = item[item.index(":") + 2:]
                    print(value)
            dict[key] = value
    except ValueError:
        pass

    for x, y in dict.items():
        print(x, y)

    return dict

# curlRequest("Concordia advanced six spots to 10th place among Canada's engineering schools in the Maclean's 2018 Program Rankings, while computer science advanced three spots into 11th position this year")
# curlRequest("TOPICS IN SOFTWARE ENGINEERING 1 - 	SOFTWARE RE-ENGINEERING")

def id_keywords_uri_pair(id_description_dict):
    id_keywords = {}
    for id in id_description_dict:
        id_keywords[id] = curlRequest(id_description_dict[id])
        print(id_keywords[id])

    fo = open("id_keywords_uri.txt", "w")
    for x, y in id_keywords.items():
        for y1, y2 in y.items():
            fo.write(x + " " + y1 + " " + y2 + "\n")
    fo.close()

    return id_keywords


from rest_framework import serializers
from flights.models import City, Airport, Route, Flight

class CitySerializer(serializers.ModelSerializer):
    class Meta:
        model = City
        fields = ("id", "name", "is_capital")

class AirportListSerializer(serializers.ModelSerializer):
    city = serializers.StringRelatedField()

    class Meta:
        model = Airport
        fields = ("id", "name", "city")

class AirportSerializer(serializers.ModelSerializer):
    class Meta:
        model = Airport
        fields = ("id", "name", "city")

    def to_representation(self, instance):
        return AirportListSerializer(instance).data

class RouteListSerializer(serializers.ModelSerializer):
    source = serializers.StringRelatedField()
    destination = serializers.StringRelatedField()

    class Meta:
        model = Route
        fields = ("id", "source", "destination", "distance")

class RouteSerializer(serializers.ModelSerializer):
    class Meta:
        model = Route
        fields = ("id", "source", "destination", "distance")

    def to_representation(self, instance):
        return RouteListSerializer(instance).data

class FlightListSerializer(serializers.ModelSerializer):
    route = serializers.StringRelatedField()
    crews = serializers.StringRelatedField(many=True)
    airplane = serializers.StringRelatedField()
    seats_available = serializers.IntegerField(read_only=True)

    class Meta:
        model = Flight
        fields = (
            "id",
            "route",
            "crews",
            "airplane",
            "departure_time",
            "arrival_time",
            "seats_available",
        )

class FlightSerializer(serializers.ModelSerializer):
    class Meta:
        model = Flight
        fields = (
            "id",
            "route",
            "crews",
            "airplane",
            "departure_time",
            "arrival_time"
        )

    def to_representation(self, instance):
        return FlightListSerializer(instance).data

class FlightDetailSerializer(serializers.ModelSerializer):
    route = serializers.StringRelatedField()
    crews = serializers.StringRelatedField(many=True)
    airplane = serializers.StringRelatedField()
    taken_seats = serializers.SerializerMethodField()

    class Meta:
        model = Flight
        fields = (
            "id",
            "route",
            "crews",
            "airplane",
            "departure_time",
            "arrival_time",
            "taken_seats",
        )

    def get_seat_and_row(self, ticket):
        return {"row": ticket.row, "seat": ticket.seat}

    def get_taken_seats(self, flight):
        return [self.get_seat_and_row(ticket) for ticket in
                flight.tickets.all()]

    def to_representation(self, instance):
        data = super().to_representation(instance)
        data["taken_seats"] = self.get_taken_seats(instance)
        return data


#!/usr/bin/env python3
# The print statements go to the console or web server access log or error log; they do not go to the web page.

from flask import Flask, render_template, request, make_response
import flask
print(f"flask {flask.__version__}")

app = Flask(__name__)

app.config["APPLICATION_ROOT"] = "/recommendit" # IDK if this actually does anything.
import os
import common
import pandas as pd
print(f"pandas {pd.__version__}")
import pyarrow  # Not refrenced in MY code, but is actually required by pandas bc I call read_feather.
print(f"pyarrow {pyarrow.__version__}")
from pathlib import Path

# Decide if running on dev host or public host, so path can be automatically changed. Basis: comcast isp vs amazon.com?
# In advance, set these env vars on the host.
# In Azure cloud: Portal | Home | wackysubs App Service | Settings | Configuration | Application Settings
# In Amazon EC2 or local dev host: Just set an env var in a startup script
adfpath = Path(os.environ['WACKYSUBSDATAPATHPART1'])
edfpath = Path(os.environ['WACKYSUBSDATAPATHPART2'])

print(f"Loading part 1 of model from path {adfpath}")
adf = pd.read_feather(adfpath);
print(f"Loading part 2 of model from path {edfpath}")
edf = pd.read_feather(edfpath);

@app.context_processor
def GetAllSubreddits():
    return dict(allsubreddits=sorted(edf.columns))

@app.route('/')
def index():
    return render_template('index.html', confMin=0.05)

# User keying in a subreddit, so offer a partially matching list.
@app.route('/offersubreddits', methods=["GET"])
def offersubreddits():
	q = request.args.get("q")
	subreddits = [subreddit for subreddit in edf.columns if subreddit.startswith(q)]
	return render_template("offersubreddits.html", subreddits=subreddits)

def GetFavsList():
	favs = request.cookies.get('favs')
	if favs is None:
		favs = ''
	favsList = [f for f in favs.split(";") if f != '']
	favsList = sorted(list(set(favsList)))
	return favsList

# User clicked button to run the recommender system
# Check the value of the action property of the submit button to decide action
@app.route('/formaction', methods=["GET"])
def formaction():
	baction = request.args.get('baction')
	if 'query' == baction:
		confMin = 0.01 if None == request.args.get('confMin') else request.args.get('confMin')
		searchTerm = request.args.get('searchTerm') # add 2 more searchTerm later
		if searchTerm is None or searchTerm == '':
			favsList = GetFavsList()
			return render_template('index.html', favorites=favsList)
		searchList=[searchTerm]
		recommendations = []
		if set(searchList).issubset(edf.columns):
			df = common.QueryModel(adf, edf, searchList, exactMatch=True, confMin=confMin)
			recommendations = common.QueryResultList(df) # [np.array(['Jokes'], dtype=object), np.array(['Showerthoughts'], dtype=object), ...
			recommendations = [list(a)[0] for a in recommendations] # ['Jokes', 'Showerthoughts', ...
			resp = make_response(render_template('index.html', recommendations=recommendations, searchList=searchList, searchTerm=searchTerm, confMin=confMin))
			# Every time they Find an item, add that to their favorites, but No duplicates, no empty strings.
			favsList = GetFavsList() + searchList
			favsList = sorted(favsList)
			favs = ';'.join(favsList)
			resp.set_cookie('favs', favs)
			return resp
		else:
			return render_template('index.html', confMin=0.05) # no such subreddit

	elif 'showfavs' == baction:
		favsList = GetFavsList()
		return render_template('index.html', favorites=favsList)
	elif 'delete' == baction:
		favsList = GetFavsList()
		item = request.args.get('delete')
		if item in favsList:
			favsList.remove(item)
		resp = make_response(render_template('index.html', favorites=favsList, baction='showfavs')) # show the (new) favs
		favs = ';'.join(favsList) # cookiefy the list
		resp.set_cookie('favs', favs) # set the favs into cookie
		return resp
	else:
		print(f"Bad parameter value in baction:*{baction}*")

if __name__ == "__main__":
    app.run()


from kata import main
from unittest import TestCase
import mock

class Test(TestCase):

    def test_template(self):
        tpl = main.TEMPLATE
        self.assertTrue(isinstance(tpl, str))

    def test_get_answers(self):
        expected = dict(noun='a', verb='b', adjective='c', adverb='d')
        answers = ['a', 'b', 'c', 'd']

        main.input = mock.Mock(side_effect=answers)
        values = main.get_answers()
        self.assertEqual(values, expected)

    @mock.patch('kata.main.print')
    def test_main(self, mock_print):
        main.TEMPLATE = template = "{vala} + {valb}"
        values = dict(vala='a', valb='b')

        expected = template.format(**values)
        main.get_answers = mock.Mock(return_value=values)
        main.main()
        self.assertEqual(mock_print.mock_calls, [mock.call(expected)])


import setuptools

with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setuptools.setup(
    name='bpmtoolbox',
    version='0.0.1',
    author='Richard Kastiak',
    author_email='kastiak.richard@gmail.com',
    description='BPM tool',
    long_description=long_description,
    long_description_content_type="text/markdown",
    url='https://github.com/Riqardos/BPMTool',
    project_urls={
        "Bug Tracker": "https://github.com/Riqardos/BPMTool/issues"
    },
    license='MIT',
    packages=['bpmtoolbox'],
    # install_requires=['AdminTask'],
)


from typing import TYPE_CHECKING

from aiogram import types
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters.state import State, StatesGroup

import handlers
from bot_init import write_user_data
from general_symbols import SpecialSymbols
from keyboards.user_options_keyboards import reply_languages_keyboard, reply_weather_provider_keyboard

if TYPE_CHECKING:
    from aiogram import Dispatcher

class FSMUserOptions(StatesGroup):
    language = State()
    weather_provider = State()

async def settings(message: types.Message):
    await FSMUserOptions.language.set()
    keyboard = reply_languages_keyboard()
    await message.delete()
    await message.bot.send_message(text=f"{SpecialSymbols.BULLET_MARK.value} Будьласка оберіть мову:",
                                   reply_markup=keyboard, chat_id=message.chat.id)

# catch language reply
async def select_language(callback_data: types.CallbackQuery, state=FSMContext):
    async with state.proxy() as data:
        data['language'] = callback_data.data
    await FSMUserOptions.next()
    await callback_data.message.edit_text(text=f"{SpecialSymbols.BULLET_MARK.value} "
                                               f"Будьласка оберіть провайдера погоди:",
                                          reply_markup=reply_weather_provider_keyboard())
    await callback_data.answer()

# catch weather provider reply
async def select_weather_provides(callback_data: types.CallbackQuery, state=FSMContext):
    async with state.proxy() as data:
        data['weather_provider'] = callback_data.data
    async with state.proxy() as data:
        write_user_data(user_id=callback_data.message.chat.id, user_data=data)
    await state.finish()
    await callback_data.message.edit_text("Налаштування закінчено!", reply_markup=None)
    await handlers.commands_handlers.start(callback_data.message)

def register_user_options_handlers(dispatcher: "Dispatcher"):
    dispatcher.register_message_handler(settings, commands="settings", state=None)
    dispatcher.register_callback_query_handler(select_language, state=FSMUserOptions.language)
    dispatcher.register_callback_query_handler(select_weather_provides, state=FSMUserOptions.weather_provider)


import numpy as np
import matplotlib.pyplot as plt
import time

def main():
    x = np.arange(-10, 10, 0.1)
    y1 = x**2
    y2 = 3*x**2+50

    f, ax = plt.subplots(1)
    plt.title("Function")
    plt.xlabel("Values of x")
    plt.ylabel("Values of y")

    ax.plot(x, y1)
    plt.wait
    ax.plot(x, y2)

    plt.show()

if __name__ == "__main__":
    main()


import requests
import pandas as pd
# from datetime import datetime
import datetime
import sqlite3
import FinanceDataReader as fdr
import pandas_datareader as pdr

def load_codeNname():        # 주식 코드 가져오기
    df = pd.read_excel('code_name.xlsx')
    return df

def get_today_data():       # 오늘 주식 다운로드

    start_time = datetime.datetime.now()
    print(f"시작시간 = {start_time}")

    df = pd.DataFrame()
    code_data = load_codeNname()
    codes = code_data['Code']
    names = code_data['Name']

    yesterday = datetime.datetime.now()- datetime.timedelta(days=4)
    yesterday = yesterday.strftime("%Y-%m-%d")
    for i, stock_code in enumerate(codes):
        new_df = pdr.naver.NaverDailyReader(stock_code, start=yesterday).read()

        stock_name = names[i]
        total_stock_count = len(codes)
        print(f'{i+1}/{total_stock_count} : {stock_name}')
        new_df[['Open', 'High', 'Low', 'Close', 'Volume']] = new_df[['Open', 'High', 'Low', 'Close', 'Volume']].apply(pd.to_numeric)       # 데이터를 숫자형태로 변경
        # 한열은 이런식으로 가능하다. df['a'] = pd.to_numeric(df['a'])

        new_df['Change'] = new_df['Close'].diff()
        try:
            new_df['Change_R'] = round(new_df['Close'].pct_change() * 100, 2)
        except:
            new_df['Change_R'] = 0

        new_df['Name'] = stock_name
        new_df = new_df.tail(1)
        df = pd.concat([df, new_df], ignore_index=True)

    df['Change_rank'] = df['Change_R'].rank(ascending=False)
    df = df.sort_values(by='Change_rank', ascending=True)      # 오름차순 정렬
    end_time = datetime.datetime.now()
    time = end_time -start_time
    print(f'걸린시간 = {time}')

    return df

def test():
    df = get_today_data()
    df.to_excel('test5.xlsx')

test()
input("작업이 완료되었습니다. 엔터를 누르면 창을 닫습니다.")


import functools
from typing import Any, Callable
import numpy as np

from flax import linen as nn
import jax
from jax import lax
from jax import random
import jax.numpy as jnp

class NerfMLP(nn.Module):
  """A Nerf MLP."""
  net_depth: int = 8  # The depth of the first part of MLP.
  net_width: int = 256  # The width of the first part of MLP.
  net_depth_condition: int = 1  # The depth of the second part of MLP.
  net_width_condition: int = 128  # The width of the second part of MLP.
  net_activation: Callable[..., Any] = nn.relu  # The activation function.
  skip_layer: int = 4  # The layer to add skip layers to.
  num_rgb_channels: int = 3  # The number of RGB channels.
  num_sigma_channels: int = 1  # The number of sigma channels.

  @nn.compact
  def __call__(self, x, condition=None):
    """Evaluate the MLP.

    Args:
      x: jnp.ndarray(float32), [batch, num_samples, feature], points.
      condition: jnp.ndarray(float32), [batch, feature], if not None, this
        variable will be part of the input to the second part of the MLP
        concatenated with the output vector of the first part of the MLP. If
        None, only the first part of the MLP will be used with input x. In the
        original paper, this variable is the view direction.

    Returns:
      raw_rgb: jnp.ndarray(float32), with a shape of
           [batch, num_samples, num_rgb_channels].
      raw_sigma: jnp.ndarray(float32), with a shape of
           [batch, num_samples, num_sigma_channels].
    """
    feature_dim = x.shape[-1]
    num_samples = x.shape[1]
    x = x.reshape([-1, feature_dim])
    dense_layer = functools.partial(
        nn.Dense, kernel_init=jax.nn.initializers.glorot_uniform())
    inputs = x
    for i in range(self.net_depth):
      x = dense_layer(self.net_width)(x)
      x = self.net_activation(x)
      if i % self.skip_layer == 0 and i > 0:
        x = jnp.concatenate([x, inputs], axis=-1)
    raw_sigma = dense_layer(self.num_sigma_channels)(x).reshape(
        [-1, num_samples, self.num_sigma_channels])

    if condition is not None:
      # Output of the first part of MLP.
      bottleneck = dense_layer(self.net_width)(x)
      # Broadcast condition from [batch, feature] to
      # [batch, num_samples, feature] since all the samples along the same ray
      # have the same viewdir.

      # Collapse the [batch, num_samples, feature] tensor to
      # [batch * num_samples, feature] so that it can be fed into nn.Dense.
      condition = condition.reshape([-1, condition.shape[-1]])
      x = jnp.concatenate([bottleneck, condition], axis=-1)
      # Here use 1 extra layer to align with the original nerf model.
      for i in range(self.net_depth_condition):
        x = dense_layer(self.net_width_condition)(x)
        x = self.net_activation(x)
    raw_rgb = dense_layer(self.num_rgb_channels)(x).reshape(
        [-1, num_samples, self.num_rgb_channels])
    return raw_rgb, raw_sigma

class MLP(nn.Module):
  """A simple MLP."""
  net_depth: int = 8  # The depth of the first part of MLP.
  net_width: int = 256  # The width of the first part of MLP.
  net_depth_condition: int = 1  # The depth of the second part of MLP.
  net_width_condition: int = 128  # The width of the second part of MLP.
  net_activation: Callable[..., Any] = nn.relu  # The activation function.
  skip_layer: int = 4  # The layer to add skip layers to.
  num_out_channels: int = 1  # The number of sigma channels.
  output_init: Callable = jax.nn.initializers.xavier_uniform()

  @nn.compact
  def __call__(self, x, condition=None):
    """Evaluate the MLP.

    Args:
      x: jnp.ndarray(float32), [batch, num_samples, feature], points.
      condition: jnp.ndarray(float32), [batch, feature], if not None, this
        variable will be part of the input to the second part of the MLP
        concatenated with the output vector of the first part of the MLP. If
        None, only the first part of the MLP will be used with input x. In the
        original paper, this variable is the view direction.

    Returns:
      raw_out: jnp.ndarray(float32), with a shape of
           [batch, num_samples, num_out_channels].
    """
    feature_dim = x.shape[-1]
    num_samples = x.shape[1]
    x = x.reshape([-1, feature_dim])
    dense_layer = functools.partial(
        nn.Dense, kernel_init=jax.nn.initializers.xavier_uniform())
    inputs = x
    for i in range(self.net_depth):
      x = dense_layer(self.net_width)(x)
      x = self.net_activation(x)
      if i % self.skip_layer == 0 and i > 0:
        x = jnp.concatenate([x, inputs], axis=-1)
    if condition is not None:
      condition = condition.reshape([-1, condition.shape[-1]])
      x = jnp.concatenate([x, condition], axis=-1)
      for i in range(self.net_depth_condition):
        x = dense_layer(self.net_width_condition)(x)
        x = self.net_activation(x)
    raw_out = nn.Dense(self.num_out_channels, kernel_init=self.output_init)(x).reshape(
        [-1, num_samples, self.num_out_channels])

    return raw_out

def cast_rays(z_vals, origins, directions):
  return origins[..., None, :] + z_vals[..., None] * directions[..., None, :]

# def sample_along_rays(key, origins, directions, num_samples, near, far,
#                       randomized, lindisp):
#   """Stratified sampling along the rays.

#   Args:
#     key: jnp.ndarray, random generator key.
#     origins: jnp.ndarray(float32), [batch_size, 3], ray origins.
#     directions: jnp.ndarray(float32), [batch_size, 3], ray directions.
#     num_samples: int.
#     near: float, near clip.
#     far: float, far clip.
#     randomized: bool, use randomized stratified sampling.
#     lindisp: bool, sampling linearly in disparity rather than depth.

#   Returns:
#     z_vals: jnp.ndarray, [batch_size, num_samples], sampled z values.
#     points: jnp.ndarray, [batch_size, num_samples, 3], sampled points.
#   """
#   batch_size = origins.shape[0]

#   t_vals = jnp.linspace(0., 1., num_samples)
#   if lindisp:
#     t_vals = 1. / (1. / near * (1. - t_vals) + 1. / far * t_vals)
#   else:
#     t_vals = near * (1. - t_vals) + far * t_vals

#   if randomized:
#     mids = 0.5 * (t_vals[..., 1:] + t_vals[..., :-1])
#     upper = jnp.concatenate([mids, t_vals[..., -1:]], -1)
#     lower = jnp.concatenate([t_vals[..., :1], mids], -1)
#     t_rand = random.uniform(key, [batch_size, num_samples])
#     t_vals = lower + (upper - lower) * t_rand
#   else:
#     # Broadcast t_vals to make the returned shape consistent.
#     t_vals = jnp.broadcast_to(t_vals[None, ...], [batch_size, num_samples])

#   coords = cast_rays(t_vals, origins, directions)
#   return t_vals, coords

def pos_enc(x, min_deg, max_deg, legacy_posenc_order=False, amp=1.0):
  """Cat x with a positional encoding of x with scales 2^[min_deg, max_deg-1].

  Instead of computing [sin(x), cos(x)], we use the trig identity
  cos(x) = sin(x + pi/2) and do one vectorized call to sin([x, x+pi/2]).

  Args:
    x: jnp.ndarray, variables to be encoded. Note that x should be in [-pi, pi].
    min_deg: int, the minimum (inclusive) degree of the encoding.
    max_deg: int, the maximum (exclusive) degree of the encoding.
    legacy_posenc_order: bool, keep the same ordering as the original tf code.

  Returns:
    encoded: jnp.ndarray, encoded variables.
  """
  if min_deg == max_deg:
    return x
  scales = jnp.array([2**i for i in range(min_deg, max_deg)])
  if legacy_posenc_order:
    xb = x[..., None, :] * scales[:, None]
    four_feat = jnp.reshape(
        jnp.sin(jnp.stack([xb, xb + 0.5 * jnp.pi], -2)),
        list(x.shape[:-1]) + [-1])
  else:
    xb = jnp.reshape((x[..., None, :] * scales[:, None]),
                     list(x.shape[:-1]) + [-1])
    four_feat = jnp.sin(jnp.concatenate([xb, xb + 0.5 * jnp.pi], axis=-1))
  return jnp.concatenate([x] + [amp * four_feat], axis=-1)

# Reference: https://github.com/google/nerfies/blob/1a38512214cfa14286ef0561992cca9398265e13/nerfies/modules.py#L231
def cosine_easing_window(min_freq_log2, max_freq_log2, num_bands, alpha):
  """Eases in each frequency one by one with a cosine.
  This is equivalent to taking a Tukey window and sliding it to the right
  along the frequency spectrum.
  Args:
    min_freq_log2: the lower frequency band.
    max_freq_log2: the upper frequency band.
    num_bands: the number of frequencies.
    alpha: will ease in each frequency as alpha goes from 0.0 to num_freqs.
  Returns:
    A 1-d numpy array with num_sample elements containing the window.
  """
  if max_freq_log2 is None:
    max_freq_log2 = num_bands - 1.0
  bands = jnp.linspace(min_freq_log2, max_freq_log2, num_bands)
  x = jnp.clip(alpha - bands, 0.0, 1.0)
  return 0.5 * (1 + jnp.cos(jnp.pi * x + jnp.pi))

def annealed_pos_enc(x, min_deg, max_deg, alpha, amp=1.0):
  if min_deg == max_deg:
    return x
  scales = jnp.array([2**i for i in range(min_deg, max_deg)])

  xb = x[..., None, :] * scales[:, None]  # [batch, sample, num_deg, channel]
  window = cosine_easing_window(min_deg, max_deg - 1, len(scales), alpha)[:, None]  # [num_deg, 1]
  four_feat = jnp.reshape(jnp.concatenate([jnp.sin(xb) * window, jnp.sin(xb + 0.5 * jnp.pi) * window], axis=-1),
                          list(x.shape[:-1]) + [-1])  # [batch, sample, channel]
  return amp * four_feat

def volumetric_rendering(rgb, density, t_vals, dirs, white_bkgd, rgb_bkgd, mask_bbox=None):
  """Volumetric Rendering Function.

  Args:
    rgb: jnp.ndarray(float32), color, [batch_size, num_samples, 3]
    density: jnp.ndarray(float32), sigma, [batch_size, num_samples, 1].
    t_vals: jnp.ndarray(float32), [batch_size, num_samples].
    dirs: jnp.ndarray(float32), [batch_size, 3].
    white_bkgd: bool.

  Returns:
    comp_rgb: jnp.ndarray(float32), [batch_size, 3].
    disp: jnp.ndarray(float32), [batch_size].
    acc: jnp.ndarray(float32), [batch_size].
    weights: jnp.ndarray(float32), [batch_size, num_samples]
  """
  eps = 1e-10
  # NOTE(bkgd): To account for boundary term, narrow down the last delta
  t_dists = jnp.concatenate([
      t_vals[..., 1:] - t_vals[..., :-1],
      jnp.broadcast_to([1e-3], t_vals[..., :1].shape)
  ], -1)
  # NOTE(eikonal)
  delta = t_dists * jnp.linalg.norm(dirs, axis=-1)
  # Note that we're quietly turning density from [..., 0] to [...].
  density_delta = density[..., 0] * delta

  # NOTE(voxelize)
  if mask_bbox is not None:
    density_delta *= mask_bbox

  # Check beta/sparsity
  # if mask is not None:
  #   mask = mask * ((1 - jnp.exp(-density_delta)) > 0.5)
  # mask = (1 - jnp.exp(-density_delta)) > 0.1
  # density_delta = density_delta * mask

  # Exponential transmittance model
  alpha = 1 - jnp.exp(-density_delta)
  trans = jnp.exp(-jnp.concatenate([
      jnp.zeros_like(density_delta[..., :1]),
      jnp.cumsum(density_delta[..., :], axis=-1)
  ], axis=-1))
  # # Linear transmittance model
  # alpha = 1 - nn.relu(1 - density_delta)
  # trans = jnp.concatenate([
  #     jnp.ones_like(alpha[..., :1], alpha.dtype),
  #     jnp.cumprod(nn.relu(1 - density_delta[..., :]) + eps, axis=-1)
  # ], axis=-1)
  weights = alpha * trans[..., :-1]

  if rgb_bkgd is not None:
    comp_rgb = (weights[..., None] * rgb).sum(axis=-2) + trans[..., -1:] * rgb_bkgd
  else:
    comp_rgb = (weights[..., None] * rgb).sum(axis=-2)
    rgb_bkgd = jnp.ones(list(trans[..., -1:].shape[:-1]) + [3])
  acc = weights.sum(axis=-1)
  distance = (weights * t_vals).sum(axis=-1) / acc
  distance = jnp.clip(jnp.nan_to_num(distance, jnp.inf), t_vals[:, 0], t_vals[:, -1])

  if white_bkgd:
    comp_rgb = comp_rgb + (1. - acc[..., None])
  return comp_rgb, distance, acc, weights, alpha, trans[..., -1:], trans[..., -1:] * lax.stop_gradient(rgb_bkgd)

def sorted_piecewise_constant_pdf(key, bins, weights, num_samples, randomized):
  """Piecewise-Constant PDF sampling from sorted bins.

  Args:
    key: jnp.ndarray(float32), [2,], random number generator.
    bins: jnp.ndarray(float32), [batch_size, num_bins + 1].
    weights: jnp.ndarray(float32), [batch_size, num_bins].
    num_samples: int, the number of samples.
    randomized: bool, use randomized samples.

  Returns:
    t_samples: jnp.ndarray(float32), [batch_size, num_samples].
  """
  # Pad each weight vector (only if necessary) to bring its sum to `eps`. This
  # avoids NaNs when the input is zeros or small, but has no effect otherwise.
  eps = 1e-5
  weight_sum = jnp.sum(weights, axis=-1, keepdims=True)
  padding = jnp.maximum(0, eps - weight_sum)
  weights += padding / weights.shape[-1]
  weight_sum += padding

  # Compute the PDF and CDF for each weight vector, while ensuring that the CDF
  # starts with exactly 0 and ends with exactly 1.
  pdf = weights / weight_sum
  cdf = jnp.minimum(1, jnp.cumsum(pdf[..., :-1], axis=-1))
  cdf = jnp.concatenate([
      jnp.zeros(list(cdf.shape[:-1]) + [1]), cdf,
      jnp.ones(list(cdf.shape[:-1]) + [1])
  ], axis=-1)

  # Draw uniform samples.
  if randomized:
    # mipnerf
    s = 1 / num_samples
    u = jnp.arange(num_samples) * s
    u += jax.random.uniform(
        key,
        list(cdf.shape[:-1]) + [num_samples],
        maxval=s - jnp.finfo('float32').eps)
    # `u` is in [0, 1) --- it can be zero, but it can never be 1.
    u = jnp.minimum(u, 1. - jnp.finfo('float32').eps)
  else:
    # Match the behavior of jax.random.uniform() by spanning [0, 1-eps].
    u = jnp.linspace(0., 1. - jnp.finfo('float32').eps, num_samples)
    u = jnp.broadcast_to(u, list(cdf.shape[:-1]) + [num_samples])

  # Identify the location in `cdf` that corresponds to a random sample.
  # The final `True` index in `mask` will be the start of the sampled interval.
  mask = u[..., None, :] >= cdf[..., :, None]

  def find_interval(x):
    # Grab the value where `mask` switches from True to False, and vice versa.
    # This approach takes advantage of the fact that `x` is sorted.
    x0 = jnp.max(jnp.where(mask, x[..., None], x[..., :1, None]), -2)
    x1 = jnp.min(jnp.where(~mask, x[..., None], x[..., -1:, None]), -2)
    return x0, x1

  bins_g0, bins_g1 = find_interval(bins)
  cdf_g0, cdf_g1 = find_interval(cdf)

  t = jnp.clip(jnp.nan_to_num((u - cdf_g0) / (cdf_g1 - cdf_g0), 0), 0, 1)
  samples = bins_g0 + t * (bins_g1 - bins_g0)
  return samples

def sample_pdf(key, bins, weights, origins, directions, z_vals, idx_grads, num_samples,
               randomized, jitter, near, stop_grad=True, resample_padding=0.01):
  """Hierarchical sampling.

  Args:
    key: jnp.ndarray(float32), [2,], random number generator.
    bins: jnp.ndarray(float32), [batch_size, num_bins + 1].
    weights: jnp.ndarray(float32), [batch_size, num_bins].
    origins: jnp.ndarray(float32), [batch_size, 3], ray origins.
    directions: jnp.ndarray(float32), [batch_size, 3], ray directions.
    z_vals: jnp.ndarray(float32), [batch_size, num_coarse_samples].
    num_samples: int, the number of samples.
    randomized: bool, use randomized samples.
    stop_grad: bool, whether or not to backprop through sampling.
    resample_padding: float, added to the weights before normalizing.

  Returns:
    z_vals: jnp.ndarray(float32),
      [batch_size, num_coarse_samples + num_fine_samples].
    points: jnp.ndarray(float32),
      [batch_size, num_coarse_samples + num_fine_samples, 3].
  """
  # Do a blurpool.

  z_samples = sorted_piecewise_constant_pdf(key, bins, weights, num_samples, randomized)

  # Compute united z_vals and sample points
  # NOTE(debug): visualize coarse and fine samples
  z_samples = jnp.sort(jnp.concatenate([z_vals[:, jitter], z_samples], axis=-1), axis=-1)  # coarse + fine samples
  if stop_grad:
    origins = lax.stop_gradient(origins)
    directions = lax.stop_gradient(directions)
    z_samples = lax.stop_gradient(z_samples)
    z_vals = lax.stop_gradient(z_vals)
    idx_grads = lax.stop_gradient(idx_grads)

  ret = jnp.ones(list(z_samples.shape) + [3 + 3 + 3])
  # Assume that z_vals is monotonically increasing / sorted
  def sorted_find_nearest(xi, x, y):
    # # Shift x points to centers for rounding
    # spacing = jnp.diff(x) / 2
    # x = x + jnp.hstack([spacing, spacing[-1]])
    # Append head, tail for indexing
    y = jnp.hstack([y[0], y, y[-1]])
    return y[jnp.searchsorted(x, xi, side="left")]
  def loop_body(i, val):
    idx = sorted_find_nearest(z_samples[i], z_vals[i], jnp.arange(z_vals.shape[1]))
    rd = directions[i, idx]
    val = val.at[i, :, :3].set(origins[i, idx] + rd * (z_samples[i] - z_vals[i, idx])[..., None])
    val = val.at[i, :, 3:6].set(rd)
    val = val.at[i, :, 6:9].set(idx_grads[i, idx])
    return val
  ret = jax.lax.fori_loop(0, ret.shape[0], loop_body, ret)
  # for i in range(z_vals.shape[0]):
  #   ret = ret.at[i].set(directions[i, sorted_find_nearest(z_samples[i], z_vals[i], jnp.arange(z_vals.shape[1]))])

  # t = jnp.concatenate([z_samples[:, 0:1] - near, z_samples[:, 1:] - z_samples[:, :-1]], axis=-1)[..., None]
  # coords = jnp.cumsum(ret[..., :3] * t, axis=1) + origins[:, 0:1]
  return z_samples, ret[..., :3], ret[..., 3:6], ret[..., 6:9]

def add_gaussian_noise(key, raw, noise_std, randomized):
  """Adds gaussian noise to `raw`, which can used to regularize it.

  Args:
    key: jnp.ndarray(float32), [2,], random number generator.
    raw: jnp.ndarray(float32), arbitrary shape.
    noise_std: float, The standard deviation of the noise to be added.
    randomized: bool, add noise if randomized is True.

  Returns:
    raw + noise: jnp.ndarray(float32), with the same shape as `raw`.
  """
  if (noise_std is not None) and randomized:
    return raw + random.normal(key, raw.shape, dtype=raw.dtype) * noise_std
  else:
    return raw


# -*- coding: utf-8 -*-
import scrapy
import re
from spCollector.items import SolutionItem

class SusesolutionSpider(scrapy.Spider):
    name = 'suseSolution'
    start_urls = ['https://lists.opensuse.org/opensuse-security-announce/']

    def parse(self, response):
        for group in response.xpath('//div[@class="year_group"]'):
            for tr in group.xpath('.//tr'):
                item_url = tr.xpath('.//td[3]/a/@href').extract_first()
                if item_url is not None:
                    if int(item_url.split('/')[0].split('-')[0]) > 2014:
                        yield scrapy.Request(url=response.urljoin(item_url), callback=self.parse_subject)

    def parse_subject(self, response):
        for td in response.xpath('//td[@class="msubject"]'):
            item_url = td.xpath('.//a/@href').extract_first()
            if item_url is not None:
                yield scrapy.Request(url=response.url.replace('date.html', item_url), callback=self.parse_item)

    def parse_item(self, response):
        content = response.xpath('//div[@class="body"]').extract_first()
        item = self.parse_solution(response.url, content)
        yield item

    def parse_solution(self, url, content):
        try:
            item = SolutionItem()
            content = content.replace('<br>', '\n')
            match_obj = re.search(r'Announcement ID:.+\n', content, re.M | re.I)
            if match_obj:
                item['id'] = match_obj.group().replace('Announcement ID:', '').strip()
                item['id'] = item['id'] .replace(' ', '')
                match_obj = re.search(r'Patch Instructions:[\s\S]+Package List:', content, re.M | re.I)
                if match_obj:
                    item['solution'] = match_obj.group().replace('Package List:', '').replace('Patch Instructions:', '').strip()
                    item['solution'] = item['solution'].replace('\n\n', '\n').strip()
                    return item
                else:
                    print("Parsing error, html file \"{0}\" is not well-formed".format(url))
            else:
                print("Parsing error, html file \"{0}\" is not well-formed".format(url))

        except:
            print("Parsing error, html file \"{0}\" is not well-formed".format(url))


from django.contrib import admin, auth
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
from django.contrib.auth.forms import UserChangeForm

from users.models import User

class UserAdmin(BaseUserAdmin):
    add_fieldsets = (
        (
            None,
            {
                "classes": ("wide",),
                "fields": ("email", "password1", "password2"),
            },
        ),
    )
    fieldsets = (
        (
            "Главная информация",
            {
                "fields": (
                    "email",
                    "password",
                )
            },
        ),
        (
            "Личная информация",
            {
                "fields": (
                    "first_name",
                    "last_name",
                    "phone_number",
                )
            },
        ),
        ("Статус", {"fields": ("is_staff", "is_superuser", "is_active")}),
    )
    list_display = ("id", "email", "phone_number", "is_superuser", "is_active")
    ordering = ["id"]
    form = UserChangeForm

admin.site.register(User, UserAdmin)
admin.site.unregister(auth.models.Group)