from __future__ import print_function, division
from skimage import morphology
from skimage import measure
from sklearn.cluster import KMeans
from skimage.transform import resize
import SimpleITK as sitk
import numpy as np
import os
from glob import glob
from tqdm import tqdm
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits.mplot3d.art3d import Poly3DCollection
import scipy.misc
from Heatmap import create_heatmap
from CropSamples import crop_samples
from Respace import respace
import pickle

def show_3d_img(img):
    for z in range(img.shape[0]):
        plt.imshow(img[z], cmap='gray')
        plt.show()
        #scipy.misc.imsave('images/{}.jpg'.format(z), img[z])

def plot_3d(image, threshold=0.5):

    # Position the scan upright,
    # so the head of the patient would be at the top facing the camera
    p = image.transpose(2,1,0)

    verts, faces = measure.marching_cubes(p, threshold)

    fig = plt.figure(figsize=(10, 10))
    ax = Axes3D(fig)
    #ax = fig.add_subplot(111, projection='3d')

    # Fancy indexing: `verts[faces]` to generate a collection of triangles
    mesh = Poly3DCollection(verts[faces], alpha=0.70)
    face_color = [0.45, 0.45, 0.75]
    mesh.set_facecolor(face_color)
    ax.add_collection3d(mesh)

    ax.set_xlim(0, p.shape[0])
    ax.set_ylim(0, p.shape[1])
    ax.set_zlim(0, p.shape[2])

    plt.show()

# Get rows in data frame associated with each file
def get_filename(fileList, case):
    for f in fileList:
        if case in f:
            return(f)

# Takes a pandas data frame read from the annotations csv file append
# reeturns a list of dictionary containing the nodules and converted coordinates
def change_nodule_coordinate_spacing(nodulesDF, origin, sourceSpacing, targetSpacing):
    nodules=[]
    targetSpacing = [targetSpacing[1], targetSpacing[2], targetSpacing[0]]
    for noduleIndex, row in nodulesDF.iterrows():
        worldCoord = np.array([row["coordX"], row["coordY"], row["coordZ"]])
        voxelCoord = np.rint(np.absolute(worldCoord - origin) / targetSpacing)
        voxelCoord = np.array([int(voxelCoord[2]), int(voxelCoord[0]), int(voxelCoord[1])])

        nodule= {
        'seriesuid' : row['seriesuid'],
        'coords' : voxelCoord,
        }
        #print(row['seriesuid'])
        if 'diameter_mm' in row.index:
            nodule['diameter_mm'] = row['diameter_mm']
        nodules.append(nodule)
    return nodules
def read_image(imageFile):
    # Read the scan
    itkImage = sitk.ReadImage(imageFile)
    npImage = sitk.GetArrayFromImage(itkImage).transpose(0,2,1)
    npZ, height, width = npImage.shape
    npOrigin = np.array(itkImage.GetOrigin())
    npSpacing = itkImage.GetSpacing()
    npSpacing = np.array([npSpacing[2], npSpacing[0], npSpacing[1]])
    return npImage, npSpacing, npOrigin

def normalize_values(npImage):
    maxHU = 400.0
    minHU = -1000.0
    npImage = (npImage-minHU) / (maxHU - minHU)
    npImage[npImage>1] = 1
    npImage[npImage<0] = 0
    npImage = npImage*2 -1
    return npImage

# Get list of image files
inputPath = "/media/amos/My Passport/Data/luna/"
outputPath = "/media/amos/My Passport/Data/luna/nodules_dataset"
annotationPath = "datasets/nodules/annotations/"
targetSpacing = np.array([1.25, 1, 1])

# For each subset
for i in range(0,9):
    print("subset" + str(i))
    subsetPath = inputPath + "subset" + str(i) + "/"
    fileList = glob(subsetPath + "*.mhd")

    # Get locations of the nodules
    nodulesDF = pd.read_csv(annotationPath + "annotations.csv")
    nodulesDF["file"] = nodulesDF["seriesuid"].map(lambda fileName: get_filename(fileList, fileName))
    nodulesDF = nodulesDF.dropna()

    # Get locations of candidates
    candidatesDF = pd.read_csv(annotationPath + "candidates.csv")
    candidatesDF["file"] = candidatesDF["seriesuid"].map(lambda fileName: get_filename(fileList, fileName))
    candidatesDF = candidatesDF.dropna()

    samples = []
    # For each image file
    for fileCount, imageFile in enumerate(tqdm(fileList)):
        #if fileCount < 52 : continue
        suid = imageFile.split("/")[-1][:-4]

        print("reading image")
        npImage, npSpacing, npOrigin = read_image(imageFile)

        # Get all nodules and candidates present
        # z x y
        print("resizing nodule coords")
        nodules = change_nodule_coordinate_spacing(nodulesDF[nodulesDF["file"] == imageFile], npOrigin, npSpacing, targetSpacing)
        #print("{} nodules in {}".format(len(nodules), suid))
        #candidates = change_nodule_coordinate_spacing(candidatesDF[candidatesDF["file"] == imageFile], npOrigin, npSpacing, targetSpacing)

        # Rescale the scan
        print("respacing image")
        scan = respace(npImage, npSpacing, targetSpacing)
        print("renormalizing")
        scan = normalize_values(scan)

        # Generate heatmap
        print("generating heatmap")
        heatmap_real = create_heatmap(nodules, scan.shape, candidates=False)
        #heatmap_candidates = create_heatmap(candidates, scan.shape, candidates=True)
        #plot_3d(heatmap_candidates)

        # Generate sample boxes to be cropped at test time
        print("generating crops")
        samples.append(crop_samples(nodules, scan.shape, np.array([64, 96, 96]), 20, 0.8))
        #coords = nodules[0]["coords"]
        #show_3d_img(scan[coords[0]-10:coords[0]+10, coords[1]-50:coords[1]+50,coords[2]-50:coords[2]+50])
        #for sample in samples[suid]:
        #    coords = sample['bounds']
        #    show_3d_img(heatmap_real[coords[0]+50:coords[0]+51, coords[1]:coords[4], coords[2]:coords[5]])
        #    show_3d_img(scan[coords[0]+50:coords[0]+51, coords[1]:coords[4], coords[2]:coords[5]])
        #samples = samples + crop_samples(candidates, scan.shape, np.array([100, 100, 125]), 10, 0.8, candidates="candidates")

        print("saving")
        #save the mask and scan as compressed npz files
        if not os.path.exists(os.path.join(outputPath, "heatmaps", "real")):
            os.makedirs(os.path.join(outputPath, "heatmaps", "real"))
        if not os.path.exists(os.path.join(outputPath, "heatmaps", "candidates")):
            os.makedirs(os.path.join(outputPath, "heatmaps", "candidates"))
        if not os.path.exists(os.path.join(outputPath, "scans_processed")):
            os.makedirs(os.path.join(outputPath, "scans_processed"))
        np.savez_compressed(os.path.join(outputPath, "heatmaps", "real", "%s" % (suid)), a = heatmap_real)
        #np.savez_compressed(os.path.join(outputPath, "heatmaps", "candidates", "%s" % (nodules[0]['seriesuid'])), a = heatmap_candidates)
        np.savez_compressed(os.path.join(outputPath, "scans_processed", "%s" % (suid)), a = scan)

        #save crop boxes
        f = open(os.path.join(outputPath, "crops.pkl"), 'wb')
        print(samples)
        pickle.dump(samples, f)
        f.close()


import torch
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
import inspect
import os
from .genotypes  import Genotype
from .operations import ReLUConvBN
from .operations import FactorizedReduce
from .operations import MixedLayer
from .operations import Identity
from .operations import OPS
from pysot.core.config import cfg
from .search_space import build_search_space
from pysot.utils.latency import predict_latency,compute_latency

class DartsCell(nn.Module):

  def __init__(self, num_node, multiplier, C_prev_prev, C_prev, C, reduction, reduction_prev, search_space):
    """
    :param num_node: 4, number of layers inside a cell
    :param multiplier: 4
    :param C_prev_prev: 48
    :param C_prev: 48
    :param C: 16
    :param reduction: indicates whether to reduce the output maps width
    :param reduction_prev: when previous cell reduced width, s1_d = s0_d//2
    in order to keep same shape between s1 and s0, we adopt prep0 layer to
    reduce the s0 width by half.
    """
    super(DartsCell, self).__init__()

    # indicating current cell is reduction or not
    self.reduction = reduction
    self.reduction_prev = reduction_prev

    # preprocess0 deal with output from prev_prev cell
    if reduction_prev:
      # if prev cell has reduced channel/double width,
      # it will reduce width by half
      self.preprocess0 = FactorizedReduce(C_prev_prev, C, affine=False)
    else:
      self.preprocess0 = ReLUConvBN(C_prev_prev, C, kernel_size=1,
                                    stride=1, padding=0, affine=False)
    # preprocess1 deal with output from prev cell
    self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0, affine=False)

    # steps inside a cell
    self.num_node = num_node  # 4
    self.multiplier = multiplier  # 4

    self.layers = nn.ModuleList()

    for i in range(self.num_node):
      # for each i inside cell, it connects with all previous output
      # plus previous two cells' output
      for j in range(2 + i):
        # for reduction cell, it will reduce the heading 2 inputs only
        stride = 2 if reduction and j < 2 else 1
        layer = MixedLayer(C, stride, op_names_list=search_space)
        self.layers.append(layer)

  def forward(self, s0, s1, weights):
    """
    :param s0:
    :param s1:
    :param weights: [14, 8]
    :return:
    """
    # print('s0:', s0.shape,end='=>')
    s0 = self.preprocess0(s0)  # [40, 48, 32, 32], [40, 16, 32, 32]
    # print(s0.shape, self.reduction_prev)
    # print('s1:', s1.shape,end='=>')
    s1 = self.preprocess1(s1)  # [40, 48, 32, 32], [40, 16, 32, 32]
    # print(s1.shape)

    states = [s0, s1]
    offset = 0
    # for each node, receive input from all previous intermediate nodes and s0, s1
    for i in range(self.num_node):  # 4
      # [40, 16, 32, 32]
      s = sum(self.layers[offset + j](h, weights[offset + j])
              for j, h in enumerate(states)) / len(states)
      offset += len(states)
      # append one state since s is the elem-wise addition of all output
      states.append(s)
      # print('node:',i, s.shape, self.reduction)

    # concat along dim=channel
    return torch.cat(states[-self.multiplier:], dim=1)  # 6 of [40, 16, 32, 32]

  def forward_latency(self, s0, s1, weights):
    """
    :param s0:
    :param s1:
    :param weights: [14, 8]
    :return:
    """
    latency = 0.

    oplatency,s0 = self.preprocess0.forward_latency(s0)
    latency+=oplatency

    oplatency,s1 = self.preprocess1.forward_latency(s1)
    latency+=oplatency

    states = [s0, s1]
    offset = 0
    # for each node, receive input from all previous intermediate nodes and s0, s1
    for i in range(self.num_node):  # 4
      # [40, 16, 32, 32]
      for j,h in enumerate(states):
        oplatency,s=self.layers[offset+j].forward_latency(h,weights[offset+j])
        latency+=oplatency
      offset+=len(states)
      states.append(s)

    channel=sum(tmp[0] for tmp in states[-self.multiplier:])
    return latency,(channel,states[-1][1],states[-1][2])

class SuperNet(nn.Module):
  """
  stack number:layer of cells and then flatten to fed a linear layer
  """

  def __init__(self, used_layers,search_space, num_ch, num_cell,
               num_node=4, multiplier=4, stem_multiplier=3, num_class=10,  img_channel=3):
    """

    :param C: 16
    :param num_cell: number of cells of current network
    :param num_node: nodes num inside cell
    :param multiplier: output channel of cell = multiplier * ch
    :param stem_multiplier: output channel of stem net = stem_multiplier * ch
    :param num_class: 10
    """
    super(SuperNet, self).__init__()

    self.used_layers=used_layers
    self.C = num_ch
    self.num_class    = num_class
    self.num_cell     = num_cell
    self.num_node     = num_node
    self.multiplier   = multiplier
    self.search_space = search_space

    # stem_multiplier is for stem network,
    # and multiplier is for general cell
    C_curr = stem_multiplier * num_ch  # 3*16
    # stem network, convert 3 channel to c_curr
    self.stem0 = nn.Sequential(
      nn.Conv2d(3, C_curr // 2, kernel_size=3, stride=2, padding=1, bias=False),
      nn.BatchNorm2d(C_curr // 2),
      nn.ReLU(inplace=True),
      nn.Conv2d(C_curr // 2, C_curr, 3, stride=2, padding=1, bias=False),
      nn.BatchNorm2d(C_curr))

    self.stem1 = nn.Sequential(
      nn.ReLU(inplace=True),
      nn.Conv2d(C_curr, C_curr, 3, stride=1, padding=1, bias=False),
      nn.BatchNorm2d(C_curr))

    # c_curr means a factor of the output channels of current cell
    # output channels = multiplier * c_curr
    C_prev_prev, C_prev, C_curr = C_curr, C_curr, num_ch  # 48, 48, 16
    self.cells = nn.ModuleList()
    reduction_prev = False
    for i in range(num_cell):

      # for layer in the middle [1/3, 2/3], reduce via stride=2
      if i in [num_cell // 3]:
      # if i in [num_cell // 3, 2 * num_cell // 3]:
        C_curr *= 2
        reduction = True
      else:
        reduction = False

      # [cp, h, h] => [multiplier*c_curr, h/h//2, h/h//2]
      # the output channels = multiplier * c_curr
      cell = DartsCell(num_node, multiplier, C_prev_prev, C_prev, C_curr, reduction, reduction_prev, search_space)
      # update reduction_prev
      reduction_prev = reduction

      self.cells += [cell]

      C_prev_prev, C_prev = C_prev, multiplier * C_curr

    # adaptive pooling output size to 1x1
    # self.global_pooling = nn.AdaptiveAvgPool2d(1)
    # since cp records last cell's output channels
    # it indicates the input channel number
    # self.classifier = nn.Linear(C_prev, num_class)

    # k is the total number of edges inside single cell, 14
    k = sum(1 for i in range(self.num_node) for j in range(2 + i))
    num_ops = len(self.search_space)  # 8

    self.alpha_normal = nn.Parameter(torch.randn(k, num_ops))
    self.alpha_reduce = nn.Parameter(torch.randn(k, num_ops))
    with torch.no_grad():
      # initialize to smaller value
      self.alpha_normal.mul_(1e-3)
      self.alpha_reduce.mul_(1e-3)
    self._arch_parameters = [self.alpha_normal, self.alpha_reduce]

  def forward(self, x):
    """
    in: torch.Size([b, 3, 255, 255])
    stem0: torch.Size([b, 144, 64, 64])
    stem1: torch.Size([b, 144, 64, 64])
    cell: 0 torch.Size([b, 192, 64, 64]) False
    cell: 1 torch.Size([b, 192, 64, 64]) False
    cell: 2 torch.Size([b, 384, 32, 32]) True
    cell: 3 torch.Size([b, 384, 32, 32]) False
    cell: 4 torch.Size([b, 384, 32, 32]) False
    cell: 5 torch.Size([b, 384, 32, 32]) False
    cell: 6 torch.Size([b, 384, 32, 32]) False
    cell: 7 torch.Size([b, 384, 32, 32]) False
    :param x:
    :return:
    """
    # print('in:', x.shape)
    # s0 & s1 means the last cells' output
    # s0 = s1 = self.stem(x)  # [b, 3, 32, 32] => [b, 48, 32, 32]
    s0 = self.stem0(x)
    # print('stem0 : ',s0.shape)
    s1 = self.stem1(s0)
    # print('stem:', s0.shape)
    # print('stem1 : ',s1.shape)
    out=[]
    for i, cell in enumerate(self.cells):
      if cell.reduction:  # if current cell is reduction cell
        weights = F.softmax(self.alpha_reduce, dim=-1)
      else:
        weights = F.softmax(self.alpha_normal, dim=-1)  # [14, 8]
      s0, s1 = s1, cell(s0, s1, weights)  # [40, 64, 32, 32]
      if i in self.used_layers:
        out.append(s1)
    #   print(i,' : ',s1.shape)

    # print("len out : ",len(out))
    # print("used_layers : ",self.used_layers)
    if len(out)==1:
      return out[0]
    else:
      return out

  def forward_latency(self,x):
    # todo : stem_forward_latency
    latency = 0.
    s0=(144,64,64)
    s1=s0
    # print("after stem shape : ",s0)
    # cell_forward_latency
    for i, cell in enumerate(self.cells):
      if cell.reduction:
        weights = F.softmax(self.alpha_reduce, dim=-1)
      else:
        weights = F.softmax(self.alpha_normal, dim=-1)
      # execute cell() firstly and then assign s0=s1, s1=result
      cell_latency,s = cell.forward_latency(s0,s1,weights)
      latency+=cell_latency
      s0,s1 = s1,s
      # print('cell:',i, s1, cell.reduction, cell.reduction_prev,cell_latency.item())
    return latency

  def genotype(self):
    def _parse(weights):
      """
      :param weights: [14, 8]
      :return:
      """
      gene = []
      n = 2
      start = 0
      for i in range(self.num_node):  # for each node
        end = start + n
        W = weights[start:end].copy()  # shape=[2, 8], [3, 8], [4, 8], [5, 8]
        # i+2 is the number of connection for node i
        # sort by descending order, get strongest 2 edges
        # note here we assume the 0th op is none op, if it's not the case this will be wrong!
        edges = np.argsort(-np.max(W[:, 1:], axis=1))[:2]
        ops   = np.argmax(W[edges, 1:], axis=1) + 1
        gene += [(self.search_space[op], edge) for op, edge in zip(ops, edges)]
        start = end
        n += 1
      return gene

    gene_normal = _parse(F.softmax(self.alpha_normal, dim=-1).data.cpu().numpy())
    gene_reduce = _parse(F.softmax(self.alpha_reduce, dim=-1).data.cpu().numpy())

    concat = range(2 + self.num_node - self.multiplier, self.num_node + 2)
    genotype = Genotype(normal=gene_normal, normal_concat=concat,
                        reduce=gene_reduce, reduce_concat=concat)

    return genotype

def darts_latency_supernet(**kwargs):
    """Constructs a Darts_latency_supernet model.

    """
    # model = NetworkCIFAR(**kwargs)
    kwargs['search_space']=build_search_space(cfg)
    model = SuperNet(**kwargs)
    return model


import pytest
from selenium import webdriver
import time
import math

@pytest.fixture(scope="function")
def browser():
    print("\nstart browser for test..")
    # инициализируем драйвер браузера. После этой команды вы должны увидеть новое открытое окно браузера
    chrome_opt = webdriver.ChromeOptions()
    # удаляем gpu ошибку из терминала
    chrome_opt.add_argument('--disable-gpu')
    # path to driver
    path = r"C:/chromedriver/chromedriver.exe"
    # declare and initialize driver variable
    browser = webdriver.Chrome(executable_path=path, options=chrome_opt)
    yield browser
    print("\nquit browser..")
    browser.quit()

@pytest.mark.parametrize('number', ["236895", "236896", "236897",
    "236898", "236899", "236903", "236904", "236905"])
def test_use_parameters(browser, number):
    link = f"https://stepik.org/lesson/{number}/step/1"
    browser.get(link)

    answer = math.log(int(time.time()))
    print(answer)

    time.sleep(7)

    text_input = browser.find_element_by_css_selector('textarea.textarea')
    text_input.clear()
    text_input.send_keys(str(answer))

    button = browser.find_element_by_css_selector(".submit-submission")
    button.click()
    time.sleep(2)

    actual_result = browser.find_element_by_css_selector(".smart-hints__hint").text
    expected_result = "Correct!"

    assert actual_result == expected_result

# pytest -s -v test_parameters.py


from django.template import Context, loader
from django.http import HttpResponse
from django.template import RequestContext
from django.shortcuts import get_object_or_404, render
from urllib import urlopen
import json
import locale
import re

def index(request):
  # Function-wide local variable
  url_prefix = "http://data.benzinga.com/stock/"

  # Convert balance to a currency string
  locale.setlocale(locale.LC_ALL, '')
  balanceFloat = request.session.get("balance", 100000.00)
  balance = locale.currency(balanceFloat, grouping=True)

  # Get the portfolio
  portfolio = request.session.get('portfolio', {})

  if request.method == 'POST':
    # If an order to buy or sell has been placed
    if request.POST.get("order", None) is not None and request.POST.get("quant", None) is not None and request.POST.get("symbol", None) is not None:

      symbol = request.POST["symbol"]
      jsonurl = urlopen(url_prefix + symbol)
      jdata = json.loads(jsonurl.read())
      if re.match('^[\w-]+$', symbol) is None:
        params = {'balance': balance, 'portfolio': portfolio, 'error': "Error: Invalid symbol"}
        return render(request, 'index.html', params)
      if jdata.get("status", None) is not None and jdata["status"] == "error":
        params = {'balance': balance, 'portfolio': portfolio, 'error': "Error: Symbol not found"}
        return render(request, 'index.html', params)

      # If an invalid quantity has been provided
      if request.POST["quant"].isdigit() is False or int(request.POST["quant"]) <= 0:
        params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio,
        'error': "Quantity must be a positive value", 'jdata': jdata}
        return render(request, 'index.html', params)

      # If an order to buy shares has been placed
      elif request.POST["order"] == "Buy" and request.POST.get("ask", None) is not None:
        quantity = int(request.POST["quant"])
        price = float(request.POST["ask"])
        if quantity*price > balanceFloat:
          # This order cannot be placed
          params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio,
          'error': "Error: Order exceeds balance", 'jdata': jdata}
          return render(request, 'index.html', params)
        if symbol in portfolio:
          # Increase the count of shares
          portfolio[symbol]["quantity"] += quantity
        else:
          # Add this symbol to the portfolio
          portfolio[symbol] = {}
          portfolio[symbol]["name"] = jdata["name"]
          portfolio[symbol]["quantity"] = quantity
          portfolio[symbol]["price"] = price
        balanceFloat -= quantity*price

        # Update the session variables and load the page
        request.session["balance"] = balanceFloat
        request.session["portfolio"] = portfolio
        balance = locale.currency(balanceFloat, grouping=True)
        params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio, 'jdata': jdata}
        return render(request, 'index.html', params)

      # If an order to sell shares has been placed
      elif request.POST["order"] == "Sell" and request.POST.get("bid", None) is not None:
        quantity = int(request.POST["quant"])
        price = float(request.POST["bid"])
        if symbol not in portfolio:
          # This order cannot be placed
          params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio,
          'error': "Error: There are no shares for this stock in the portfolio", 'jdata': jdata}
          return render(request, 'index.html', params)
        elif quantity > int(portfolio[symbol]["quantity"]):
          # This order cannot be placed
          params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio,
          'error': "Error: Selling more shares for this stock than in the portfolio", 'jdata': jdata}
          return render(request, 'index.html', params)
        portfolio[symbol]["quantity"] -= quantity
        if portfolio[symbol]["quantity"] == 0:
          # Remove this stock from the portfolio
          del portfolio[symbol]
        balanceFloat += quantity*price

        # Update the session variables and load the page
        request.session["balance"] = balanceFloat
        request.session["portfolio"] = portfolio
        balance = locale.currency(balanceFloat, grouping=True)
        params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio, 'jdata': jdata}
        return render(request, 'index.html', params)

      # A bad order request has been made; return the default page
      else:
        params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio}
        return render(request, 'index.html', params)

    # If a symbol is being retrieved
    elif request.POST.get("symbol", None) is not None:
      symbol = request.POST["symbol"]
      jsonurl = urlopen(url_prefix + symbol)
      jdata = json.loads(jsonurl.read())

      if re.match('^[\w-]+$', symbol) is None:
        params = {'balance': balance, 'portfolio': portfolio, 'error': "Error: Invalid symbol"}
        return render(request, 'index.html', params)
      if jdata.get("status", None) is not None and jdata["status"] == "error":
        params = {'balance': balance, 'portfolio': portfolio, 'error': "Error: Symbol not found"}
        return render(request, 'index.html', params)

      params = {'symbol': symbol, 'balance': balance, 'portfolio': portfolio, 'jdata': jdata}
      return render(request, 'index.html', params)

    # Invalid POST parameters; return the default page
    else:
      params = {'balance': balance, 'portfolio': portfolio}
      return render(request, 'index.html', params)

  else:
    # Default page
    params = {'balance': balance, 'portfolio': portfolio}
    return render(request, 'index.html', params)


import pygame
from sprites import *
from config import *
import sys

class Game:
    def __init__(self):
        pygame.init()
        self.screen = pygame.display.set_mode((WIN_WIDTH, WIN_HEIGHT), pygame.RESIZABLE)
        self.clock = pygame.time.Clock()
        self.running = True
        self.font = pygame.font.Font(None, 50)

        self.intro_background = pygame.image.load('menu.jpg')
        self.go_background = pygame.image.load('Gameover.jpg')

    def createTilemap(self):
        for i, row in enumerate(tilemap):
            for j, column in enumerate(row):
                if column == 'A':
                    Ground(self, j, i)
                if column == 'B':
                    Block(self, j, i)
                if column == 'E':
                    Enemy(self, j, i)
                if column == "P":
                    Player(self, j, i)

    def new(self):
        # a new game starts
        self.playing = True

        self.all_sprites = pygame.sprite.LayeredUpdates()
        self.blocks = pygame.sprite.LayeredUpdates()
        self.enemies = pygame.sprite.LayeredUpdates()
        self.attacks = pygame.sprite.LayeredUpdates()

        self.createTilemap()

    def events(self):
        # game loop events
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                self.playing = False
                self.running = False

    def update(self):
        # game loop updates
        self.all_sprites.update()

    def draw(self):
        # game loop draw
        self.screen.fill(BLACK)
        self.all_sprites.draw(self.screen)
        self.clock.tick(FPS)
        pygame.display.update()

    def main(self):
        #game loop
        while self.playing:
            self.events()
            self.update()
            self.draw()

    def game_over(self):
        restart_button = Button(606.5, 480, 338, 95, WHITE, BLACK, 'TRY AGAIN', 70)

        for sprite in self.all_sprites:
            sprite.kill()

        while self.running:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    self.running = False

            mouse_pos = pygame.mouse.get_pos()
            mouse_pressed = pygame.mouse.get_pressed()

            if restart_button.is_pressed(mouse_pos, mouse_pressed):
                self.new()
                self.main()

            self.screen.blit(self.go_background, (0, 0))
            self.screen.blit(restart_button.image, restart_button.rect)
            self.clock.tick(FPS)
            pygame.display.update()

    def intro_screen(self):
        intro = True

        title = self.font.render('', True, BLACK)
        title_rect = title.get_rect(x=10, y=10)

        play_button = Button(455, 400, 338, 95, WHITE, BLACK, 'PLAY', 80)
        exit_button = Button(455, 550, 338, 95, WHITE, BLACK, 'EXIT', 80)

        while intro:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    intro = False
                    self.running = False

            mouse_pos = pygame.mouse.get_pos()
            mouse_pressed = pygame.mouse.get_pressed()

            if play_button.is_pressed(mouse_pos, mouse_pressed):
                intro = False
            if exit_button.is_pressed(mouse_pos, mouse_pressed):
                intro = False
                self.running = False

            self.screen.blit(self.intro_background, (0, 0))
            self.screen.blit(title, title_rect)
            self.screen.blit(play_button.image, play_button.rect)
            self.screen.blit(exit_button.image, exit_button.rect)
            self.clock.tick(FPS)
            pygame.display.update()

g = Game()
g.intro_screen()
g.new()
while g.running:
    g.main()
    g.game_over()

pygame.quit()
sys.exit()


import json
import argparse
from flask import Flask, jsonify, redirect, render_template
import subprocess
import requests
app = Flask(__name__)

json_path = "./config/val/val.json"

@app.route('/')
def hello():
    names = get_names()
    # return 'on
off
on'
    return render_template('index.html', names=names)

@app.route('/on')
def on():
    res = send_ir('light:on')
    if res:
        return jsonify({'status': False})
    else:
        return jsonify({'status': True})

@app.route('/off')
def off():
    res = send_ir('light:off')
    if res:
        return jsonify({'status': False})
    else:
        return jsonify({'status': True})

@app.route("/send/<name>")
def archive(name=""):
    if name == "":
        return jsonify({'status': False})
    else:
        res = send_ir(name)

        if res:
            return jsonify({'status': False})
        else:
            return jsonify({'status': True})

@app.route("/names")
def names():
    names = get_names()
    ret = []
    for name in names:
        ret.append({'name': name})
    return jsonify(ret)

def send_ir(name):
    command = ['python', 'irrp.py', '-p', '-g27',
               '-f', json_path, name]
    res = subprocess.call(command)
    return res

def get_names():
    with open(json_path, "r") as f:
        records = json.load(f)
    keys = list(records.keys())
    keys.reverse()
    return keys

if __name__ == "__main__":
    app.run(debug=True, host='0.0.0.0')


import tweepy

def get_api(cfg):
  auth = tweepy.OAuthHandler(cfg['consumer_key'], cfg['consumer_secret'])
  auth.set_access_token(cfg['access_token'], cfg['access_token_secret'])
  return tweepy.API(auth)

def tweet(contents : str):
  cfg = {
    "consumer_key"        : "",
    "consumer_secret"     : "",
    "access_token"        : "",
    "access_token_secret" : ""
    }

  api = get_api(cfg)
  if(contents.count(contents) <= 140):
    status = api.update_status(status=contents)
    return "Tweet sent"
  else:
      print("Too many characters to tweet!")
      return "Tweet error: Too many characters to tweet"


# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""A very simple MNIST classifier.
See extensive documentation at
https://www.tensorflow.org/get_started/mnist/beginners
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import sys

from tensorflow.examples.tutorials.mnist import input_data

import tensorflow as tf
from ffnn import mlp
import pdb

FLAGS = None

def main(_):

# Import data
    mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
    mlp_name = 'TEST_MLP'
    dtype = tf.float32
    input_size = 784
    output_size = 10
    hidden_size = 100000
    act_func = None

    weight_params = {'dtype': dtype,
                    # 'weight_init': 'uniform',
                     'weight_init': 'normal',
                     'weight_init_mean': -0.000005,
                     'weight_init_stddev': 0.000005,
                    # 'weight_init_max': 0.05,
                    # 'weight_init_min': -5,
                     'weight_decay': None,
                     'variable_share': False}

    mlp_params = {'name': mlp_name,
                  'IO_dtype': dtype,
                  'input_size': input_size,
                  'output_size': output_size,
                  'hidden_size': hidden_size,
                  'hidden_act': act_func,
                  'output_act': None,
                  'weight_params': weight_params}

    mlp_net = mlp(mlp_params)
    with tf.Graph().as_default():
        input_pl = tf.placeholder(dtype=dtype,
                                  shape=(None, input_size),
                                  name='input_pl')
        output_pl = tf.placeholder(dtype=dtype,
                                   shape=(None, output_size),
                                   name='input_pl')
        mnist_logits = mlp_net.build(input_pl)
        sess = tf.Session()
        cross_entropy = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits(labels=output_pl,
                                                    logits=mnist_logits))
        # train_step = tf.train.AdamOptimizer(learning_rate=0.000001,
        #                                     epsilon=0.1).minimize(cross_entropy)
        train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
        correct_prediction = tf.equal(tf.argmax(mnist_logits, 1),
                                      tf.argmax(output_pl, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction,
                                          tf.float32))
        sess.run(tf.global_variables_initializer())
        for _ in range(1000):
            batch_xs, batch_ys = mnist.train.next_batch(1000)
            sess.run(train_step, feed_dict={input_pl: batch_xs,
                                            output_pl: batch_ys})

            # Test trained model
            print(sess.run(accuracy, feed_dict={input_pl: mnist.test.images,
                                                output_pl: mnist.test.labels}))

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data',
                      help='Directory for storing input data')
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


# =============================================================================
# Results_bar_chart.py
# =============================================================================
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

results = pd.read_csv('Results.csv', parse_dates=True,
                      index_col=0, error_bad_lines=False)

#results = results.drop(['DTR'])

mae = results.iloc[:, 0]
rmse = results.iloc[:, 1]

labels = results.index

x = np.arange(len(labels))  # the label locations
width = 0.45  # the width of the bars

# Plots
plt.rc('xtick', labelsize=18)
plt.rc('ytick', labelsize=18)

fig, ax = plt.subplots()
ax2 = ax.twinx()
rects1 = ax.bar(x - width/2, mae, width,
                label='MAE', color='#e62929', zorder=10)
rects2 = ax2.bar(x + width/2, rmse, width, label='RMSE', color='#3d4de0')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('MAE', fontsize=18)
ax2.set_ylabel('RMSE', fontsize=18)
ax.set_xlabel('Model', fontsize=18)
ax.set_title('Performance of all models', fontsize=20)
ax.set_xticks(x)
ax.set_xticklabels(labels, fontsize=16)
#ax.legend(loc='upper right', fontsize=13)
leg2 = fig.legend([rects1, rects2],
                  ['MAE', 'RMSE'],
                  bbox_to_anchor=(0.125, 1.01),
                  bbox_transform=ax.transAxes, fontsize=18)
ax.grid(True)
ax.autoscale()

def autolabel(rects, axi):
    """Attach a text label above each bar in *rects*, displaying its height."""
    if axi == 1:
        for rect in rects:
            height = rect.get_height()
            ax.annotate('{}'.format(height),
                        xy=(rect.get_x() + rect.get_width() / 2 - 0.02, height),
                        xytext=(0, 3),  # 3 points vertical offset
                        textcoords="offset points",
                        ha='center', va='bottom',
                        fontsize=14, color='#e62929')
    else:
        for rect in rects:
            height = rect.get_height()
            ax2.annotate('{}'.format(height),
                         xy=(rect.get_x() + rect.get_width() / 2 + 0.02, height),
                         xytext=(0, 3),  # 3 points vertical offset
                         textcoords="offset points",
                         ha='center', va='bottom',
                         fontsize=14, color='#3d4de0')

autolabel(rects1, 1)
autolabel(rects2, 2)

fig.tight_layout()

plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
June 24, 2019

Figure 8. A) Alongshore beach width change for Bogue Banks for 1997-2016 (green), 1997-2010 (red),
and 2010-2016 (blue). B) Density distributions of beach width change for Bogue Banks for 1997-2016,
1997-2010, and 2010-2016. Positive values indicate progradation, negative values indicate erosion.
C) Mean beach width change for the entire island, Fort Macon, non-fenced areas, and fenced areas
colored by time period (with 95% confidence intervals).

- Creates the figure and a .txt file with the statistics used

@author: michaelitzkin
"""

import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import scipy.signal as sig
import scipy.stats as stats
import seaborn as sns
import pandas as pd
import numpy as np
import os

pd.set_option('mode.chained_assignment', None)

"""
Utility functions
"""

def confidence_intervals(data):
    """
    Calculate the 95% confidence interval
    """

    x_bar = np.nanmean(data)    # Mean value
    s = np.nanstd(data)         # Standard deviation
    n = len(data)               # Sample size

    lo_conf = x_bar - (1.96 * (s / np.sqrt(n)))  # Lower bound of confidence interval
    hi_conf = x_bar + (1.96 * (s / np.sqrt(n)))  # Upper bound of confidence interval

    conf_range = hi_conf - lo_conf  # Size of the 95% confidence interval

    return lo_conf, hi_conf, conf_range

def chunks(l, n):
    """
    Split list l into n sized chunks

    https://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks
    """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def estimated_autocorrelation(x):
    """
    http://stackoverflow.com/q/14297012/190597
    http://en.wikipedia.org/wiki/Autocorrelation#Estimation

    Called by:
    - rate_uncertainty()
    """
    n = len(x)
    variance = np.nanvar(x)
    x -= np.nanmean(x)
    r = np.correlate(x, x, mode='full')[-n:]
    # assert np.allclose(r, np.array([(x[:n - k]*x[-(n - k):]).sum() for k in range(n)]))
    result = r / (variance * np.arange(n, 0, -1))
    return result

def fft_smooth(y, points):
    """
    Smooth noisy data with an FFT filter

    https://stackoverflow.com/questions/20618804/how-to-smooth-a-curve-in-the-right-way
    """
    box = np.ones(points)/points
    y_smooth = np.convolve(y, box, mode='same')
    return y_smooth

def lidar_vertical_error(yr):
    """
    Return the vertical error of the lidar data for the
    given year. Returns in meters
    """

    if yr in ['1997', '1998', '1999', '2000', '2004', '2010', '2011']:
        return 0.15
    elif yr == '2005':
        return 0.20
    elif yr == '2014':
        return 0.062
    elif yr == '2016':
        return 0.20

def mean_residual_error(df, delta):
    """
    Calculate the mean residual error for the initial and final
    time periods. Than use that to calculate an average rate error.
    - df: Dataframe being worked on
    - delta: Dataframe of change rates
    - window: Window to average over
    """

    # Perform an autocorrelation to determine the window size
    autocorr = estimated_autocorrelation(delta)
    sig_vals = np.where(autocorr <= 0.05)
    n_star = np.nanmin(sig_vals)

    row = 0
    for beta in chunks(delta, n_star):

        # Calculate the average change rate of the
        # current chunk of profiles
        mu_beta = np.nanmean(beta)

        # Calculate the residuals for the current chunk
        residuals = beta - mu_beta

        # Calculate the MRE for the current chunk
        mre = np.sqrt((1 / len(beta)) * np.nansum(residuals ** 2))

        # Add the error to the dataframe
        df[row:row+len(beta)] = mre

        # Increment the row
        row += len(beta)

    return df

def rate_uncertainty(t0, tf, pos_uncertainty):
    """
    Calculate uncertainty in the change rate, based on
    methods in Hapke et al. (2011)

    - t0: beginning time
    - tf: end time
    - pos_uncertainties: positional uncertainty

    Called by:
    - shoreline_change_df()
    """

    # Convert the times to ints
    t0, tf = int(t0), int(tf)

    # Find the rate uncertainty based on the positional uncertainties
    ur = pos_uncertainty / (tf - t0)

    return ur

def regional_uncertainty(r, ur, alpha=0.05):
    """
    Calculate reigonal uncertainty. From Hapke et al (2011),
    equation 9

    - r: Calculated rates of change
    - alpha: significance value (default = 0.05)
    """

    # Perform an autocorrelation on r to determine
    # the number of independent samples
    r_acr = estimated_autocorrelation(ur[~np.isnan(ur)])
    r_acr_vals = np.where(r_acr <= alpha)
    n_star = np.nanmin(r_acr_vals)

    # Calculate the mean uncertainty
    Ur_bar = np.nanmean(ur)

    # Calculate the uncertainty
    Urq_bar = (1 / np.sqrt(n_star)) * Ur_bar
    Urq_bar = np.around(Urq_bar, decimals=2)
    return Urq_bar

"""
Functions to load an format data
"""

def beach_width_change_df(widths, rate=False):
    """
    Calculate the average rate of change for the beach width for each interval and its associated
    uncertainty. The uncertainty method is based on that used in Hapke et al (2012)
    but modified for the data outputted from Automorph.
    """

    # Set the intervals to work with
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    new_df = widths[['1997', '2010', '2016', '1997-2010', '1997-2016', '2010-2016']]

    # At this point, there is a column for the x-position of the shoreline for every year.
    # Add columns for the rates
    for interval in intervals:

        # Create new columns
        orig_df, begin_df, end_df, begin_time, end_time, dt = setup_change_df(interval, new_df)

        # Calculate the change rate between the years being analyzed
        change = end_df - begin_df
        new_df[interval + ' Change'] = change
        new_df[interval + ' Mean'] = np.nanmean(change)

        # Calculate uncertainty in the MHW position
        up1, up2 = mhw_uncertainty(begin_time, end_time)

        # Calculate uncertainty in the dune toe position
        up3, up4 = dlow_uncertainty(begin_df, end_df, begin_time, end_time)

        # Calculate the rate uncertainty and regional uncertainty
        # based on the positional uncertainty
        position_uncertainty = np.sqrt((up1 ** 2) + (up2 ** 2) + (up3 ** 2) + (up4 ** 2))
        ur = rate_uncertainty(begin_time, end_time, position_uncertainty)
        region_uncertainty = regional_uncertainty(change, ur)
        new_df[interval + ' Mean Residual Error'] = mean_residual_error(new_df[interval + ' Mean Residual Error'], change)
        new_df[interval + ' Uncertainty'] = abs(ur)
        new_df[interval + ' Regional Uncertainty'] = abs(region_uncertainty)

    # Save the dataframe as a .csv
    save_name = os.path.join('..', 'Data', 'Beach Widths.csv')
    new_df.to_csv(save_name)

    # Return the new dataframe
    return new_df

def change_df(d1997, d2010, d2016):
    """
    Make a dataframe of change rates between lidar
    surveys and their smoothed trends

    - d1997: Dataframe with values from 1997
    - d2010: Dataframe with values from 2010
    - d2016: Dataframe with values from 2016
    """

    # Set the intervals to work with
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    # Concatenate the data into one dataframe
    new_df = pd.concat([d1997, d2010, d2016], axis=1, keys=['1997', '2010', '2016'])

    # Add columns for the rates
    for interval in intervals:

        # Add a column with the change rate
        new_df[interval] = (new_df[interval[-4:]] - new_df[interval[0:4]])

        # Add a column with the smoothed trend
        new_col_name = interval + ' Trend'
        new_df[new_col_name] = sig.savgol_filter(x=new_df[interval], window_length=1001, polyorder=3)
        new_df[interval + ' FFT'] = fft_smooth(new_df[interval], 2002)

    return new_df

def dlow_uncertainty(d0, df, t0, tf):
    """
    Calculate uncertainty in the position of the dune
    toe, based on methods in Hapke et al (2011)

    - d0: Data at time 0
    - df: Data at time f
    - t0: time 0
    - tf: time f
    """

    # Load the y_mhw values and beach slopes for the current years to calculate uncertainty
    data_begin = load_data(year=int(t0), variable=['x_toe', 'Foreshore Slope'])
    data_end = load_data(year=int(tf), variable=['x_toe', 'Foreshore Slope'])

    # Find the uncertainty in the position of the dune toe
    toe_error_yr1 = lidar_vertical_error(str(t0)) / data_begin['Foreshore Slope']
    toe_error_yr2 = lidar_vertical_error(str(tf)) / data_begin['Foreshore Slope']

    # Find uncertainty based on deviation from the average
    toe_mean_yr1 = np.nanmean(data_begin['x_toe'].dropna())
    toe_mean_yr2 = np.nanmean(data_end['x_toe'].dropna())
    toe_error_2_yr1 = abs(data_begin['x_toe'] - toe_mean_yr1)
    toe_error_2_yr2 = abs(data_end['x_toe'] - toe_mean_yr2)

    # Find uncertainty based on the confidence interval
    _, _, ci0 = confidence_intervals(d0)
    _, _, cif = confidence_intervals(df)

    # Find the positional uncertainty for each year
    up1 = np.sqrt((toe_error_yr1 ** 2) + (toe_error_2_yr1 ** 2) + (ci0 ** 2))
    up2 = np.sqrt((toe_error_yr2 ** 2) + (toe_error_2_yr2 ** 2) + (cif ** 2))

    return up1, up2

def load_data(year, variable):
    """
    Load the data into Pandas dataframes. Supply the years needed
    as the input argument

    - year: Int for the year to look at
    - variable: String with the name of the column to use
    """

    # Set a path to the data
    fname = os.path.join('..', 'Data', 'Morphometrics for Bogue ' + str(year) + '.csv')

    # Load the data and pull the needed column
    if variable is not None:
        data = pd.read_csv(fname, header=0, delimiter=',')[variable]
    else:
        data = pd.read_csv(fname, header=0, delimiter=',')

    return data

def load_fence_locations(y):
    """
    Create a dataframe with profile numbers, category,
    and an associated y-value. The y-value is passed to the
    alongshore plotting function and is the y-axis position

    0 - Fort Macon
    1 - Non-fenced
    2 - Fenced
    """

    # Load the data. Can be from any year 2010-2016, results
    # won't change for this application
    data = load_data(2010, None)

    # Create a fence category column and initialize everything to
    # a value of 0
    data['Category'] = 'darkred'
    data['Number'] = 0

    # Set the rest of the categories
    fort_macon = 2229  # Profiles in from the end where Fort Macon starts
    data['Category'].iloc[:-fort_macon].loc[np.isnan(data['x_fence'])] = 'dimgray'
    data['Category'].iloc[:-fort_macon].loc[~np.isnan(data['x_fence'])] = 'cornflowerblue'

    data['Number'].iloc[:-fort_macon].loc[np.isnan(data['x_fence'])] = 1
    data['Number'].iloc[:-fort_macon].loc[~np.isnan(data['x_fence'])] = 2

    # Add a y-value column
    data['Y'] = y

    # Sort out fences using chunks
    for n in chunks(data[:fort_macon], 5):
        if np.any(data['x_fence'].iloc[n.index] > 0):
            data['Number'].iloc[n.index] = 2
        else:
            data['Number'].iloc[n.index] = 1

    # Just take the needed data
    data = data[['Profile No.', 'Category', 'Number', 'Y']]

    # Return the dataframe
    return data

def mhw_uncertainty(t0, tf):
    """
    Calculate uncertainty in the position of MHW on the profile
    based on methods in Hapke et al. (2011)

    - begin: dataframe of shoreline positions for the beginning time
    - end: dataframe of shoreline positions for the end time
    - t0: beginning time
    - tf: end time

    Called by:
    - shoreline_change_df()
    """

    # Load the y_mhw values and beach slopes for the current years to calculate uncertainty
    data_begin = load_data(year=int(t0), variable=['y_mhw', 'Beach Slope'])
    data_end = load_data(year=int(tf), variable=['y_mhw', 'Beach Slope'])

    # Use the confidence intervals as the first component of the error
    ci_df = pd.read_csv(os.path.join('..', 'Data', 'Shoreline Confidence Intervals.csv'))

    # Find the uncertainty in Y as the difference between the measured y-value
    # and the accepted value of MHW (0.34m). Do this for both years
    mhw_real = 0.34
    y_error_yr1 = abs(data_begin['y_mhw'] - mhw_real)
    y_error_yr2 = abs(data_end['y_mhw'] - mhw_real)

    # Find the uncertainty in X as the vertical error in the LiDAR
    # data divided by the beach slope
    x_error_yr1 = lidar_vertical_error(str(t0)) / data_begin['Beach Slope']
    x_error_yr2 = lidar_vertical_error(str(tf)) / data_end['Beach Slope']

    # Incorporate 95% confidence interval from the regression into the error analysis. There are more
    # confidence intervals than profiles, remove the unnecessary ones
    ci_error_1 = ci_df[str(t0) + ' CI'].iloc[:len(x_error_yr1)]
    ci_error_2 = ci_df[str(tf) + ' CI'].iloc[:len(x_error_yr1)]

    # Convert to arrays
    x_error_yr1, x_error_yr2 = np.asarray(x_error_yr1, dtype=np.float64), np.asarray(x_error_yr2, dtype=np.float64)
    y_error_yr1, y_error_yr2 = np.asarray(y_error_yr1, dtype=np.float64), np.asarray(y_error_yr2, dtype=np.float64)
    ci_error_1, ci_error_2 = np.asarray(ci_error_1, dtype=np.float64), np.asarray(ci_error_2, dtype=np.float64)

    # Find the positional uncertainty based on the uncertainty in X and Y
    up1 = np.sqrt((x_error_yr1 ** 2) + (y_error_yr1 ** 2) + (ci_error_1 ** 2))
    up2 = np.sqrt((x_error_yr2 ** 2) + (y_error_yr2 ** 2) + (ci_error_2 ** 2))

    return up1, up2

def setup_change_df(interval, orig_df):
    """
    Prepare the dataframe for the change dataframes
    with uncertainty values

    Called by:
    - shoreline_change_df()
    """

    # Get the two times associated with the interval
    begin_time = str(interval[0:4])  # Initial time as a string
    end_time = str(interval[-4:])  # Final time as a string
    dt = (int(end_time) - int(begin_time))  # Time span as an int
    begin_df = orig_df[begin_time]  # Create a dataframe of shoreline positions from the initial time
    end_df = orig_df[end_time]  # Create a dataframe of shoreline positions from the final time

    # Create new column names
    new_col_name_1 = interval + ' Change'
    new_col_name_2 = interval + ' Mean'
    new_col_name_3 = interval + ' Uncertainty'
    new_col_name_4 = interval + ' Regional Uncertainty'
    new_col_name_5 = interval + ' Mean Residual Error'

    # Need to copy other columns before working
    orig_df[new_col_name_1] = orig_df['1997']
    orig_df[new_col_name_2] = orig_df['1997']
    orig_df[new_col_name_3] = orig_df['1997']
    orig_df[new_col_name_4] = orig_df['1997']
    orig_df[new_col_name_5] = orig_df['1997']

    return orig_df, begin_df, end_df, begin_time, end_time, dt

"""
Make the figure
"""

def bar_plots(data, fences, ax3, ax4, ax5, ax6, rate, text_y, bar_y, color, param):
    """
    This function makes a series of bar plots showing the change rates broken down by
    Fort Macon, non-fenced, and fenced locations as well as the island overall. Each location will
    have three bars for 1997-2016, 1997-2010, 2010-2016

    Called by;
    - make_overall_change_plot_w_kde()
    """

    conf_type = 95

    # Set colors and locations
    colors = ['red', 'blue', 'green']
    locations = ['Fort Macon', 'Non-fenced', 'Fenced']

    # Make a dataframe for all locations
    all_locs = pd.DataFrame({'1997-2010': data['1997-2010'].dropna(),
                             '2010-2016': data['2010-2016'].dropna(),
                             '1997-2016': data['1997-2016'].dropna()})
    all_locs['ID'] = 'All'
    all_locs = pd.melt(all_locs, id_vars=['ID'], value_vars=['1997-2010', '2010-2016', '1997-2016'])

    # Make a barplot of the data for all locations
    sns.barplot(x='ID', y='value', ci=conf_type, hue='variable', data=all_locs,
                palette=colors, estimator=np.nanmedian, errcolor='black', errwidth=2, ax=ax3)

    # Loop through and plot Fort Macon, non-fenced areas, and fenced areas
    for ii, ax in enumerate([ax4, ax5, ax6]):

        # Set temporary data
        use_data = data.loc[fences['Number'] == ii]

        # Make a dataframe for the location
        temp_locs = pd.DataFrame({'1997-2010': use_data['1997-2010'].dropna(),
                                  '2010-2016': use_data['2010-2016'].dropna(),
                                  '1997-2016': use_data['1997-2016'].dropna()})
        temp_locs['ID'] = locations[ii]
        temp_locs = pd.melt(temp_locs, id_vars=['ID'], value_vars=['1997-2010', '2010-2016', '1997-2016'])

        # Make a barplot of the data for the location
        sns.barplot(x='ID', y='value', ci=conf_type, hue='variable', data=temp_locs,
                    palette=colors, estimator=np.nanmedian, errcolor='black', errwidth=2, ax=ax)

    # Set common axis properties
    for ax in [ax3, ax4, ax5, ax6]:

        # Set grids
        add_grid([ax], xax=False)

        # Add a zero line
        ax.axhline(y=0, color='black', linewidth=2, linestyle='--', zorder=10)

        # Set the y-limits
        ax.set_ylim(bottom=-bar_y, top=bar_y)
        ax.set_yticks([-bar_y, (-bar_y)/2, 0, bar_y/2, bar_y])

        # ax.set_ylim(bottom=-25, top=bar_y)
        # ax.set_yticks([(-bar_y) / 2, 0, bar_y / 2, bar_y])

        # Set the y-axis
        if ax == ax3:
            if rate:
                ax.set_ylabel('Median ' + param + '\nChange Rate (m/yr)',
                              color=color,
                              fontname='Arial',
                              fontsize=12,
                              fontweight='bold')
            else:
                ax.set_ylabel('Median ' + param + '\nChange (m)',
                              color=color,
                              fontname='Arial',
                              fontsize=12,
                              fontweight='bold')
        else:
            ax.set_ylabel('')
            plt.setp(ax.get_yticklabels(), visible=False)
            ax.tick_params(left=False)

        # Remove the x-axis label
        ax.set_xlabel('')

        # Remove the legend
        ax.legend_.remove()

        # Set the spines
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['bottom'].set_linewidth(2)
        ax.spines['bottom'].set_color(color)
        ax.tick_params(width=2)
        if ax == ax3:
            ax.spines['left'].set_linewidth(2)
            ax.spines['left'].set_color(color)
        else:
            ax.spines['left'].set_visible(False)

    # Add a "C" to ax3
    ax3.text(x=-0.35,
             y=bar_y - (bar_y * 0.20),
             s='C.',
             color='black',
             fontname='Arial',
             fontsize=14,
             fontweight='bold')

def tight_and_transparent(fig, axs, color, **kwargs):
    """
    Give the figure a tight layout and a transparent backgorund
    """
    for ax in axs:
        ax.tick_params(colors=color)
        for tick in ax.get_xticklabels():
            tick.set_fontname('Arial')
        for tick in ax.get_yticklabels():
            tick.set_fontname('Arial')

    plt.tight_layout(**kwargs)
    fig.patch.set_color('w')
    fig.patch.set_alpha(0.0)

def set_spines(axs, color):
    """
    Set spines on the lower and left borders. Remove the others
    """

    for ax in axs:

        # Set the spines
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_linewidth(2)
        ax.spines['bottom'].set_linewidth(2)
        ax.spines['left'].set_color(color)
        ax.spines['bottom'].set_color(color)

        # Make the tick widths match
        ax.tick_params(width=2)

def add_grid(axs, xax=True):
    """
    Add a grid to the plot. Set xax=False
    if you only want horizontal lines
    """

    for ax in axs:
        ax.grid(color='grey', linewidth=0.25, zorder=0)
        ax.xaxis.grid(xax)

def make_overall_change_plot_w_kde(data, fences, param, y_min, y_max, y_ticks, y_tick_labels,
                                   y_text, text_y, bar_y, spacing, rate=False, color='black'):
    """
    Make a plot of change rates for the pre-fencing (1997-2010),
    post-fencing (2010-2016), and overall (1997-2016) time periods

    Add a KDE plot in the second subplot

    Called by:
    - Shoreline_Change.py
    """

    # Setup the plot
    fig = plt.figure(dpi=300)
    categories = ['Fort Macon', 'Non-Fenced', 'Fenced']
    gs = gridspec.GridSpec(nrows=2, ncols=4, wspace=0.0, hspace=0.3)
    ax1 = fig.add_subplot(gs[0, 0:3])
    ax2 = fig.add_subplot(gs[0, 3], sharey=ax1)
    ax3 = fig.add_subplot(gs[1, 0])
    ax4 = fig.add_subplot(gs[1, 1], sharey=ax3)
    ax5 = fig.add_subplot(gs[1, 2], sharey=ax3)
    ax6 = fig.add_subplot(gs[1, 3], sharey=ax3)
    lwidth = 2
    max_x = 0
    marker_shape = 'X'
    marker_alpha = 0.075
    x_max = 40000
    exes = [5000, 19300, 32000]
    yrs = ['1997-2016', '1997-2010', '2010-2016']
    labs = ['\n(Overall)', '\n(Pre-Fencing)', '\n(Post-Fencing)']
    cols = ['darkred', 'darkgreen', 'darkblue']
    markers = ['lightsalmon', 'lightgreen', 'lightblue']

    text_years = ['1997-2010', '2010-2016', '1997-2016']
    text_labs = ['\n(Pre-Fencing)', '\n(Post-Fencing)', '\n(Overall)']
    text_cols = ['darkred', 'darkblue', 'green']

    # Add a grid for the top plots
    add_grid([ax1, ax2], xax=True)

    # Plot the data
    for ii, years in enumerate(['1997-2010', '1997-2016', '2010-2016']):

        # Scatter plot of the raw data
        sns.scatterplot(x=data.index,
                        y=years,
                        data=data,
                        linewidth=0,
                        color=markers[ii],
                        marker=marker_shape,
                        alpha=marker_alpha,
                        ax=ax1,
                        zorder=2)

        # Trend line over the raw data
        sns.lineplot(x=data.index,
                     y=years + ' Trend',
                     data=data,
                     linewidth=2,
                     color=cols[ii],
                     ax=ax1,
                     zorder=5)

        # KDE Plot in the second axis
        kde1 = sns.kdeplot(data[years].dropna(),
                           shade=True,
                           color=cols[ii],
                           linewidth=lwidth,
                           legend=False,
                           vertical=True,
                           ax=ax2,
                           zorder=2)

        # Add text labels
        ax1.text(x=exes[ii],
                 y=y_text - (2 * spacing),
                 s=text_years[ii] + text_labs[ii],
                 color=text_cols[ii],
                 fontname='Arial',
                 fontsize=12,
                 fontweight='bold',
                 ha='center',
                 va='center',
                 zorder=8)

        # Get the largest value in the KDE plot to set the axis limit later
        x1, y1 = kde1.get_lines()[0].get_data()
        if np.nanmax(x1) > max_x:
            max_x = np.around(a=np.nanmax(x1), decimals=2)

    # Add a zero-line
    ax1.axhline(y=0,
                xmin=0, xmax=x_max,
                color='black',
                linewidth=2,
                linestyle='--',
                zorder=7)
    ax2.axhline(y=0,
                xmin=0,
                xmax=max_x + 1,
                color='black',
                linewidth=2,
                linestyle='--',
                zorder=7)

    # Add an "A" and "B" label
    ax1.text(x=-0.5, y=y_max - (y_max * 0.20),
             s='A.',
             color='black',
             fontname='Arial',
             fontsize=14,
             fontweight='bold')
    ax2.text(x=max_x - (max_x * 0.91), y=y_max - (y_max * 0.20),
             s='B.',
             color='black',
             fontname='Arial',
             fontsize=14,
             fontweight='bold')

    # Set the x-axis
    ax1.set_xlim(right=x_max)
    ax1.set_xticks([0, 5000, 10000, 15000, 20000, 25000, 30000, 35000, 40000])
    ax1.set_xticklabels(['0', '5', '10', '15', '20', '25', '30', '35', ''])
    ax1.set_xlabel('Alongshore Distance (km)',
                   color=color,
                   fontname='Arial',
                   fontsize=12,
                   fontweight='bold')

    # Set the y-axis
    ax1.set_ylim(bottom=y_min, top=y_max)
    ax1.set_yticks(y_ticks)
    ax1.set_yticklabels(y_tick_labels)
    if rate:
        ax1.set_ylabel(param + ' Change\nRate (m/yr)',
                       color=color,
                       fontname='Arial',
                       fontsize=12,
                       fontweight='bold')
    else:
        ax1.set_ylabel(param + ' Change (m)',
                       color=color,
                       fontname='Arial',
                       fontsize=12,
                       fontweight='bold')
    plt.setp(ax2.get_yticklabels(), visible=False)

    # Add a barplot of change rates on the bottom of the figure
    bar_plots(data, fences, ax3, ax4, ax5, ax6, rate, text_y, bar_y, color, param)

    # Set the spines for the top plots
    set_spines([ax1, ax2], color)

    # Set a tight layout and a transparent background
    tight_and_transparent(fig, [ax1, ax2, ax3, ax4, ax5, ax6], color)

    # Save the figure
    title = 'Figure 8'
    save_name = os.path.join('..', 'Figures', title + '.png')
    plt.savefig(save_name, bbox_inches='tight', dpi='figure')
    plt.close()
    print(f'Figure saved: {save_name}\n')

"""
Functions to make a .txt file
"""

def make_stats(data, avg, fences, param):
    """
    Calculate change statistics about Bogue Banks

    - Mean, median rates for 1997-2010, 2010-2016, and 1997-2016
    - T-Test for 1997-2010 v. 2010-2016 v. 1997-2016
    """

    # Set the intervals to work with
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    # Create a .txt file
    partial_fname = f'(Figure 8) {param} Change Statistics.txt'
    fname = os.path.join('..', 'Text Files', partial_fname)
    f = open(fname, 'w')

    # Write a header
    f.write('%s Change Statistics for Bogue Banks between 1997-2010, 2010-2016, 1997-2016\r\n' % param)
    f.write('\t- Summary statistics between Surveys\r\n')
    f.write('\t- KS-tests to compare pre- and post- fencing rates\r\n')
    f.write('\t- Mann-Whitney U-Tests between pre-fence, post-fence, and overall periods\r\n\r\n\r\n')

    # Write statistics for all of the profiles
    make_stats_section(f, data, avg, 'All Profiles', intervals)

    # Write stats for the different sections
    for ii, sect in enumerate(['Fort Macon', 'Non-Fenced', 'Fenced']):
        make_stats_section(f, data.loc[fences['Number'] == ii], avg.loc[fences['Number'] == ii], sect, intervals)

    # Compare Fort Macon to non-fenced to fenced
    mann_whitney_test_by_location(f, data, fences, bonferroni=3)

    # Close the file
    f.close()

def make_stats_section(f, data, avg, section, intervals):
    """
    Write a section in the stats file. This function is called by
    make_stats()

    Called by:
    - make_stats()
    """

    # Write a section header
    header_len = 35
    section_part = section + ' (n = ' + str(len(data)) + ')'
    lead_in = '#' * 2
    end_out = '#' * (header_len - len(section_part) - 4)
    f.write('%s\n' % ('#' * header_len))
    f.write('%s %s %s\n' % (lead_in, section_part, end_out))
    f.write('%s\n\n' % ('#' * header_len))

    # Write summary statistics for the data
    f.write('Summary Statistics (m/yr):\r\n')
    for yrs in intervals:
        f.write('%s' % (summary_statistics_alongshore(data, avg, yrs)))
    f.write('\r\n\r\n')

    # Perform an unequal variances T-test to compare the pre- and post-fencing rates to
    # each other. It is not important if the 1997-2016 is similar or dissimilar to either
    ks_test(f, data)

    f.write('\r\n\r\n')

def mann_whitney_test_by_location(f, data, fences, bonferroni=None):
    """
    Perform an unequal variances T-test and print the
    results to the stats file. This version compares
    Fort Macon v Non-Fenced v Fenced

    f: open text file with stats
    data: data to work on
    fences: dataframe with fence locations

    Called by:
    - make_stats()
    """

    bonferroni_alpha = 0.05 / 3

    # Set a list of intervals
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    f.write('\r\n%s\r\n' % ('#' * 50))
    f.write('%s %s %s\r\n' % ('#' * 13, 'Compare locations', '#' * 18))
    f.write('%s\r\n\r\n' % ('#' * 50))

    # Write H-Test header
    f.write('Mann-Whitney U-Test with Bonferroni Correction (alpha = 0.05, bonf alpha = %f)\r\n' % bonferroni_alpha)
    f.write('H0: The sample populations have the same distribution\r\n')
    f.write('H1: The samples populations do not have the same distribution\r\n\r\n')

    for yrs in intervals:

        # Write a header for the years
        header_len = 25
        section_part = yrs
        lead_in = '#' * 2
        end_out = '#' * (header_len - len(section_part) - 4)
        f.write('%s\n' % ('#' * header_len))
        f.write('%s %s %s\n' % (lead_in, section_part, end_out))
        f.write('%s\n\n' % ('#' * header_len))

        # Perform tests
        u1, p1 = stats.mannwhitneyu(x=data[yrs].loc[fences['Number'] == 0], y=data[yrs].loc[fences['Number'] == 1],
                                    alternative='two-sided')
        u2, p2 = stats.mannwhitneyu(x=data[yrs].loc[fences['Number'] == 0], y=data[yrs].loc[fences['Number'] == 2],
                                    alternative='two-sided')
        u3, p3 = stats.mannwhitneyu(x=data[yrs].loc[fences['Number'] == 1], y=data[yrs].loc[fences['Number'] == 2],
                                    alternative='two-sided')

        # Print results
        if p1 < bonferroni_alpha:
            f.write('Fort Macon v. Non-Fenced:\tReject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u1, p1))
        else:
            f.write('Fort Macon v. Non-Fenced:\tFail to reject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u1, p1))
        if p2 < bonferroni_alpha:
            f.write('Fort Macon v. Fenced\t:\tReject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u2, p2))
        else:
            f.write('Fort Macon v. Fenced\t:\tFail to reject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u2, p2))
        if p3 < bonferroni_alpha:
            f.write('Non-Fenced v. Fenced\t:\tReject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n\r\n' % (u3, p3))
        else:
            f.write('Non-Fenced v. Fenced\t:\tFail to reject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n\r\n' % (u3, p3))

def ks_test(f, data):
    """
    Perform a Kolmogorov-Smirnov test and print
    the results to the stats file. This version only compares
    years within one area (Fort Macon, non-fenced, fenced)

    f: open text file with stats
    data: data to compare

    Called by:
    - make_stats_section()
    """

    # Write KS-Test header
    f.write('Kolmogorov-Smirnov Test (alpha = 0.05)\r\n')
    f.write('H0: The sample populations come from the same distribution\r\n')
    f.write('H1: The samples populations do not come from the same distribution\r\n\r\n')

    d_stat, p_stat = stats.mannwhitneyu(data['1997-2010'], data['2010-2016'])
    d_round, p_round = str(np.around(d_stat, decimals=4)), str(np.around(p_stat, decimals=4))
    if p_stat <= 0.05:
        f.write('Reject the null hypothesis (D = %s\tp = %s)\r\n' % (d_round, p_round))
    else:
        f.write('Fail to reject the null hypothesis (D = %s\tp = %s)\r\n' % (d_round, p_round))

def summary_statistics_alongshore(data, avg, yrs):
    """
    Calculate summary statistics and format in a string

    - data: dataframe being worked on
    - avg: dataframe with uncertainties
    - yrs: years being looked at

    Called by:
    - make_stats_section()
    """

    # Set the number of decimals to round to
    dec = 1

    # Calculate the mean
    average = np.around(np.nanmean(avg[yrs + ' Change']), decimals=dec)

    # Calculate 2*sd
    sd_2 = np.around(np.std(avg[yrs + ' Change']), decimals=dec)

    # Calculate the median
    med = np.around(np.median(avg[yrs + ' Change']), decimals=dec)

    # Calculate the mode
    most = np.around(stats.mode(avg[yrs + ' Change'])[0][0], decimals=dec)

    # Calculate the mean uncertainty
    unc = np.around(np.nanmean(avg[yrs + ' Mean Residual Error']), decimals=2)
    min_unc = np.around(np.nanmin(avg[yrs + ' Mean Residual Error']), decimals=2)
    max_unc = np.around(np.nanmax(avg[yrs + ' Mean Residual Error']), decimals=2)
    avg_unc = np.around(np.nanmean(avg[yrs + ' Uncertainty']), decimals=2)

    # Calculate the percent increasing and decreasing
    pos = np.around((len(data[yrs][data[yrs] > 0]) / len(data[yrs])) * 100, decimals=dec)
    neg = np.around((len(data[yrs][data[yrs] < 0]) / len(data[yrs])) * 100, decimals=dec)

    # Calculate the 95% confidence interval
    x_bar = np.nanmean(data[yrs])  # Mean value
    s = np.nanstd(data[yrs])  # Standard deviation
    n = len(data[yrs])  # Sample size
    lo_conf = x_bar - (1.96 * (s / np.sqrt(n)))  # Lower bound of confidence interval
    hi_conf = x_bar + (1.96 * (s / np.sqrt(n)))  # Upper bound of confidence interval
    conf_int = np.around(hi_conf - lo_conf, decimals=dec)

    # Calculate the standard error on the mean
    # autocorr = estimated_autocorrelation(data[yrs])
    # sig_vals = np.where(autocorr <= 0.05)
    # n_star = np.nanmin(sig_vals)
    # sem = np.around(sd_2 / np.sqrt(n_star), decimals=dec)
    sem = np.around(stats.sem(data[yrs], nan_policy='omit'), decimals=dec)

    # Create a nicely formatted string
    res = f'{yrs}:\tMean: {average}\tSD: {sd_2}\tMedian: {med}\tMode: {most}\tMean Uncertainty: {avg_unc}\tMean MRE: {unc} ({min_unc} - {max_unc})\t95 Pct: {conf_int}\tPrograding Pct: {pos}\tEroding Pct: {neg}\tSEM: {sem}\r\n'

    return res

"""
Main function
"""

def main():
    """
    Main program function
    """

    # Load the data
    data_1997 = load_data(year=1997, variable='Beach Width')
    data_2010 = load_data(year=2010, variable='Beach Width')
    data_2016 = load_data(year=2016, variable='Beach Width')

    # Make a dataframe of Dhighs
    widths = change_df(d1997=data_1997, d2010=data_2010, d2016=data_2016)
    widths_errors = beach_width_change_df(widths=widths, rate=False)

    # Load a dataframe with fence locations identified
    fences = load_fence_locations(y=-7.5)

    # Make the figure
    y_min = -75
    y_text = y_min - (y_min * 0.35) + 3
    yticks = [y_min, -50, -25, 0, 25, 50, -y_min]
    make_overall_change_plot_w_kde(data=widths,
                                   fences=fences,
                                   param='Beach Width',
                                   y_min=y_min,
                                   y_max=y_min * -1,
                                   y_ticks=yticks,
                                   y_tick_labels=[str(ii) for ii in yticks],
                                   text_y=40 * 0.80,
                                   bar_y=60,
                                   y_text=y_text,
                                   spacing=1.5,
                                   rate=False,
                                   color='black')

    # Calculate statistics
    make_stats(widths, widths_errors, fences, 'Beach Width')

if __name__ == '__main__':
    main()


import os
import sys
import yaml
import timeit
import shutil
from datetime import datetime

from crotal import utils
from crotal import logger
from crotal import settings
from crotal.utils import remove_dir
from crotal.controller import Controller
from crotal.plugins.pinyin.pinyin import PinYin

LOGO = \
"""  ____ ____   ___ _____  _    _
 / ___|  _ \\ / _ \\_   _|/ \\  | |
| |   | |_) | | | || | / _ \\ | |
| |___|  _ <| |_| || |/ ___ \\| |___
 \____|_| \_\\\\___/ |_/_/   \\_\\_____|
"""

POST_SAMPLE =\
"""---
title: "{0}"
date: {1}
categories:
slug: {2}
---

"""

PAGE_SAMPLE = \
"""---
layout: page.html
title: "{0}"
date: {1}
url: {2}
description: {3}
---

## A demo page!

This is a demo page.
"""

class Command(object):

    @staticmethod
    def generate(full=False, silent=False, is_preview=True):
        if full and os.path.exists(settings.PUBLISH_DIR):
            shutil.rmtree(settings.PUBLISH_DIR)
        utils.load_config_file()
        setattr(settings, 'THEME_DIR', os.path.join('themes', settings.theme))
        setattr(settings, 'TEMPLATES_DIR', os.path.join(settings.THEME_DIR, 'public'))
        setattr(settings, 'THEME_STATIC_DIR', os.path.join(settings.THEME_DIR, 'static'))
        start = timeit.default_timer()

        controller = Controller(full=full)
        copydir_time = timeit.default_timer()
        if not silent:
            logger.info('Static files copied.')

        controller.load(is_preview)
        if not silent:
            logger.info('Source loaded.')

        get_posts_time = timeit.default_timer()

        controller.save()
        save_other_files_time = timeit.default_timer()

        controller.save_posts()
        if not silent:
            logger.info('Posts saved.')
        controller.save_pages()
        if not silent:
            logger.info('Pages saved.')
        controller.save_categories()
        controller.save_tags()

        save_posts_time = timeit.default_timer()
        controller.save_db()
        remove_dir(settings.PUBLISH_DIR)

        if not silent:
            logger.info('%d posts generated.' % len(controller.post_collector.posts))
            logger.info('%d pages generated.' % len(controller.page_collector.pages))
            logger.info('Site generated, master %s!' % settings.author)

    @staticmethod
    def init():
        if len(sys.argv) != 3:
            site_name = 'crotal'
        else:
            site_name = sys.argv[2]
        curr = os.path.dirname(os.path.abspath(__file__))
        dir = os.path.join(curr, "init")
        utils.copy_dir(dir, site_name)
        logger.blue_text(message=LOGO)
        utils.init_git_repo(site_name)
        logger.success('Site created.')

    @staticmethod
    def create_post():
        if not os.path.exists(settings.CONFIG_PATH):
            logger.error('No "_config.yml" file found for the current directory.')
            sys.exit()
        if len(sys.argv) != 3:
            logger.error('Please specify the post title.')
        else:
            title = sys.argv[2]
            now = datetime.now()
            pub_time = now.strftime('%Y-%m-%d %H:%M')
            pinyin = PinYin()
            pinyin.load_word()
            slug = pinyin.hanzi2pinyin_split(string=title, split='-')
            new_post = POST_SAMPLE.format(title, pub_time, slug)
            file_title = now.strftime("%Y-%m-%d") + '-' + slug + '.markdown'
            file_path = os.path.join(settings.POSTS_DIR, file_title)
            open(os.path.join(settings.BASE_DIR, file_path), 'w+').write(new_post)
            logger.success(' '.join([file_path, 'created.']))

    @staticmethod
    def create_page():
        if not os.path.exists(settings.CONFIG_PATH):
            logger.error('No "_config.yml" file found for the current directory.')
            sys.exit()
        title = logger.info_input('Page Title')
        url = logger.info_input('Page URL (.e.g, /foo/bar/):')
        description = logger.info_input('Page Description:')
        pinyin = PinYin()
        slug = pinyin.hanzi2pinyin_split(string=title, split="-")
        now = datetime.now()
        pub_date = now.strftime("%Y-%m-%d %H:%M")
        page = PAGE_SAMPLE.format(title, pub_date, url, description)
        file_path = os.path.join(settings.PAGES_DIR, slug + '.markdown')
        open(os.path.join(settings.BASE_DIR, file_path), 'w+').write(page)
        logger.success('You can browse the page by {0} After generating the site.'.format(url))


import typing

from octobot.database import Database
import telegram
import json

from settings import Settings

def create_db_entry_name(chat: typing.Union[telegram.Chat, int]):
    if isinstance(chat, telegram.Chat):
        chat = chat.id
    return f"admcache:{chat}"

def reset_cache(chat: typing.Union[telegram.Chat, int]):
    return Database.redis.delete(create_db_entry_name(chat))

def check_perms(chat: typing.Union[telegram.Chat, int], user: typing.Union[telegram.User, int],
                permissions_to_check: set, bot=None):
    db_entry = create_db_entry_name(chat)

    if isinstance(user, telegram.User):
        user_id = user.id
    else:
        user_id = user
    if isinstance(chat, telegram.Chat):
        chat_id = chat.id
    else:
        chat_id = chat
    if "is_bot_owner" in permissions_to_check:
        if Settings.owner == user_id: permissions_to_check.remove("is_bot_owner")
        return Settings.owner == user_id, permissions_to_check
    if not str(chat_id).startswith("-100"):
        return True, []
    if Database.redis is not None and Database.redis.exists(db_entry) == 1:
        adm_list = json.loads(Database.redis.get(db_entry).decode())
    else:
        if bot is None:
            adm_list_t = chat.get_administrators()
        else:
            adm_list_t = bot.get_chat_administrators(chat_id=chat_id)
        adm_list = []
        for admin in adm_list_t:
            adm_list.append(admin.to_dict())
        if Database.redis is not None:
            Database.redis.set(db_entry, json.dumps(adm_list))
            Database.redis.expire(db_entry, 240)
    for member in adm_list:
        if int(member["user"]["id"]) == int(user_id):
            if member["status"] == "creator":
                return True, []
            if "is_admin" in permissions_to_check:
                permissions_to_check.remove("is_admin")
            for user_permission in permissions_to_check.copy():
                if member[user_permission]:
                    permissions_to_check.remove(user_permission)
            break
    return len(permissions_to_check) == 0, permissions_to_check

def permissions(*perms_args, **perms_kwargs):
    """
    Decorator to check permissions.

    :param perms: Valid permissions that can be found on :class:`telegram.ChatMember` + is_admin. Example: `@permissions(can_delete_messages=True)`. The value does not matter.
    """
    perms = set(perms_kwargs.keys())
    perms.update(perms_args)

    def decorator(function):
        def wrapper(bot, context):
            if context.chat is not None:
                res, missing_perms = check_perms(context.chat, context.user, perms.copy())
                if res:
                    function(bot, context)
                else:
                    context.reply(context.localize(
                        "Sorry, you can't execute this command cause you lack following permissions: {}").format(
                        ', '.join(missing_perms)))

        return wrapper

    return decorator

def not_admin(function):
    """
    Decorator to check if sender is not an admin.
    """
    perms = {"is_admin"}

    def wrapper(bot, context):
        if context.chat is not None:
            res, missing_perms = check_perms(context.chat, context.user, perms.copy())
            if not res:
                function(bot, context)

    return wrapper

def my_permissions(*perms_args, **perms_kwargs):
    """
    Decorator to check bots own permissions.

    :param perms: Valid permissions that can be found on :class:`telegram.ChatMember` + is_admin. Example: `@my_permissions(can_delete_messages=True)`. The value does not matter.
    """
    perms = set(perms_kwargs.keys())
    perms.update(perms_args)
    if "is_admin" not in perms:
        perms.add("is_admin")

    def decorator(function):
        def wrapper(bot, context):
            if context.chat is not None:
                res, missing_perms = check_perms(context.chat, bot.me, perms.copy())
                if res:
                    function(bot, context)
                elif "is_admin" not in missing_perms:
                    context.reply(context.localize(
                        "Sorry, you can't execute this command cause I lack following permissions: {}").format(
                        ', '.join(missing_perms)))

        return wrapper

    return decorator


from pytest_mock import MockerFixture
from click.testing import Result, CliRunner

from kakaowork.consts import Limit
from kakaowork.models import (
    ErrorCode,
    BotStatus,
    ColorTone,
    ProfileNameFormat,
    ProfilePositionFormat,
    ErrorField,
    DepartmentField,
    SpaceField,
    BotField,
    DepartmentListResponse,
    SpaceResponse,
    BotResponse,
)
from kakaowork.command import (
    _command_aliases,
    cli,
    departments,
    spaces,
    bots,
)

def test__command_aliases():
    assert _command_aliases() == {
        'user': 'users',
        'conversation': 'conversations',
        'conv': 'conversations',
        'message': 'messages',
        'msg': 'messages',
        'department': 'departments',
        'dept': 'departments',
        'space': 'spaces',
        'bot': 'bots',
    }

class TestDepartmentsCommand:
    def test_departments_list_success(self, mocker: MockerFixture, cli_runner: CliRunner):
        resp = DepartmentListResponse(
            success=True,
            departments=[
                DepartmentField(
                    id='1',
                    ids_path='1',
                    parent_id='',
                    space_id='1',
                    name='name',
                    code='code',
                    user_count=1,
                    has_child=False,
                    depth=0,
                    users_ids=[1],
                    leader_ids=[1],
                    ancestry='anc',
                )
            ],
        )
        mocker.patch('kakaowork.client.Kakaowork.Departments.list', return_value=resp)

        expected_output = 'ID:\t\t1\nName:\t\tname\nCode:\t\tcode\nUser count:\t1\n'

        res = cli_runner.invoke(departments, ['list'])
        assert res.exit_code == 0
        assert res.output == expected_output

        res = cli_runner.invoke(cli, ['--app-key', 'dummy', 'department', 'list'])
        assert res.exit_code == 0
        assert res.output == expected_output

        res = cli_runner.invoke(cli, ['--app-key', 'dummy', 'dept', 'list'])
        assert res.exit_code == 0
        assert res.output == expected_output

    def test_departments_list_error(self, mocker: MockerFixture, cli_runner: CliRunner):
        resp = DepartmentListResponse(
            success=False,
            error=ErrorField(code=ErrorCode.UNAUTHORIZED, message='message'),
        )
        mocker.patch('kakaowork.client.Kakaowork.Departments.list', return_value=resp)

        res = cli_runner.invoke(departments, ['list'])
        assert res.exit_code == 1
        assert res.output == 'Error code:\tunauthorized\nMessage:\tmessage\n'

        res = cli_runner.invoke(departments, ['list', '--limit', 0])
        assert res.exit_code == 2
        assert "Error: Invalid value for '-l' / '--limit': 0 is not in the range" in res.output

        res = cli_runner.invoke(departments, ['list', '--limit', 101])
        assert res.exit_code == 2
        assert "Error: Invalid value for '-l' / '--limit': 101 is not in the range" in res.output

class TestSpacesCommand:
    def test_spaces_info_success(self, mocker: MockerFixture, cli_runner: CliRunner):
        resp = SpaceResponse(
            success=True,
            space=SpaceField(
                id=1,
                kakaoi_org_id=1,
                name='name',
                color_code='default',
                color_tone=ColorTone.LIGHT,
                permitted_ext=['*'],
                profile_name_format=ProfileNameFormat.NAME_NICKNAME,
                profile_position_format=ProfilePositionFormat.POSITION,
                logo_url='http://localhost/image.png',
            ),
        )
        mocker.patch('kakaowork.client.Kakaowork.Spaces.info', return_value=resp)

        expected_output = ("ID:\t1\nOrgID:\t1\nName:\tname\nColor code:\tdefault\nColor tone:\tlight\nPermitted ext:\t['*']\n"
                           "Profile name format:\tname_nickname\nProfile position format:\tposition\nLogo URL:\thttp://localhost/image.png\n")

        res = cli_runner.invoke(spaces, ['info'])
        assert res.exit_code == 0
        assert res.output == expected_output

        res = cli_runner.invoke(cli, ['--app-key', 'dummy', 'space', 'info'])
        assert res.exit_code == 0
        assert res.output == expected_output

    def test_spaces_info_error(self, mocker: MockerFixture, cli_runner: CliRunner):
        resp = SpaceResponse(
            success=False,
            error=ErrorField(code=ErrorCode.UNAUTHORIZED, message='message'),
        )
        mocker.patch('kakaowork.client.Kakaowork.Spaces.info', return_value=resp)

        res = cli_runner.invoke(spaces, ['info'])
        assert res.exit_code == 1
        assert res.output == 'Error code:\tunauthorized\nMessage:\tmessage\n'

class TestBotsCommand:
    def test_bots_info_success(self, mocker: MockerFixture, cli_runner: CliRunner):
        resp = BotResponse(
            success=True,
            info=BotField(bot_id=1, title='name', status=BotStatus.ACTIVATED),
        )
        mocker.patch('kakaowork.client.Kakaowork.Bots.info', return_value=resp)

        expected_output = 'ID:\t1\nName:\tname\nStatus:\tactivated\n'

        res = cli_runner.invoke(bots, ['info'])
        assert res.exit_code == 0
        assert res.output == expected_output

        res = cli_runner.invoke(cli, ['--app-key', 'dummy', 'bot', 'info'])
        assert res.exit_code == 0
        assert res.output == expected_output

    def test_bots_info_error(self, mocker: MockerFixture, cli_runner: CliRunner):
        response = BotResponse(
            success=False,
            error=ErrorField(code=ErrorCode.UNAUTHORIZED, message='message'),
        )
        mocker.patch('kakaowork.client.Kakaowork.Bots.info', return_value=response)

        res = cli_runner.invoke(bots, ['info'])
        assert res.exit_code == 1
        assert res.output == 'Error code:\tunauthorized\nMessage:\tmessage\n'


from keras.layers import Flatten, Dense, Embedding
from keras import Sequential

from keras.datasets import imdb
from keras import preprocessing

max_features = 1000
max_len = 20

(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)

# turn list of integers to 2D tensor of shape (samples, max_len)
x_train = preprocessing.sequence.pad_sequences(x_train, maxlen=max_len)
x_test = preprocessing.sequence.pad_sequences(x_test, maxlen=max_len)

# creating the network
"""
The network will learn 8-dimensional embeddings for each of the 10,000 words, turn the input integer sequences
(2D integer tensor) into embedded sequences (3D float tensor), flatten the tensor to 2D, and train a single Dense
layer on top for classification.

"""

model = Sequential()

model.add(Embedding(10000, 8, input_length=max_len))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))

model.compile(optimizer='rmsprop',
              loss='binary_crossentropy',
              metrics=['acc'])

history = model.fit(x_train, y_train, epochs=10, batch_size=32, validation_split=0.2)


from .forms import UploadFileForm
from job_app.models import Job, IsOpen, JobTitle, Status, City, State, ZipCode, County
from nurse_manage.models import AvailableTime, Skill
from uploader.models import Document
from nurse_manage.models import Nurse, Discipline, AvailableTime
from django.shortcuts import render
import csv, openpyxl
from  nurse_manage.helper.available_time_generator import AvailableTimeGenerator

# Create your views here.
def upload_job(request):
    # handle file upload
    if request.method == 'POST':
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            file = Document(file=request.FILES['file'])
            file.save()
            ifile = open(file.file.path, "r", encoding='utf-8', errors='ignore')
            readers = csv.reader(ifile)
            readers.__next__()  # skip the first line (headers)

            error = []
            success = []

            for row in readers:
                title = JobTitle.objects.get_or_create(name=row[0])
                status = Status.objects.get_or_create(name=row[1])
                city = City.objects.get_or_create(name=row[2])
                state = State.objects.get_or_create(name=row[3])
                zip_code = ZipCode.objects.get_or_create(code=row[4])
                is_open = IsOpen.objects.get_or_create(open=str2bool(row[5]))

                try:
                    job = Job(title=title[0], status=status[0], city=city[0], state=state[0],
                              zip_code=zip_code[0], is_open=is_open[0], description=row[6])
                    job.save()
                    success.append(title[0])
                except Exception as e:
                    error.append(title[0])

            context = {'errors': error, 'success': success}
            return render(request, 'django_excel/result.html', context)

    form = UploadFileForm()  # an empty unbound form
    return render(request, 'django_excel/upload.html', {'form': form})

def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")

def upload_city(request):
    if request.method == 'POST':
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            file = Document(file=request.FILES['file'])
            file.save()
            ifile = open(file.file.path, "r", encoding='utf-8')
            readers = csv.reader(ifile)

            county = request.POST.getlist('county')[0]
            error = []
            success = []
            counties = []
            county_level = False

            for row in readers:
                city_name = row[0]
                try:
                    city_name = city_name.strip()
                    county, created = County.objects.get_or_create(name=county)
                    city = City.objects.create(name=city_name, county=county)
                    city.save()
                    success.append(city_name)

                except Exception as e:
                    print(e)
                    error.append(city_name)

            context = {'errors': error, 'success': success}
            return render(request, 'django_excel/result.html', context)

    form = UploadFileForm()  # an empty unbound form
    return render(request, 'django_excel/upload_city.html', {'form': form})

def upload_time(request):
    if request.method == 'POST':
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            file = Document(file=request.FILES['file'])
            file.save()
            ifile = open(file.file.path, "r", encoding='utf-8')
            readers = csv.reader(ifile)

            readers.__next__()  # skip the first line (headers)

            error = []
            success = []

            for row in readers:
                code = row[2]

                try:
                    available_time = AvailableTime(code=code)
                    available_time.save()
                    success.append(str(code))

                except Exception as e:
                    print(e)
                    error.append(str(code))

            context = {'errors': error, 'success': success}
            return render(request, 'django_excel/result.html', context)

    form = UploadFileForm()  # an empty unbound form
    return render(request, 'django_excel/upload_time.html', {'form': form})

def upload_nurse(request):
    values = ['name', 'title', 'dateOfHire', 'status', 'skill', 'area', 'cellphone', 'work_phone', 'email', 'comment']

    if request.method == 'POST':
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            file = Document(file=request.FILES['file'])
            file.save()

            wb = openpyxl.load_workbook(file.file.path)
            sheet = wb.get_sheet_by_name("Staff List")

            error = []
            success = []
            a_time_gen = AvailableTimeGenerator()
            try:
                for r in range(1, 60):

                    for c, v in zip(range(1, 9), range(0, len(values))):
                        values[v] = sheet.cell(row=r, column=c).value

                    name = values[0]

                    while name != None and name != "Employee Name":
                        name = values[0]
                        discipline = values[1]

                        status = values[3]

                        skill = values[4]

                        covered_areas = values[5]

                        cell_phone = values[6]
                        email = values[8]
                        comment = values[9]
                        available_time = a_time_gen.generate_time_list()
                        try:
                            # create and populate nurse instance:
                            nurse, created = Nurse.objects.get_or_create(name=name, cell_phone=cell_phone)
                            print(nurse, created)
                            if discipline != None:
                                list = discipline.split(",")
                                for e in list:
                                    if e != None:
                                        e = e.strip()
                                        discipline, created = Discipline.objects.get_or_create(name=e)
                                        print(discipline)
                                        nurse.disciplines.add(discipline)

                            if skill != None:
                                list = skill.split(",")

                                for e in list:
                                    if e != None:
                                        e = e.strip()
                                        skill, created = Skill.objects.get_or_create(name=e)
                                        nurse.skills.add(skill)

                            # populate fake available time

                            print("=========================")
                            print(available_time)
                            for e in available_time:
                                a_time = AvailableTime.objects.get(code=e)
                                print(a_time)
                                nurse.availableTime.add(a_time)
                                print("Success")

                            if covered_areas != None:
                                list = covered_areas.split(',')

                                for e in list:
                                    if e != None:
                                        e = e.strip()
                                        area, created = City.objects.get_or_create(name=e)
                                        print(area)
                                        nurse.covered_areas.add(area)

                        except Exception as e:
                            error.append(nurse.name)

                        success.append(nurse.name)
                        break
            except Exception as e:
                print(e)

            context = {'errors': error, 'success': success}
            return render(request, 'django_excel/result.html', context)

    form = UploadFileForm()  # an empty unbound form
    return render(request, 'django_excel/upload_nurse.html', {'form': form})