#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np

# In[2]:

from tensorflow.keras.datasets import mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()

# ##  Visualizing the Image Data

# In[3]:

import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')

# In[4]:

x_train.shape

# In[5]:

single_image = x_train[0]

# In[6]:

single_image

# In[7]:

single_image.shape

# In[8]:

plt.imshow(single_image)

# # PreProcessing Data

# ## Labels

# In[9]:

y_train

# In[10]:

y_test

# In[11]:

from tensorflow.keras.utils import to_categorical

# In[12]:

y_train.shape

# In[13]:

y_example = to_categorical(y_train)

# In[14]:

y_example

# In[15]:

y_example.shape

# In[16]:

y_example[0]

# In[17]:

y_cat_test = to_categorical(y_test,10)

# In[18]:

y_cat_train = to_categorical(y_train,10)

# In[19]:

y_cat_test.shape

# ### Processing X Data
#
# We should normalize the X data

# In[21]:

single_image.max()

# In[22]:

single_image.min()

# In[23]:

x_train = x_train/255
x_test = x_test/255

# In[24]:

scaled_single = x_train[0]

# In[25]:

scaled_single.max()

# In[26]:

plt.imshow(scaled_single)

# ## Reshaping the Data

# In[27]:

x_train.shape

# In[28]:

x_test.shape

# In[29]:

x_train = x_train.reshape(60000, 28, 28, 1)

# In[30]:

x_train.shape

# In[31]:

x_test = x_test.reshape(10000,28,28,1)

# In[32]:

x_test.shape

# # Training the Model

# In[33]:

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten

# In[34]:

model = Sequential()

# CONVOLUTIONAL LAYER
model.add(Conv2D(filters=32, kernel_size=(4,4),input_shape=(28, 28, 1), activation='relu',))
# POOLING LAYER
model.add(MaxPool2D(pool_size=(2, 2)))

# FLATTEN IMAGES FROM 28 by 28 to 764 BEFORE FINAL LAYER
model.add(Flatten())

# 128 NEURONS IN DENSE HIDDEN LAYER (YOU CAN CHANGE THIS NUMBER OF NEURONS)
model.add(Dense(128, activation='relu'))

# LAST LAYER IS THE CLASSIFIER, THUS 10 POSSIBLE CLASSES
model.add(Dense(10, activation='softmax'))

# https://keras.io/metrics/
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy']) # we can add in additional metrics https://keras.io/metrics/

# In[35]:

model.summary()

# In[36]:

from tensorflow.keras.callbacks import EarlyStopping

# In[37]:

early_stop = EarlyStopping(monitor='val_loss',patience=2)

# ## Train the Model

# In[38]:

model.fit(x_train,y_cat_train,epochs=10,validation_data=(x_test,y_cat_test),callbacks=[early_stop])

# ## Evaluate the Model

# In[42]:

model.metrics_names

# In[43]:

losses = pd.DataFrame(model.history.history)

# In[44]:

losses.head()

# In[45]:

losses[['accuracy','val_accuracy']].plot()

# In[46]:

losses[['loss','val_loss']].plot()

# In[50]:

print(model.metrics_names)
print(model.evaluate(x_test,y_cat_test,verbose=0))

# In[52]:

from sklearn.metrics import classification_report,confusion_matrix

# In[53]:

predictions = model.predict_classes(x_test)

# In[54]:

y_cat_test.shape

# In[55]:

y_cat_test[0]

# In[56]:

predictions[0]

# In[57]:

y_test

# In[59]:

print(classification_report(y_test,predictions))

# In[60]:

confusion_matrix(y_test,predictions)


from matplotlib import pyplot as plt
from torch.utils.data import Dataset, DataLoader
import torch
import os
import numpy as np

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

class EVRPDataset(Dataset):
    def __init__(self, train_size, num_nodes, t_limit, capacity, num_afs=3, seed=520):
        super().__init__()
        self.size = train_size

        torch.manual_seed(seed)
        afs = torch.rand(2,num_afs+1)
        customers = torch.rand(train_size, 2, num_nodes, device=device)

        self.static = torch.cat((afs.unsqueeze(0).repeat(train_size, 1, 1), customers), dim=2).to(device)  # (train_size, 2, num_nodes+4)

        self.dynamic = torch.ones(train_size, 3, 1+num_afs+num_nodes, device=device)   # time duration, capacity, demands
        self.dynamic[:, 0, :] *= t_limit
        self.dynamic[:, 1, :] *= capacity
        self.dynamic[:, 2, :num_afs+1] = 0

        seq_len = self.static.size(2)
        self.distances = torch.zeros(train_size, seq_len, seq_len, device=device)
        for i in range(seq_len):
            self.distances[:, i] = cal_dis(self.static[:, 0, :], self.static[:, 1, :], self.static[:, 0, i:i+1], self.static[:, 1, i:i+1])

    def __len__(self):
        return self.size

    def __getitem__(self, idx):
        return self.static[idx], self.dynamic[idx], self.distances[idx]    # dynamic: None

    def update_mask(self, what):
        pass

    def update_dynamic(self, old_idx, idx, mask, dynamic, distances, dis_by_afs, capacity, velocity, cons_rate, t_limit, num_afs):
        """
        :param old_idx: (batch*beam, 1)
        :param idx: ditto
        :param mask: (batch*beam, seq_len)
        :param dynamic: (batch*beam, dynamic_features, seq_len)
        :param distances: (batch*beam, seq_len, seq_len)
        :param dis_by_afs: (batch*beam, seq_len)
        :param capacity, velocity, cons_rate, t_limit, num_afs: scalar
        :return: updated dynamic AFS = merh pou kanoume recharge
        """

        #TODO: kane th maska apo 1:paw, 0 den paw, se 0 paw, -inf den paw
        # mask[mask == 1] = 0# mporeis na pas
        # mask[mask == 0] =  float('-inf')
        dis = distances[torch.arange(distances.size(0)), old_idx.squeeze(1), idx.squeeze(1)].unsqueeze(1)
        depot = idx.eq(0).squeeze(1)
        afs = (idx.gt(0) & idx.le(num_afs)).squeeze(1)
        fs = idx.le(num_afs).squeeze(1)
        customer = idx.gt(num_afs).squeeze(1)  # TODO: introduce num_afs
        time = dynamic[:, 0, :].clone()
        fuel = dynamic[:, 1, :].clone()
        demands = dynamic[:, 2, :].clone()

        time -= dis / velocity
        time[depot] = t_limit #exeis full diathesimh wra
        time[afs] -= 0.25 # katanalwneis ligh wra gia na forthseis
        time[customer] -= 0.5  # katanalwneis ligh wra gia na afhseis to proion

        fuel -= cons_rate * dis
        fuel[fs] = capacity
        demands.scatter_(1, idx, 0)

        dynamic = torch.cat((time.unsqueeze(1), fuel.unsqueeze(1), demands.unsqueeze(1)), dim=1).to(device)
        #print(f'Here are some demands left{((dynamic[:, 2, :] == 0)==False).nonzero()}')
        mask.scatter_(1, idx, float('-inf'))
        # forbid passing by afs if leaving depot, allow if returning to depot; forbid from afs to afs, not necessary but convenient
        mask[fs, 1:num_afs + 1] = float('-inf')
        mask[afs, 0] = 0
        # kapou edw
        mask[customer, :num_afs + 1] = 0  # phgaine
        mask[fs] = torch.where(demands[fs] > 0, torch.zeros(mask[fs].size(), device=device), mask[fs])
        #print(f'non inf mask {torch.nonzero(mask != float('-inf')).squeeze()}')
        # path1: ->Node->Depot
        dis1 = distances[torch.arange(distances.size(0)), idx.squeeze(1)].clone()
        fuel_pd0 = cons_rate * dis1 #poso fuel xreiazetai na paei apo node sto depot
        time_pd0 = dis1 / velocity # poso time xreiazetai
        dis1[:, num_afs + 1:] += distances[:, 0, num_afs + 1:]
        fuel_pd1 = cons_rate * dis1
        time_pd1 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + distances[:, 0, :]) / velocity
        time_pd1[:, 1:num_afs + 1] += 0.25
        time_pd1[:, num_afs + 1:] += 0.5

        # path2: ->Node-> Station-> Depot(choose the station making the total distance shortest)
        dis2 = distances[:, 1:num_afs + 1, :].gather(1, dis_by_afs[1].unsqueeze(1)).squeeze(1)
        dis2[:, 0] = 0
        dis2 += distances[torch.arange(distances.size(0)), idx.squeeze(1)]
        fuel_pd2 = cons_rate * dis2
        time_pd2 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + dis_by_afs[0]) / velocity
        time_pd2[:, 1:num_afs + 1] += 0.25
        time_pd2[:, num_afs + 1:] += 0.5

        # path3: ->Node-> Station-> Depot(choose the closest station to the node), ignore this path temporarily
        # the next node should be able to return to depot with at least one way; otherwise, mask it
        mask[~((fuel >= fuel_pd1) & (time >= time_pd1) | (fuel >= fuel_pd2) & (time >= time_pd2))] = float('-inf')
        # edw ginetai h malakia
        mask[(fuel < fuel_pd0) | (time < time_pd0)] = float('-inf')

        all_masked = mask[:, num_afs + 1:].eq(0).sum(1).le(0)
        mask[all_masked, 0] = 0  # unmask the depot if all nodes are masked

        return dynamic

    def update_dynamic2(self, old_idx, idx, mask, dynamic, distances, dis_by_afs, capacity, velocity, cons_rate, t_limit,
                       num_afs):
        """
        1: mporeis na pas
        0: den mporeis na pas
        :param old_idx: (batch*beam, 1)
        :param idx: ditto
        :param mask: (batch*beam, seq_len)
        :param dynamic: (batch*beam, dynamic_features, seq_len)
        :param distances: (batch*beam, seq_len, seq_len)
        :param dis_by_afs: (batch*beam, seq_len)
        :param capacity, velocity, cons_rate, t_limit, num_afs: scalar
        :return: updated dynamic
        """
        dis = distances[torch.arange(distances.size(0)), old_idx.squeeze(1), idx.squeeze(1)].unsqueeze(1)
        episkepthkes_depot = idx.eq(0).squeeze(1)
        afs = (idx.gt(0) & idx.le(num_afs)).squeeze(1) #episkepthkes fulling station
        fs = idx.le(num_afs).squeeze(1) ##episkepthkes fulling station
        customer = idx.gt(num_afs).squeeze(1)  # TODO: introduce num_afs
        time = dynamic[:, 0, :].clone()
        fuel = dynamic[:, 1, :].clone()
        demands = dynamic[:, 2, :].clone()

        time -= dis / velocity
        time[episkepthkes_depot] = t_limit
        time[afs] -= 0.25
        time[customer] -= 0.5

        fuel -= cons_rate * dis # poso kausimo xalases?
        fuel[fs] = capacity # edw episkfthkes fuel station
        #demands.scatter_(1, idx, 1) # mhpws 1?

        print(f'Chose indexes:{idx.squeeze()}')
        demands.scatter_(1, idx, 0) # satisfy the demands of the customer you visit

        # dynamic: [time, fuel, demand]
        dynamic = torch.cat((time.unsqueeze(1), fuel.unsqueeze(1), demands.unsqueeze(1)), dim=1).to(device)

        # mask.scatter_(1, idx, float('-inf'))
        mask.scatter_(1, idx, 0) # MHN XANAPAS STO IDIO

        # forbid passing by afs if leaving depot, allow if returning to depot; forbid from afs to afs, not necessary but convenient
        mask[fs, 1:num_afs + 1] = 0#float('-inf')
        mask[afs, 0] = 1 #0

        mask[customer, :num_afs + 1] = 1 # phgaine ok # 0
        # edw ta kanei mhden enw emeis theloume 1!
        mask[fs] = torch.where(demands[fs] > 0, torch.zeros(mask[fs].size(), device=device), mask[fs])

        # path1: ->Node->Depot
        dis1 = distances[torch.arange(distances.size(0)), idx.squeeze(1)].clone()
        fuel_pd0 = cons_rate * dis1
        time_pd0 = dis1 / velocity
        dis1[:, num_afs + 1:] += distances[:, 0, num_afs + 1:]
        fuel_pd1 = cons_rate * dis1
        time_pd1 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + distances[:, 0, :]) / velocity
        time_pd1[:, 1:num_afs + 1] += 0.25
        time_pd1[:, num_afs + 1:] += 0.5

        # path2: ->Node-> Station-> Depot(choose the station making the total distance shortest)
        dis2 = distances[:, 1:num_afs + 1, :].gather(1, dis_by_afs[1].unsqueeze(1)).squeeze(1)
        dis2[:, 0] = 1 #0
        dis2 += distances[torch.arange(distances.size(0)), idx.squeeze(1)]
        fuel_pd2 = cons_rate * dis2
        time_pd2 = (distances[torch.arange(distances.size(0)), idx.squeeze(1)] + dis_by_afs[0]) / velocity
        time_pd2[:, 1:num_afs + 1] += 0.25
        time_pd2[:, num_afs + 1:] += 0.5

        # path3: ->Node-> Station-> Depot(choose the closest station to the node), ignore this path temporarily
        # the next node should be able to return to depot with at least one way; otherwise, mask it
        mask[~((fuel >= fuel_pd1) & (time >= time_pd1) | (fuel >= fuel_pd2) & (time >= time_pd2))] = 0#float('-inf')

        mask[(fuel < fuel_pd0) | (time < time_pd0)] = 0# float('-inf')

        #all_masked = mask[:, num_afs + 1:].eq(0).sum(1).le(0)
        all_masked = mask[:,num_afs + 1:].eq(0).all(dim=1)
        mask[all_masked, 0] = 1#0  # unmask the depot if all nodes are masked

        mask[mask == 0] = 1
        mask[mask == float('-inf')] = 0
        return dynamic

def cal_dis(lon1, lat1, lon2, lat2):
    """
    Calculate the great circle distance between two points
    on the earth (specified in decimal degrees)
    """
    # degrees to radians
    lon1, lat1, lon2, lat2 = map(lambda x: x/180*np.pi, [lon1, lat1, lon2, lat2])
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = torch.pow(torch.sin(dlat / 2), 2) + torch.cos(lat1) * torch.cos(lat2) * torch.pow(torch.sin(dlon / 2), 2)
    c = 2 * 4182.44949 * torch.asin(torch.sqrt(a))    # miles
    # c = 2 * atan2(sqrt(a), sqrt(1 - a)) * 4182.44949
    return c

def reward_func2(tours, static, distances, beam_width=1):
    """
    :param tours: LongTensor, (batch*beam, seq_len)
    :param static: (batch, 2, num_nodes)
    :param distances: (batch, num_nodes, num_nodes)
    :param beam_width: set beam_width=1 when training
    :return: reward: Euclidean distance between each consecutive point， (batch)
    :return: locs: (batch, 2, seq_len)
    """
    bb_size, seq_len = tours.size()
    batch_size = static.size(0)
    depot = torch.zeros(bb_size, 1, dtype=torch.long, device=device)
    tours = torch.cat((depot, tours, depot), dim=1)         # start from depot, end at depot(although some have ended at depot)
    id0 = torch.arange(bb_size).unsqueeze(1).repeat(1, seq_len+1)
    reward = distances.repeat(beam_width, 1, 1)[id0, tours[:, :-1], tours[:, 1:]].sum(1)    # (batch*beam)
    # (batch*beam) -> (batch), choose the best reward
    reward, id_best = torch.cat(torch.chunk(reward.unsqueeze(1), beam_width, dim=0), dim=1).min(1)  # (batch)
    bb_idx = torch.arange(batch_size, device=device) + id_best * batch_size
    tours = tours[bb_idx]
    # print(tours)
    tours = tours.unsqueeze(1).repeat(1, static.size(1), 1)
    locs = torch.gather(static, dim=2, index=tours)  # (batch, 2, seq_len+)
    return reward, locs

def reward_func( static,tours, distances, beam_width=1):
    """
    :param tours: LongTensor, (batch*beam, seq_len)
    :param static: (batch, 2, num_nodes)
    :param distances: (batch, num_nodes, num_nodes)
    :param beam_width: set beam_width=1 when training
    :return: reward: Euclidean distance between each consecutive point， (batch)
    :return: locs: (batch, 2, seq_len)
    """
    bb_size, seq_len = tours.size()
    batch_size = static.size(0)
    depot = torch.zeros(bb_size, 1, dtype=torch.long, device=device)
    tours = torch.cat((depot, tours, depot), dim=1)         # start from depot, end at depot(although some have ended at depot)
    id0 = torch.arange(bb_size).unsqueeze(1).repeat(1, seq_len+1)
    reward = distances.repeat(beam_width, 1, 1)[id0, tours[:, :-1], tours[:, 1:]].sum(1)    # (batch*beam)

    # (batch*beam) -> (batch), choose the best reward
    reward, id_best = torch.cat(torch.chunk(reward.unsqueeze(1), beam_width, dim=0), dim=1).min(1)  # (batch)
    bb_idx = torch.arange(batch_size, device=device) + id_best * batch_size
    # print(tours)
    tours = tours[bb_idx]
    #print(tours)
    tours = tours.unsqueeze(1).repeat(1, static.size(1), 1)
    locs = torch.gather(static, dim=2, index=tours)  # (batch, 2, seq_len+)
    return reward#, locs

def render_func(locs, static, num_afs, save_dir="."):
    """
    :param locs: (batch, 2, sel_len)
    :param static: (batch, 2, num_nodes+1)
    :param num_afs: scalar
    :param save_dir: path to save figure
    :return: None
    """
    plt.close('all')
    data = locs[-1].cpu().numpy()   # (2, num_nodes+1), just plot the last one
    # demands = dynamic[-1, 1, 1:].cpu().numpy()*capacity   # (num_nodes), depot excluded
    # coords = static[-1][:, 1:].cpu().numpy().T     # (num_nodes, 2)
    plt.plot(data[0], data[1], zorder=1)
    origin_locs = static[-1, :, :].cpu().numpy()
    plt.scatter(origin_locs[0], origin_locs[1], s=4, c='r', zorder=2)
    plt.scatter(origin_locs[0, 0], origin_locs[1, 0], s=20, c='k', marker='*', zorder=3)  # depot
    plt.scatter(origin_locs[0, 1:num_afs+1], origin_locs[1, 1:num_afs+1], s=20, c='b', marker='+', zorder=4)    # afs
    for i, coords in enumerate(origin_locs.T[1:]):
        plt.annotate('%d' % (i+1), xy=coords, xytext=(2, 2), textcoords='offset points')  # mark numbers
    if os.path.isfile(save_dir):
        os.remove(save_dir)
    plt.axis('equal')
    plt.savefig(save_dir, dpi=400)


#  This file is part of Recent changes Goat compatible Discord webhook (RcGcDw).
#
#  RcGcDw is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  RcGcDw is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.

import argparse

parser = argparse.ArgumentParser(description="Start RcGcDw")
parser.add_argument("--test", action='store_true', help="mode used for testing, sends only 5 entries of both rc and discussion changes and sends daily overview")
parser.add_argument("--settings", default="settings.json", type=argparse.FileType(encoding='utf8'), help="provides a path to settings file (default ./settings.json)")
command_args, unknown = parser.parse_known_args()


import argparse
import ast
import re
import sys

_BRACKETS_PATTERN = re.compile(r"\[[^\]]*\]")
_QUOTES_PATTERN = re.compile(r"(\"[^\"]*\"|\'[^\']*\')")
_WHITESPACE_PATTERN = re.compile(r"\s+")

_NUMBER_PATTERN = re.compile(r"[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?")

class Interval(object):
  """Represents an interval between a start and end value."""

  def __init__(self, start, start_included, end, end_included):
    self.start = start
    self.start_included = start_included
    self.end = end
    self.end_included = end_included

  def contains(self, value):
    if value < self.start or value == self.start and not self.start_included:
      return False
    if value > self.end or value == self.end and not self.end_included:
      return False
    return True

  def __eq__(self, other):
    return (self.start == other.start and
            self.start_included == other.start_included and
            self.end == other.end and
            self.end_included == other.end_included)

def parse_command(command):
  """Parse command string into a list of arguments.

  - Disregards whitespace inside double quotes and brackets.
  - Strips paired leading and trailing double quotes in arguments.
  - Splits the command at whitespace.

  Nested double quotes and brackets are not handled.

  Args:
    command: (str) Input command.

  Returns:
    (list of str) List of arguments.
  """

  command = command.strip()
  if not command:
    return []

  brackets_intervals = [f.span() for f in _BRACKETS_PATTERN.finditer(command)]
  quotes_intervals = [f.span() for f in _QUOTES_PATTERN.finditer(command)]
  whitespaces_intervals = [
      f.span() for f in _WHITESPACE_PATTERN.finditer(command)
  ]

  if not whitespaces_intervals:
    return [command]

  arguments = []
  idx0 = 0
  for start, end in whitespaces_intervals + [(len(command), None)]:
    # Skip whitespace stretches enclosed in brackets or double quotes.

    if not any(interval[0] < start < interval[1]
               for interval in brackets_intervals + quotes_intervals):
      argument = command[idx0:start]

      # Strip leading and trailing double quote if they are paired.
      if (argument.startswith("\"") and argument.endswith("\"") or
          argument.startswith("'") and argument.endswith("'")):
        argument = argument[1:-1]
      arguments.append(argument)
      idx0 = end

  return arguments

def extract_output_file_path(args):
  """Extract output file path from command arguments.

  Args:
    args: (list of str) command arguments.

  Returns:
    (list of str) Command arguments with the output file path part stripped.
    (str or None) Output file path (if any).

  Raises:
    SyntaxError: If there is no file path after the last ">" character.
  """

  if args and args[-1].endswith(">"):
    raise SyntaxError("Redirect file path is empty")
  elif args and args[-1].startswith(">"):
    try:
      _parse_interval(args[-1])
      if len(args) > 1 and args[-2].startswith("-"):
        output_file_path = None
      else:
        output_file_path = args[-1][1:]
        args = args[:-1]
    except ValueError:
      output_file_path = args[-1][1:]
      args = args[:-1]
  elif len(args) > 1 and args[-2] == ">":
    output_file_path = args[-1]
    args = args[:-2]
  elif args and args[-1].count(">") == 1:
    gt_index = args[-1].index(">")
    if gt_index > 0 and args[-1][gt_index - 1] == "=":
      output_file_path = None
    else:
      output_file_path = args[-1][gt_index + 1:]
      args[-1] = args[-1][:gt_index]
  elif len(args) > 1 and args[-2].endswith(">"):
    output_file_path = args[-1]
    args = args[:-1]
    args[-1] = args[-1][:-1]
  else:
    output_file_path = None

  return args, output_file_path

def parse_tensor_name_with_slicing(in_str):
  """Parse tensor name, potentially suffixed by slicing string.

  Args:
    in_str: (str) Input name of the tensor, potentially followed by a slicing
      string. E.g.: Without slicing string: "hidden/weights/Variable:0", with
      slicing string: "hidden/weights/Variable:0[1, :]"

  Returns:
    (str) name of the tensor
    (str) slicing string, if any. If no slicing string is present, return "".
  """

  if in_str.count("[") == 1 and in_str.endswith("]"):
    tensor_name = in_str[:in_str.index("[")]
    tensor_slicing = in_str[in_str.index("["):]
  else:
    tensor_name = in_str
    tensor_slicing = ""

  return tensor_name, tensor_slicing

def validate_slicing_string(slicing_string):
  """Validate a slicing string.

  Check if the input string contains only brackets, digits, commas and
  colons that are valid characters in numpy-style array slicing.

  Args:
    slicing_string: (str) Input slicing string to be validated.

  Returns:
    (bool) True if and only if the slicing string is valid.
  """

  return bool(re.search(r"^\[(\d|,|\s|:)+\]$", slicing_string))

def _parse_slices(slicing_string):
  """Construct a tuple of slices from the slicing string.

  The string must be a valid slicing string.

  Args:
    slicing_string: (str) Input slicing string to be parsed.

  Returns:
    tuple(slice1, slice2, ...)

  Raises:
    ValueError: If tensor_slicing is not a valid numpy ndarray slicing str.
  """
  parsed = []
  for slice_string in slicing_string[1:-1].split(","):
    indices = slice_string.split(":")
    if len(indices) == 1:
      parsed.append(int(indices[0].strip()))
    elif 2 <= len(indices) <= 3:
      parsed.append(
          slice(*[
              int(index.strip()) if index.strip() else None for index in indices
          ]))
    else:
      raise ValueError("Invalid tensor-slicing string.")
  return tuple(parsed)

def parse_indices(indices_string):
  """Parse a string representing indices.

  For example, if the input is "[1, 2, 3]", the return value will be a list of
  indices: [1, 2, 3]

  Args:
    indices_string: (str) a string representing indices. Can optionally be
      surrounded by a pair of brackets.

  Returns:
    (list of int): Parsed indices.
  """

  # Strip whitespace.
  indices_string = re.sub(r"\s+", "", indices_string)

  # Strip any brackets at the two ends.
  if indices_string.startswith("[") and indices_string.endswith("]"):
    indices_string = indices_string[1:-1]

  return [int(element) for element in indices_string.split(",")]

def parse_ranges(range_string):
  """Parse a string representing numerical range(s).

  Args:
    range_string: (str) A string representing a numerical range or a list of
      them. For example:
        "[-1.0,1.0]", "[-inf, 0]", "[[-inf, -1.0], [1.0, inf]]"

  Returns:
    (list of list of float) A list of numerical ranges parsed from the input
      string.

  Raises:
    ValueError: If the input doesn't represent a range or a list of ranges.
  """

  range_string = range_string.strip()
  if not range_string:
    return []

  if "inf" in range_string:
    range_string = re.sub(r"inf", repr(sys.float_info.max), range_string)

  ranges = ast.literal_eval(range_string)
  if isinstance(ranges, list) and not isinstance(ranges[0], list):
    ranges = [ranges]

  # Verify that ranges is a list of list of numbers.
  for item in ranges:
    if len(item) != 2:
      raise ValueError("Incorrect number of elements in range")
    elif not isinstance(item[0], (int, float)):
      raise ValueError("Incorrect type in the 1st element of range: %s" %
                       type(item[0]))
    elif not isinstance(item[1], (int, float)):
      raise ValueError("Incorrect type in the 2nd element of range: %s" %
                       type(item[0]))

  return ranges

def parse_memory_interval(interval_str):
  """Convert a human-readable memory interval to a tuple of start and end value.

  Args:
    interval_str: (`str`) A human-readable str representing an interval
      (e.g., "[10kB, 20kB]", "<100M", ">100G"). Only the units "kB", "MB", "GB"
      are supported. The "B character at the end of the input `str` may be
      omitted.

  Returns:
    `Interval` object where start and end are in bytes.

  Raises:
    ValueError: if the input is not valid.
  """
  str_interval = _parse_interval(interval_str)
  interval_start = 0
  interval_end = float("inf")
  if str_interval.start:
    interval_start = parse_readable_size_str(str_interval.start)
  if str_interval.end:
    interval_end = parse_readable_size_str(str_interval.end)
  if interval_start > interval_end:
    raise ValueError(
        "Invalid interval %s. Start of interval must be less than or equal "
        "to end of interval." % interval_str)
  return Interval(interval_start, str_interval.start_included,
                  interval_end, str_interval.end_included)

def parse_time_interval(interval_str):
  """Convert a human-readable time interval to a tuple of start and end value.

  Args:
    interval_str: (`str`) A human-readable str representing an interval
      (e.g., "[10us, 20us]", "<100s", ">100ms"). Supported time suffixes are
      us, ms, s.

  Returns:
    `Interval` object where start and end are in microseconds.

  Raises:
    ValueError: if the input is not valid.
  """
  str_interval = _parse_interval(interval_str)
  interval_start = 0
  interval_end = float("inf")
  if str_interval.start:
    interval_start = parse_readable_time_str(str_interval.start)
  if str_interval.end:
    interval_end = parse_readable_time_str(str_interval.end)
  if interval_start > interval_end:
    raise ValueError(
        "Invalid interval %s. Start must be before end of interval." %
        interval_str)
  return Interval(interval_start, str_interval.start_included,
                  interval_end, str_interval.end_included)

def _parse_interval(interval_str):
  """Convert a human-readable interval to a tuple of start and end value.

  Args:
    interval_str: (`str`) A human-readable str representing an interval
      (e.g., "[1M, 2M]", "<100k", ">100ms"). The items following the ">", "<",
      ">=" and "<=" signs have to start with a number (e.g., 3.0, -2, .98).
      The same requirement applies to the items in the parentheses or brackets.

  Returns:
    Interval object where start or end can be None
    if the range is specified as "<N" or ">N" respectively.

  Raises:
    ValueError: if the input is not valid.
  """
  interval_str = interval_str.strip()
  if interval_str.startswith("<="):
    if _NUMBER_PATTERN.match(interval_str[2:].strip()):
      return Interval(start=None, start_included=False,
                      end=interval_str[2:].strip(), end_included=True)
    else:
      raise ValueError("Invalid value string after <= in '%s'" % interval_str)
  if interval_str.startswith("<"):
    if _NUMBER_PATTERN.match(interval_str[1:].strip()):
      return Interval(start=None, start_included=False,
                      end=interval_str[1:].strip(), end_included=False)
    else:
      raise ValueError("Invalid value string after < in '%s'" % interval_str)
  if interval_str.startswith(">="):
    if _NUMBER_PATTERN.match(interval_str[2:].strip()):
      return Interval(start=interval_str[2:].strip(), start_included=True,
                      end=None, end_included=False)
    else:
      raise ValueError("Invalid value string after >= in '%s'" % interval_str)
  if interval_str.startswith(">"):
    if _NUMBER_PATTERN.match(interval_str[1:].strip()):
      return Interval(start=interval_str[1:].strip(), start_included=False,
                      end=None, end_included=False)
    else:
      raise ValueError("Invalid value string after > in '%s'" % interval_str)

  if (not interval_str.startswith(("[", "("))
      or not interval_str.endswith(("]", ")"))):
    raise ValueError(
        "Invalid interval format: %s. Valid formats are: [min, max], "
        "(min, max), <max, >min" % interval_str)
  interval = interval_str[1:-1].split(",")
  if len(interval) != 2:
    raise ValueError(
        "Incorrect interval format: %s. Interval should specify two values: "
        "[min, max] or (min, max)." % interval_str)

  start_item = interval[0].strip()
  if not _NUMBER_PATTERN.match(start_item):
    raise ValueError("Invalid first item in interval: '%s'" % start_item)
  end_item = interval[1].strip()
  if not _NUMBER_PATTERN.match(end_item):
    raise ValueError("Invalid second item in interval: '%s'" % end_item)

  return Interval(start=start_item,
                  start_included=(interval_str[0] == "["),
                  end=end_item,
                  end_included=(interval_str[-1] == "]"))

def parse_readable_size_str(size_str):
  """Convert a human-readable str representation to number of bytes.

  Only the units "kB", "MB", "GB" are supported. The "B character at the end
  of the input `str` may be omitted.

  Args:
    size_str: (`str`) A human-readable str representing a number of bytes
      (e.g., "0", "1023", "1.1kB", "24 MB", "23GB", "100 G".

  Returns:
    (`int`) The parsed number of bytes.

  Raises:
    ValueError: on failure to parse the input `size_str`.
  """

  size_str = size_str.strip()
  if size_str.endswith("B"):
    size_str = size_str[:-1]

  if size_str.isdigit():
    return int(size_str)
  elif size_str.endswith("k"):
    return int(float(size_str[:-1]) * 1024)
  elif size_str.endswith("M"):
    return int(float(size_str[:-1]) * 1048576)
  elif size_str.endswith("G"):
    return int(float(size_str[:-1]) * 1073741824)
  else:
    raise ValueError("Failed to parsed human-readable byte size str: \"%s\"" %
                     size_str)

def parse_readable_time_str(time_str):
  """Parses a time string in the format N, Nus, Nms, Ns.

  Args:
    time_str: (`str`) string consisting of an integer time value optionally
      followed by 'us', 'ms', or 's' suffix. If suffix is not specified,
      value is assumed to be in microseconds. (e.g. 100us, 8ms, 5s, 100).

  Returns:
    Microseconds value.
  """
  def parse_positive_float(value_str):
    value = float(value_str)
    if value < 0:
      raise ValueError(
          "Invalid time %s. Time value must be positive." % value_str)
    return value

  time_str = time_str.strip()
  if time_str.endswith("us"):
    return int(parse_positive_float(time_str[:-2]))
  elif time_str.endswith("ms"):
    return int(parse_positive_float(time_str[:-2]) * 1e3)
  elif time_str.endswith("s"):
    return int(parse_positive_float(time_str[:-1]) * 1e6)
  return int(parse_positive_float(time_str))

def evaluate_tensor_slice(tensor, tensor_slicing):
  """Call eval on the slicing of a tensor, with validation.

  Args:
    tensor: (numpy ndarray) The tensor value.
    tensor_slicing: (str or None) Slicing of the tensor, e.g., "[:, 1]". If
      None, no slicing will be performed on the tensor.

  Returns:
    (numpy ndarray) The sliced tensor.

  Raises:
    ValueError: If tensor_slicing is not a valid numpy ndarray slicing str.
  """

  _ = tensor

  if not validate_slicing_string(tensor_slicing):
    raise ValueError("Invalid tensor-slicing string.")

  return tensor[_parse_slices(tensor_slicing)]

def get_print_tensor_argparser(description):
  """Get an ArgumentParser for a command that prints tensor values.

  Examples of such commands include print_tensor and print_feed.

  Args:
    description: Description of the ArgumentParser.

  Returns:
    An instance of argparse.ArgumentParser.
  """

  ap = argparse.ArgumentParser(
      description=description, usage=argparse.SUPPRESS)
  ap.add_argument(
      "tensor_name",
      type=str,
      help="Name of the tensor, followed by any slicing indices, "
      "e.g., hidden1/Wx_plus_b/MatMul:0, "
      "hidden1/Wx_plus_b/MatMul:0[1, :]")
  ap.add_argument(
      "-n",
      "--number",
      dest="number",
      type=int,
      default=-1,
      help="0-based dump number for the specified tensor. "
      "Required for tensor with multiple dumps.")
  ap.add_argument(
      "-r",
      "--ranges",
      dest="ranges",
      type=str,
      default="",
      help="Numerical ranges to highlight tensor elements in. "
      "Examples: -r 0,1e-8, -r [-0.1,0.1], "
      "-r \"[[-inf, -0.1], [0.1, inf]]\"")
  ap.add_argument(
      "-a",
      "--all",
      dest="print_all",
      action="store_true",
      help="Print the tensor in its entirety, i.e., do not use ellipses.")
  ap.add_argument(
      "-s",
      "--numeric_summary",
      action="store_true",
      help="Include summary for non-empty tensors of numeric (int*, float*, "
      "complex*) and Boolean types.")
  ap.add_argument(
      "-w",
      "--write_path",
      type=str,
      default="",
      help="Path of the numpy file to write the tensor data to, using "
      "numpy.save().")
  return ap


"""initial migrate

Revision ID: 53bde6239219
Revises:
Create Date: 2021-07-15 11:18:33.376830

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '53bde6239219'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('cryptozaken',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('zaaknaam', sa.String(length=64), nullable=True),
    sa.Column('verdachte', sa.String(length=64), nullable=True),
    sa.Column('currency', sa.String(length=64), nullable=True),
    sa.Column('aantal', sa.Float(), nullable=True),
    sa.Column('totale_waarde', sa.Float(), nullable=True),
    sa.Column('datum_tijd', sa.String(length=64), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('cryptozaken')
    # ### end Alembic commands ###


import sqlite3
import random

conn = sqlite3.connect('airConditioners.db')
conn2 = sqlite3.connect('boilers.db')
c=conn.cursor()
c2=conn2.cursor()

c.execute("""CREATE TABLE airConditionersData (
          airConditionerNumber integer,
          open_close text
           )""")

c2.execute("""CREATE TABLE boilersData (
           boilersNumber integer,
           open_close text
           )""")
conn.commit()
conn2.commit()

def insert_airConditioners(airConditionerNumber, open_close):
    with conn:
        c.execute("INSERT INTO airConditionersData VALUES (:airConditionerNumber, :open_close)",{'airConditionerNumber':airConditionerNumber, 'open_close':open_close})

def update_airConditioners(airConditionerNumber, open_close):
    with conn:
        c.execute("""UPDATE from airConditionersData SET open_close = :open_close WHERE airConditionerNumber =:airConditionerNumber""",{'airConditionerNumber':airConditionerNumber, 'open_close':open_close})

def get_status_by_number(airConditionerNumber):
    c.execute("SELECT * FROM airConditionersData WHERE airConditionerNumber=:airConditionerNumber", {'airConditionerNumber': airConditionerNumber})
    return c.fetchall()

def insert_boilers(boilersNumber, open_close):
    with conn2:
        c2.execute("INSERT INTO boilersData VALUES (:boilersNumber, :open_close)",{'boilersNumber':boilersNumber, 'open_close':open_close})

def update_window(boilersNumber, open_close):
    with conn2:
        c2.execute("""UPDATE from boilersData SET open_close = :open_close WHERE boilersNumber =:boilersNumber""",{'boilersNumber':boilersNumber, 'open_close':open_close})

def get_status_by_number(boilersNumber):
    c.execute("SELECT * FROM boilersData WHERE boilersNumber=:boilersNumber", {'boilersNumber': boilersNumber})
    return c.fetchall()

for x in range(21):

    mylist1 = ['open', 'close']
    insert_airConditioners(x+1,random.choice(mylist1))
    insert_boilers(x+1,random.choice(mylist1))

conn.close()
conn2.close()


from lzma import LZMADecompressor, LZMAError, FORMAT_AUTO
import optparse
import os, copy
import struct
from datetime import datetime, timedelta
from urllib import request
from urllib.error import HTTPError
from core.mongodb import Mongodb
import calendar
import pandas as pd

pymongo = Mongodb(host='127.0.0.1', port=27017)

def get_weekday(datetime_obj, week_day="monday"):
    """
    获取指定时间的当周的星期x
    :param datetime_obj: 时间
    :param week_day: 指定的星期x
    :return:
    """
    d = dict(zip(("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"),
                 range(7)))  # datetime 模块中，星期一到星期天对应数字 0 到 6
    delta_hour = timedelta(days=1)  # 改变幅度为 1 天
    while datetime_obj.weekday() != d.get(week_day):
        if datetime_obj.weekday() > d.get(week_day):
            datetime_obj -= delta_hour
        elif datetime_obj.weekday() < d.get(week_day):
            datetime_obj += delta_hour
        else:
            pass
    return datetime_obj

def get_first_weekday(year, month, n=1, w="monday"):
    """
    获取 year 年，month 月 的第n个星期w和倒数第n个星期w的日期
    :param year: 指定年份，如 2019
    :param month: 指定月份，如 6
    :param n: 第n个
    :param w: 指定的星期w
    :return:
    """
    # 获取第一和最后一天
    d = dict(zip(("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"),
                 range(7)))  # datetime 模块中，星期一到星期天对应数字 0 到 6
    weekday, count_day = calendar.monthrange(year=year,
                                             month=month)  # 返回指定月份第一天（即1号）的星期日期，和本月的总天数 https://blog.csdn.net/tz_zs/article/details/86629959
    first_day = datetime(year=year, month=month, day=1)  # <type 'datetime.datetime'>
    last_day = datetime(year=year, month=month, day=count_day)
    # first_day, last_day = get_month_firstday_and_lastday(year=year, month=month, n=1)

    # 第1个星期w
    if first_day.weekday() > d.get(w):  # 说明本周的星期w在上个月
        datetime_obj = first_day + timedelta(weeks=1)
    else:
        datetime_obj = first_day
    datetime_obj += timedelta(weeks=n - 1)
    first_weekday = get_weekday(datetime_obj=datetime_obj, week_day=w)

    return first_weekday

def decompress_lzma(data):
    results = []
    len(data)
    while True:
        decomp = LZMADecompressor(FORMAT_AUTO, None, None)
        try:
            res = decomp.decompress(data)
        except LZMAError:
            if results:
                break
            else:
                raise
        results.append(res)
        data = decomp.unused_data
        if not data:
            break
        if not decomp.eof:
            raise LZMAError("Compressed data ended before the end-of-stream marker was reached")
    return b"".join(results)

def tokenize(buffer):
    token_size = 20
    token_count = int(len(buffer) / token_size)
    tokens = list(map(lambda x: struct.unpack_from('>3I2f', buffer, token_size * x), range(0, token_count)))
    return tokens

def normalize_tick(symbol, day, time, ask, bid, ask_vol, bid_vol):
    date = day + timedelta(milliseconds=time)

    # TODO 網羅する。この通過ペア以外も有るかも
    if any(map(lambda x: x in symbol.lower(), ['usdrub', 'xagusd', 'xauusd', 'jpy'])):
        point = 1000
    else:
        point = 100000

    return [date, ask / point, bid / point, round(ask_vol * 1000000), round(bid_vol * 1000000)]

def format_to_csv_for_ticks(ticks):
    df = pd.DataFrame(ticks, columns=['date', 'Ask', 'Bid', 'AskVolume', 'BidVolume'])
    df = df.drop(['Ask', 'AskVolume', 'BidVolume'], axis=1)
    df['date'] = pd.to_datetime(df['date'])
    df.set_index('date', inplace=True)
    return df

def format_to_csv_for_candle(ticks, scale, price_type):
    df = pd.DataFrame(ticks, columns=['date', 'Ask', 'Bid', 'AskVolume', 'BidVolume'])
    if price_type == "bid":
        df = df.drop(['Ask', 'AskVolume', 'BidVolume'], axis=1)
    elif price_type == "ask":

        df = df.drop(['Bid', 'AskVolume', 'BidVolume'], axis=1)
    else:
        df = df.drop(['AskVolume', 'BidVolume'], axis=1)
    df['date'] = pd.to_datetime(df['date'])
    df.set_index('date', inplace=True)
    if scale.lower() in ["4h", "1d"]:
        # is_dst_result = is_dst(df.index[0])
        is_dst_result, _, _ = is_dst(datetime.strptime(df.index[0].strftime("%Y-%m-%d"), "%Y-%m-%d"))
        if is_dst_result:
            # 夏令时，4h的开始时间是凌晨1点
            h4_start_hour = "21h"
        else:
            h4_start_hour = "22h"
        df_ohlc = df.resample(scale, offset=h4_start_hour).ohlc()
        df_ohlcv = df_ohlc.assign(Volume=df.iloc[:, 0].resample(scale, offset=h4_start_hour).count())
    else:
        df_ohlc = df.resample(scale).ohlc()
        df_ohlcv = df_ohlc.assign(Volume=df.iloc[:, 0].resample(scale).count())

    return df_ohlcv

def is_dst(day):
    dst_start = get_first_weekday(year=day.year, month=3, n=2, w="sunday")
    dst_end = get_first_weekday(year=day.year, month=11, n=1, w="sunday")
    if dst_start <= day < dst_end:
        return True, dst_start, dst_end
    return False, dst_start, dst_end

def load_bi5(path, symbol, day: datetime):
    file_name = f'{day.hour:02d}h_ticks.bi5'
    file_path = f'{path}\{symbol}\{day.year:04d}\{day.month - 1:02d}\{day.day:02d}\{file_name}'
    print(f'loading: {file_path}')

    # req = request.Request(url)
    try:
        with open(file_path, "rb") as res:
            res_body = res.read()
    except Exception as e:
        print('load failed. continuing...save_path=%s' % file_path)

        url_prefix = 'https://datafeed.dukascopy.com/datafeed'
        url = f'{url_prefix}/{symbol}/{day.year:04d}/{day.month - 1:02d}/{day.day:02d}/{file_name}'
        print("download url:%s" % url)
        if not os.path.exists(file_path[:file_path.rfind("\\")]):
            os.makedirs(file_path[:file_path.rfind("\\")])
        req = request.Request(url)
        try:
            with request.urlopen(req) as res:
                if res.code == 200:
                    res_body = res.read()

                    with open(file_path, "wb") as f:
                        f.write(res_body)
                else:
                    print("download fail:%s" % url)
        except HTTPError:
            print('download failed. continuing..')

    data = []
    if len(res_body):
        try:
            data = decompress_lzma(res_body)
        except LZMAError:
            print('decompress failed. continuing..')

    tokenized_data = tokenize(data)

    ticks_hour = list(map(lambda x: normalize_tick(symbol, day, *x), tokenized_data))
    return ticks_hour

def from_local_path(path, symbol, day):
    ticks_day = []

    result, dst_start, dst_end = is_dst(day)
    if result:
        # end_time = day + timedelta(hours=0) + timedelta(days=1)
        # if dst_start == day:
        #     day += timedelta(hours=20)
        #
        # while True:
        #     if day == end_time:
        #         break
        #     day = day + timedelta(hours=1)
        #     ticks_hour = load_bi5(path, symbol, day)
        #     ticks_day.extend(ticks_hour)

        if dst_start == day + timedelta(days=1):
            return []
        end_time = day + timedelta(hours=21) + timedelta(days=1)
        if dst_end == day:
            day = day + timedelta(hours=-3)
        else:
            day = day + timedelta(hours=-3) + timedelta(days=1)
        while True:
            if day == end_time:
                break

            ticks_hour = load_bi5(path, symbol, day)
            day = day + timedelta(hours=1)
            ticks_day.extend(ticks_hour)

    else:
        if dst_start == day + timedelta(days=1):
            return []
        end_time = day + timedelta(hours=22) + timedelta(days=1)
        if dst_end == day:
            day = day + timedelta(hours=-2)
        else:
            day = day + timedelta(hours=-2) + timedelta(days=1)
        while True:
            if day == end_time:
                break

            ticks_hour = load_bi5(path, symbol, day)
            day = day + timedelta(hours=1)
            ticks_day.extend(ticks_hour)
    print("end...........")
    return ticks_day

def main():
    parser = optparse.OptionParser(
        usage='Usage: python {} [options] <symbol> <start: yyyy-mm-dd> <end: yyyy-mm-dd>'.format(
            os.path.basename(__file__)))
    parser.add_option('-c', action="store", metavar='time_scale', dest="c", help="candlestick. ex: 1min, 1H, 1D")
    parser.add_option('-d', action="store", metavar='output_dir', dest="d", default='./', help="output directory.")
    parser.add_option('-t', action="store", metavar='price_type', dest="t", default='bid',
                      help="select price type. ex: bid, ask, all")
    parser.add_option('--mongo', action="store_true", help="save data to mongodb")
    parser.add_option('-s', action="store", metavar='source_dir', dest="s", default='./', help="source data directory.")

    (options, args) = parser.parse_args()

    if len(args) < 3:
        parser.print_help()
        exit(1)

    symbol = args[0].upper()
    start_date = datetime.strptime(args[1], '%Y-%m-%d')
    end_date = datetime.strptime(args[2], '%Y-%m-%d')

    save_mongo = True if options.mongo else False
    output_dir = options.d
    output_suffix = f'_{options.c}' if options.c is not None else ''
    output_csv = f'{output_dir}/{symbol}_{start_date.strftime("%Y-%m-%d")}_{end_date.strftime("%Y-%m-%d")}{output_suffix}_{options.t}.csv'

    d = start_date

    result_df = pd.DataFrame()
    while d <= end_date:
        df = pd.DataFrame()
        ticks_day = from_local_path(options.s, symbol, d)

        if options.c is None:

            df = format_to_csv_for_ticks(ticks_day)
            result_df = pd.concat([result_df, df])
            # result_df.columns = ["bid"]

        else:
            if ticks_day:
                df = format_to_csv_for_candle(ticks_day, options.c, options.t)

                df.columns = ["open", "high", "low", "close", "volume"] if options.t in ["bid", "ask"] else ["ask_open",
                                                                                "ask_high",
                                                                                "ask_low",
                                                                                "ask_close",
                                                                                "bid_open",
                                                                                "bid_high",
                                                                                "bid_low",
                                                                                "bid_close",
                                                                                "volume"]

                result_df = pd.concat([result_df, df])

            # f.write(format_to_csv_for_candle(ticks_day, options.c))

        d += timedelta(days=1)

        if save_mongo:
            try:
                df = df.reset_index()
                df["complete"] = 1.0
                df["date"] = df["date"].astype(str)
                df["volume"] = df['volume'].astype(float)
                df['timestamp'] = df["date"].apply(lambda x: x.split(" ")[1])
                data = df.to_dict(orient='records')
                pymongo.conn["%s_duka" %symbol][options.c].insert_many(data)
            except Exception as e:
                print(e)

    if not save_mongo:
        result_df.to_csv(output_csv)

if __name__ == '__main__':
    main()


from django.db.models.deletion import ProtectedError
from rest_framework import mixins, serializers
from rest_framework.viewsets import GenericViewSet

class DestroyModelMixin(mixins.DestroyModelMixin):
    """
    Destroy a model instance.  Raise a validation error if a ProtectedError is raised by a database constraint.
    """

    def perform_destroy(self, instance):
        try:
            instance.delete()
        except ProtectedError as e:
            msg = 'Referenced by protected {} {}'.format(
                e.protected_objects.model._meta.label,
                ', '.join([str(x) for x in e.protected_objects.values_list('id', flat=True)])
            )
            raise serializers.ValidationError(msg)

class ModelViewSet(mixins.CreateModelMixin,
                   mixins.RetrieveModelMixin,
                   mixins.UpdateModelMixin,
                   DestroyModelMixin,
                   mixins.ListModelMixin,
                   GenericViewSet):
    """
    A viewset that provides default `create()`, `retrieve()`, `update()`, `partial_update()`, `destroy()` and `list()`
    actions.  This class incorporates system-wide customizations of the basic DRF viewet behavior; in this case, uses
    our DestroyModelMixin.
    """
    pass


"""
This file is part of the everest project.
See LICENSE.txt for licensing, CONTRIBUTORS.txt for contributor information.

Package setup file.

Created on Nov 3, 2011.
"""

import os

from distribute_setup import use_setuptools
use_setuptools()

from setuptools import setup
from setuptools import find_packages

here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.txt')).read()

setup_requirements = []

install_requirements = [
    ]

tests_requirements = install_requirements + [
    'nose>=1.1.0,<=1.1.99',
    'nosexcover>=1.0.4,<=1.0.99',
    'coverage==3.4',
    ]

setup(name='tractor',
      version='0.1',
      description=
        'A Python library to manipulate Trac tickets programmatically.',
      long_description=README,
      classifiers=[
        "Development Status :: 4 - Beta",
        "Environment :: Web Environment",
        "Intended Audience :: Developers",
        "Intended Audience :: Information Technology",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Topic :: Internet :: WWW/HTTP",
        ],
      author='Anna-Antonia Berger',
      author_email='berger@cenix.com',
      license="MIT",
      url='https://github.com/cenix/tractor',
      keywords='web trac',
      packages=find_packages(),
      include_package_data=True,
      zip_safe=False,
      setup_requires=setup_requirements,
      install_requires=install_requirements,
      tests_require=tests_requirements,
      test_suite="tractor/tests",
      entry_points="""
      """
      )


"""add columns to user

Revision ID: cbf708203191
Revises: 240d251d908b
Create Date: 2017-12-21 12:34:05.816547

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'cbf708203191'
down_revision = '240d251d908b'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('image_filename', sa.String(), nullable=True))
    op.add_column('users', sa.Column('image_url', sa.String(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('users', 'image_url')
    op.drop_column('users', 'image_filename')
    # ### end Alembic commands ###


import numpy as np

from gym.spaces import Box
from gym import ObservationWrapper

class GrayScaleObservation(ObservationWrapper):
    r"""Convert the image observation from RGB to gray scale. """
    def __init__(self, env, keep_dim=False):
        super(GrayScaleObservation, self).__init__(env)
        self.keep_dim = keep_dim

        assert len(env.observation_space.shape) == 3 and env.observation_space.shape[-1] == 3
        obs_shape = self.observation_space.shape[:2]
        if self.keep_dim:
            self.observation_space = Box(low=0, high=255, shape=(obs_shape[0], obs_shape[1], 1), dtype=np.uint8)
        else:
            self.observation_space = Box(low=0, high=255, shape=obs_shape, dtype=np.uint8)

    def observation(self, observation):
        import cv2
        observation = cv2.cvtColor(observation, cv2.COLOR_RGB2GRAY)
        if self.keep_dim:
            observation = np.expand_dims(observation, -1)
        return observation


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@created: <data>
#@author: <name>
#@contact: <email>

import sys
import argparse
import os
import os.path

def main(input_dir, outdirectory):
    ''' Function description.
    '''

    def fasta_reader(file):
        fasta = {}
        header = None
        with open(file) as fh:
            for i, line in enumerate(fh):
                line = line.strip()
                if line.startswith(">"):
                    if header:
                        fasta[header]="".join(seq)
                    header = line
                    seq = []
                else:
                    seq.append(line)
        if header:
            fasta[header] = "".join(seq)
        return fasta

    def revcomp(seq):
        new_seq = ""
        for i in seq[::-1]:
            if i == "A":
                new_seq += "T"
            elif i == "T":
                new_seq += "A"
            elif i == "G":
                new_seq += "C"
            else:
                new_seq += "G"
        return new_seq

    path = os.path.abspath(input_dir)
    inp_dir_list = os.listdir(path)
    for i in inp_dir_list:
        if ".fna" in i:
            input_fasta = path + "/" + i
            print("INPUT FASTA FILE IS %s" % input_fasta)
        if ".gff" in i:
            input_gff = path + "/" + i
            print("INPUT GFF FILE IS %s" % input_gff)

    prefix = input_fasta.split("/")[-1].split(".")[0]
    outdir = outdirectory + "/" + prefix
    print("OUTPUT DIRECTORY IS %s" % outdir)
    out_file = outdir + "/" + prefix + "_genes_seq.txt"
    print("OUTPUT FILE IS %s" % out_file)

    if os.path.exists(outdir):
        pass
    else:
        os.mkdir(outdir)

    fasta = fasta_reader(input_fasta)

    #gff_reader
    i = 0
    with open(out_file, "w") as fw:
        with open(input_gff) as fh:
            for line in fh:
                if line.startswith("#"):
                    continue
                if line.startswith(">"): # to read until fasta part starts
                    break
                line = line.strip().split("\t")
                if not "gene" in line[8]:
                    continue
                gene = line[8].split("gene")[1].split(";")[0][1:]
                contig = line[0]
                start = int(line[3])
                stop = int(line[4])
                strain = line[6]
                product = line[8].split(";")[-1]
                new_header = ">%s:%s:%s:%s:%s:%s" % (gene, prefix, contig, start, stop, product)
                i+=1
                for header, seq in fasta.items():
                    if header == ">" + contig:
                        fw.write(new_header + "\n")
                        if strain == "+":
                            fw.write(seq[start-1:stop] + "\n")
                        elif strain == "-":
                            fw.write(revcomp(seq[start-1:stop]) + "\n") #add reverce compliment
    fw.close()

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Prokka runner')
    parser.add_argument('-i','--input', help='path to prokka dir', required=True)
    parser.add_argument('-o','--outdir', help='output directory', required=True)
    parser.add_argument('-d','--dry', help='dry run', action='store_true')
    args = vars(parser.parse_args())

    input_dir = os.path.abspath(args["input"])
    outdirectory = os.path.abspath(args["outdir"])
    dry = args["dry"]

    if os.path.exists(outdirectory):
        pass
    else:
        os.mkdir(outdirectory)

    main(input_dir, outdirectory)
    print("DONE for path %s" % input_dir)


'''
TODO: 未完成（难，优先度低）
动态的validator set的生成机制
new validators to join and existing validators to leave
#first version: finished
#second version: deposit mechanism and first dynasty initialization finished
#third verison: transaction need to do
'''
import copy
import logging

logger = logging.getLogger(__name__)
from configuration.generalParameters import INIT_DYNASTY

class Dynasty:
    def __init__(self):
        # [(new,expert,retired)] d+2
        self.dynasties = [[[], INIT_DYNASTY, []], [[], [], []], [[], [], []]]
        self.current_epoch = 0  # d
        self.join_community = [] + INIT_DYNASTY
        # {validator_addr:[deposit,epoch]}
        self.deposit_bank = {}
        self.withdraw_delay = 1

    def joinDynasty(self, validator_address, deposit=1):
        '''
        func: apply to join the validator set
        args:
            validatorAddress: the identity of new validator
            deposit: the deposit of the new validator
        '''

        if not deposit or deposit <= 0:
            logger.warning('warning: validator %s please pay the deposit' % validator_address)
            return

        # 如果曾经加入过，就再也不能加入了
        if validator_address not in self.join_community:
            self.deposit_bank[validator_address] = [deposit, self.current_epoch]
            self.dynasties[self.current_epoch + 2][0].append(validator_address)
            # join
            self.join_community.append(validator_address)

    def quitDynasty(self, validator_address):
        '''
        func: apply to quit the validator set
        args:
            validatorAddress: the identity of new validator
        '''
        if validator_address in self.dynasties[self.current_epoch][1]:
            # join_dynasty changed to quit dynasty(d+2+5)
            self.deposit_bank[validator_address][1] = self.current_epoch + 2 + self.withdraw_delay
            self.dynasties[self.current_epoch + 2][2].append(validator_address)

        elif validator_address in self.dynasties[self.current_epoch + 1][0]:
            # join_dynasty changed to quit dynasty(d+2+5)
            self.deposit_bank[validator_address][1] = self.current_epoch + 2 + self.withdraw_delay
            self.dynasties[self.current_epoch + 2][0].remove(validator_address)
            self.dynasties[self.current_epoch + 2][2].append(validator_address)
        else:
            logger.warning('warning: %s not in the new and expert validator', str(validator_address))

    def dynastyChange(self):
        '''
        func: dynasty change
        '''
        self.current_epoch += 1

        self.dynasties.append([[], [], []])

        current_dynasty = self.dynasties[self.current_epoch]
        previous_dynasty = self.dynasties[self.current_epoch - 1]

        previous_new = set(copy.deepcopy(previous_dynasty[0]))
        previous_og = set(copy.deepcopy(previous_dynasty[1]))

        current_retire = set(copy.deepcopy(current_dynasty[2]))
        current_og = list(previous_og.union(previous_new) - current_retire)

        current_dynasty[1] = current_og
        #
        # self.dynasties.append(copy.deepcopy(self.dynasties[-1]))
        # # new validator move to expert validator
        # new_people = copy.deepcopy(self.dynasties[-1][0])
        # # print(self.dynasties[-1][1])
        # # print(new_people)
        # # self.dynasties[-1][1]=new_people
        # for validatorAddress in new_people:
        #     self.dynasties[-1][1].append(validatorAddress)
        # self.dynasties[-1][0].clear()
        # # print('current bank',self.deposit_bank)
        # removed_key = []
        # for k,v in self.deposit_bank.items():
        #     # print('-----------',k,v)
        #     if v[1] == self.current_epoch:
        #         removed_key.append(k)
        # if removed_key:
        #     for key in removed_key:
        #         self.deposit_bank.pop(key)
        #     removed_key = []
        #

# if __name__ == '__main__':
#     a = Dynasty()
#     #------join
#     a.join(1,10)
#     a.join(2,10)
#     a.join(3,0)
#     print('current dynasty ',a.current_epoch,'total dynasty',a.dynasties)
#     print('deposit bank ',a.deposit_bank)
#     #------dynastyChange
#     a.dynastyChange()
#     print('current dynasty ',a.current_epoch,'total dynasty',a.dynasties)
#     # #------
#     a.join(4,22)
#     print('current dynasty ',a.current_epoch,'total dynasty',a.dynasties)
#     print('deposit bank ', a.deposit_bank)
#     # #------wrong quit
#     a.quit(5)
#     print(a.dynasties)
#     # #------right quit
#     a.quit(1)
#     print('current dynasty ',a.current_epoch,'total dynasty',a.dynasties)
#     print('deposit bank ', a.deposit_bank)
#     # #------dynasty change
#     a.dynastyChange()
#     print('current dynasty ',a.current_epoch,'total dynasty',a.dynasties)
#     print('deposit bank ', a.deposit_bank)
#     a.dynastyChange()
#     print('current dynasty ', a.current_epoch, 'total dynasty', a.dynasties)
#     print('deposit bank ', a.deposit_bank)
#     a.dynastyChange()
#     print('current dynasty ', a.current_epoch, 'total dynasty', a.dynasties)
#     print('deposit bank ', a.deposit_bank)
#     print(a.join_community)


from __future__ import (absolute_import, division, print_function)

import matplotlib as mpl
import matplotlib.pyplot as plt
# The two statemens below are used mainly to set up a plotting
# default style that's better than the default from Matplotlib 1.x
# Matplotlib 2.0 supposedly has better default styles.
import seaborn as sns
plt.style.use('bmh')
from pathlib import Path
data_folder = Path('D:/OneDrive/AAMOSM2018/0828mapdata')

#data_folder = Path('C:/Users/langzx/OneDrive/AAMOSM2018/0828mapdata')
from shapely.geometry import Point, Polygon
from matplotlib.lines import Line2D
import pandas as pd
import geopandas as gpd
from geopandas import GeoSeries, GeoDataFrame
#import pyepsg
mpl.__version__, pd.__version__, gpd.__version__

from shapely.wkt import loads
from fiona.crs import from_epsg
from pyproj import Proj
 #Proj('init=epsg:4326', preserve_flags=True)
# Determine the CRS of the GeoDataFrame

GeoSeries([loads('POINT(1 2)'), loads('POINT(1.5 2.5)'), loads('POINT(2 3)')])

subbasin = gpd.read_file(data_folder/"subbasins.shp")
subbasin_p4 =  subbasin.to_crs(epsg=4326)
subbasin.crs
subbasin.crs = from_epsg(4326)
subbasin_p4 =  subbasin.to_crs(epsg = 4326)
subbasin.crs = {'init' :'epsg:4326'}
subbasin.crs
subbasin['geometry']

#subbasin.plot( color='red', markersize=100, figsize=(4, 4))
gage = gpd.read_file(data_folder/"gage2.shp")
gs = GeoSeries([Point(-120, 45), Point(-121.2, 46), Point(-122.9, 47.5),Point(-122.9, 47.5),Point(-122.9, 47.5)])
#subbasin['Watershed_Zone'] = subbasin.watershed.map(str) + " " + subbasin.Zone

#subbasin = gpd.read_file("C:\\Users\langzx\\Onedrive\\AAMOSM2018\\newmapdata\\subs1.shp")
stream =  gpd.read_file(data_folder/ "LeSueur_Streams.shp")
## project
subbasin.crs = {'init' :'epsg:4326'}
subbasin.crs
#subbasin_p4 =  subbasin.to_crs(epsg=4326)
gage.crs = {'init' :'epsg:4326'}

#gage_p4.plot(marker='*', color='red', markersize=100, figsize=(4, 4))

#gage2_p4 =  gage2.to_crs
stream.crs= {'init' :'epsg:4326'}

len(subbasin.columns.values)
subbasin.columns.values
type(subbasin['geometry'])

subbasin['coords'] = subbasin['geometry'].apply(lambda x: x.representative_point().coords[:])
subbasin['coords'] = [coords[0] for coords in subbasin['coords']]

f, ax = plt.subplots(1, figsize=(20, 20))
ax.set_title('')

# Other nice categorical color maps (cmap) include 'Set2' and 'Set3'
subbasin.plot(ax = ax, column = 'watershed', linewidth=0.8, cmap='summer_r',edgecolor='#B3B3B3', legend = True)
stream.plot(ax = ax, edgecolor='blue')
gage.plot(ax = ax, marker='*', color='red', markersize=400)
ax.set_ylim([43.6, 44.3])
plt.axis('equal');

##Annotation
for idx, row in subbasin.iterrows():
    ax.annotate(s=row['Zone'], xy=row['coords'],
                 verticalalignment='center')

#plt.axis('equal');
custom_lines = [Line2D([0], [0], color=cmap(0.), lw=20),
                Line2D([0], [0], color=cmap(.5), lw=20),
                Line2D([0], [0], color=cmap(1.), lw=20),
                Line2D([0], [0], marker='*', color='w',markersize=20, markerfacecolor='r'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w')]
legend = ax.legend(custom_lines, ['COB', 'LES', 'MAP','Gages','Zone 1 = Upland','Zone 2 = Transitional','Zone 3 = Incised'], fontsize=20)
legend.get_frame().set_facecolor('white')

type(subbasin.ix[23, 'geometry'])
subbasin['coords'] = subbasin['geometry'].apply(lambda x: x.representative_point().coords[:])
subbasin['coords'] = [coords[0] for coords in subbasin['coords']]

subbasin.plot( cmap='summer_r', legend=True, k=3, figsize=(12, 15))
for idx, row in subbasin.iterrows():
    plt.annotate(s=row['Zone'], xy=row['coords'],
                 horizontalalignment='center')
gage.plot( marker='*', color='red', markersize=200)
stream.plot(ax=ax, edgecolor='blue')

subbasin.plot(column='watershed', cmap='summer_r',  k=3, figsize=(8, 10));

subbasin.plot(color='white', edgecolor='black', figsize=(8, 8));

# Visualize
ax = subbasin.plot()

gdf.columns.values
gdf.plot(cmap='Set2', figsize=(10, 10));

ax = subbasin.plot(color='white', edgecolor='black')
ax.set_axis_off()
gdf.plot(ax = ax, edgecolor='white')
plt.show()

##################

plt.figure(figsize=(20,5))
cmap = plt.cm.summer_r
t = ("Zone 1 = Upland\n"
     "Zone 2 = Transitional\n"
     "Zone 3 = Incised")

f, ax = plt.subplots(1, figsize=(20,20))
ax.set_facecolor('white')
ax.set_ylim([43.6, 44.3])
ax.grid(False)
# Other nice categorical color maps (cmap) include 'Set2' and 'Set3'

type(subbasin.ix[23, 'geometry'])
subbasin['coords'] = subbasin['geometry'].apply(lambda x: x.representative_point().coords[:])
subbasin['coords'] = [coords[0] for coords in subbasin['coords']]

subbasin.plot(ax=ax,column='watershed',linewidth=0.8, cmap='summer_r',edgecolor='#B3B3B3', legend = True)
gage.plot(ax = ax, marker='*', color='red', markersize=200)
stream.plot(ax = ax, edgecolor='blue')

for idx, row in subbasin.iterrows():
    ax.annotate(s=row['Zone'], xy=row['coords'],
                 verticalalignment='center')

#plt.axis('equal');
custom_lines = [Line2D([0], [0], color=cmap(0.), lw=20),
                Line2D([0], [0], color=cmap(.5), lw=20),
                Line2D([0], [0], color=cmap(1.), lw=20),
                Line2D([0], [0], marker='*', color='w',markersize=20, markerfacecolor='r'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w'),
                Line2D([0], [0],  color='w',markersize=20, markerfacecolor='w')]
legend = ax.legend(custom_lines, ['COB', 'LES', 'MAP','Gages','Zone 1 = Upland','Zone 2 = Transitional','Zone 3 = Incised'], fontsize=20)
legend.get_frame().set_facecolor('white')

#ax.text(0.97, 0.87, t,
#       verticalalignment='top', horizontalalignment='right',
#        transform=ax.transAxes,
#        fontsize=15,wrap=True)
#ax.text(0.5,0.5,u'\u25B2 \nN ', ha='center', fontsize=20, family='Arial')
plt.savefig('mosmmap.pdf', bbox_inches='tight')

subbasin.crs
# Visualize
ax = subbasin.plot()

gdf.columns.values
gdf.plot(cmap='Set2', figsize=(10, 10));

ax = subbasin.plot(color='white', edgecolor='black')
ax.set_axis_off()
gdf.plot(ax = ax, edgecolor='white')
plt.show()


from typing import List
import pdb

solution_json = {
    "date": "2021/4/29",
    "design": 0,
    "coding": 7,
    "runtime": "48 ms",
    "memory": "15.4 MB"
}

class Solution:
    def thirdMax(self, nums: List[int]) -> int:
    	d = False
    	v = nums
    	n = len(v)

    	hash = {}
    	for i in range(n):
    		if v[i] not in hash:
    			hash[v[i]] = True

    	ls = [*hash.keys()]
    	ls.sort(reverse=True)
    	if d:
    		print(ls)

    	n = len(ls)
    	if n >= 3:
    		out = ls[2]
    	elif n in [1, 2]:
    		out = ls[0]
    	else:
    		out = None

    	return out


import logging
import urllib

from logging.handlers import SysLogHandler

from cache import cached_result
from config import cfg
from flask import Flask, jsonify, abort, request

app = Flask(__name__)

API_MOUNTPOINT = '/api'
API_VERSION = 'v1'

handler = SysLogHandler(address='/dev/log', facility=SysLogHandler.LOG_USER)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("music-discovery - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger = logging.getLogger('music-discovery')
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)

def api_url(url):
    url = url[1:] if url.startswith('/') else url
    a = '/'.join([x for x in [API_MOUNTPOINT, API_VERSION, url]])
    return a

# @app.route(api_url('/artist/bio'), methods=['POST'])
def artist_bio(req):
    """Fetches the artist biography summary text from last.fm

    Expects JSON paramater 'artist'"""

    artist = req.get('result').get('parameters').get('artist')
    logger.info('received {0} request for artist="{1}"'.format('artist_bio', artist))
    bio = cached_result('get_artist_info', [artist], {})
    speech = 'Unable to locate biography for {artist}'.format(artist=artist)

    if bio and len(bio) > 0:
        speech = bio

    return jsonify(
        {'speech': speech,
         'displayText': speech,
         'source': 'last-assist'})

# @app.route(api_url('/artist/toptracks'), methods=['POST'])
def artist_top_tracks(req):
    """Fetches top tracks for artist from last.fm

    Expects JSON paramater 'artist'"""

    artist = req.get('result').get('parameters').get('artist')
    logger.info('received {0} request for artist="{1}"'.format('artist_top_tracks', artist))
    top_tracks = cached_result('get_artist_top_tracks', [artist], {})
    speech = 'Unable to locate top tracks for {artist}'.format(artist=artist)

    if top_tracks is not None:
        speech = 'The most popular songs for {artist} are'.format(artist=artist)
        for track in top_tracks:
            speech += '. ' + track

    return jsonify(
        {'speech': speech,
         'displayText': speech,
         'source': 'last-assist'})

# @app.route(api_url('/artist/similar'), methods=['POST'])
def artist_similar(req):
    """Fetches artists that are similar to the given artist

    Expects JSON paramater 'artist'"""

    artist = req.get('result').get('parameters').get('artist')
    logger.info('received {0} request for artist="{1}"'.format('artist_similar', artist))
    sim_artists = cached_result('get_similar_artists', [artist], {})
    speech = 'Unable to locate similar artists for {artist}'.format(artist=artist)

    if sim_artists and len(sim_artists) > 0:
        speech = 'The following artists are similar to {artist}'.format(
            artist=artist)
        for art in sim_artists:
            speech += u'. {artist}'.format(artist=art)

    return jsonify(
        {'speech': speech,
         'displayText': speech,
         'source': 'last-assist'})

# @app.route(api_url('/track/similar'), methods=['POST'])
def track_similar(req):
    """Given a song title and artist, this will retrieve like or similar songs.

    Expects JSON paramater 'artist' and 'track'"""

    artist = req.get('result').get('parameters').get('artist')
    track = req.get('result').get('parameters').get('track')
    logger.info('received {0} request for artist="{1}", track="{2}"'.format('track_similar', artist, track))
    sim_tracks = cached_result('get_similar_tracks', [artist, track], {})
    speech = 'Unable to locate similar tracks for song {track} by {artist}'.format(
        track=track, artist=artist)

    if sim_tracks and len(sim_tracks) > 0:
        speech = 'The following songs are similar to {track} by {artist}'.format(
            track=track, artist=artist)
        for track in sim_tracks:
            speech += '. ' + track[0] + ' by ' + track[1]

    return jsonify(
        {'speech': speech,
         'displayText': speech,
         'source': 'last-assist'}
    )

@app.route(api_url('/apiai-hook'), methods=['POST'])
def apiai_hook():
    """A single entry point into the web hook. Request is then dispersed
    to the appropriate method based on the 'action' passed from API.ai.
    API.ai allows only a single fullfillment endpoint per agent."""

    route = {
        'artist_bio': artist_bio,
        'artist_top_tracks': artist_top_tracks,
        'artist_similar': artist_similar,
        'track_similar': track_similar,
    }

    req = request.get_json(silent=True, force=True)
    response = {}
    try:
        response = route[req.get('result').get('action')](req)
    except (KeyError, AttributeError) as e:
        logger.error('Invalid action specified, error="{0}".'.format(e))
        return jsonify(response)

    return response

@app.errorhandler(500)
def server_error(e):
    logger.exception('An error occurred during a request.')
    return 'An internal error occurred.', 500

if __name__ == '__main__':
    app.run(debug=False)


import torch
import torch.nn as nn
import torch.nn.functional as F
import _C_gemm

class sparse_mm_dense_atf(torch.autograd.Function):
    @staticmethod
    def forward(ctx, sparse: torch.Tensor, dense: torch.Tensor):
        # sparse: [M, N]  dense: [N, P]  y:[M, P]
        if sparse.requires_grad or dense.requires_grad:
            ctx.save_for_backward(sparse, dense)
        y = torch.zeros(size=[sparse.shape[0], dense.shape[1]], dtype=torch.float, device=sparse.device)
        _C_gemm.sparse_mm_dense_cusparse(sparse, dense, y)
        # y = torch.mm(sparse, dense)
        return y

    @staticmethod
    def backward(ctx, grad_output):
        # grad_output: [M, P]
        sparse, dense = ctx.saved_tensors
        grad_sparse = grad_dense = None
        if ctx.needs_input_grad[0]:
            grad_sparse = grad_output.mm(dense.t())
        if ctx.needs_input_grad[1]:
            grad_dense = torch.zeros_like(dense.data)
            _C_gemm.sparse_mm_dense_cusparse(sparse.t(), grad_output, grad_dense)
            # grad_dense = sparse.t().mm(grad_output)
        return grad_sparse, grad_dense

def sparse_mm_dense(sparse: torch.Tensor, dense: torch.Tensor):
    '''
    * :ref:`API in English <sparse_mm_dense-en>`

    .. _sparse_mm_dense-cn:

    :param sparse: 稀疏2D tensor
    :type sparse: torch.Tensor
    :param dense: 稠密2D tensor
    :type dense: torch.Tensor
    :return: sparse 和 dense 的矩阵乘
    :rtype: torch.Tensor

    对输入的稀疏的二维矩阵 ``sparse`` 和稠密的二维矩阵 ``dense`` 进行矩阵乘法。

    .. warning::

        代码内部的实现方式是，首先将 ``sparse`` 转换为稀疏矩阵格式，然后再调用相关库进行运算。如果 ``sparse`` 不够稀疏，则该函数的速度会比普通矩阵乘法 ``torch.mm`` 慢很多。

    .. warning::

        稀疏矩阵的乘法存在一定的计算误差，但误差并不显著，或可忽略。

    .. warning::

        本函数不支持CPU。

    * :ref:`中文API <sparse_mm_dense-cn>`

    .. _sparse_mm_dense-en:

    :param sparse: a 2D sparse tensor
    :type sparse: torch.Tensor
    :param dense: a 2D dense tensor
    :type dense: torch.Tensor
    :return: a matrix multiplication of the matrices ``dense`` and ``sparse``
    :rtype: torch.Tensor

    Performs a matrix multiplication of the matrices ``dense`` and ``sparse``.

    .. admonition:: Warning
        :class: warning

        This function is implemented by converting ``sparse`` to a sparse format and doing a sparse matrix multiplication. If the sparsity of ``sparse`` is not high enough, the speed of this function will be slower than ``torch.mm``.

    .. admonition:: Warning
        :class: warning

        There are some numeral errors when doing the sparse matrix multiplication. But the errors are not significant.

    .. admonition:: Warning
        :class: warning

        This function does not support to run on cpu.
    '''
    return sparse_mm_dense_atf.apply(sparse, dense)


#!/usr/bin/python
# encoding:utf-8
"""
@author: lance
@version: 1.0.0
@license: Apache Licence
@file: bert_exporter.py
@time: 2021/6/9 20:52
"""
from importlib import import_module

import torch

from utils import build_iterator

PAD, CLS = '[PAD]', '[CLS]'  # padding符号, bert中综合信息符号

dataset = 'data'
x = import_module('models.bert')
config = x.Config(dataset)
model = x.Model(config).to(config.device)
model.load_state_dict(torch.load('data/saved_dict/bert_2021_4_15_cpu.ckpt'))

def read(s):
    pad_size = 32
    contents = []

    token = config.tokenizer.tokenize(s.strip())
    token = [CLS] + token
    seq_len = len(token)
    mask = []
    token_ids = config.tokenizer.convert_tokens_to_ids(token)

    if pad_size:
        if len(token) < pad_size:
            mask = [1] * len(token_ids) + [0] * (pad_size - len(token))
            token_ids += ([0] * (pad_size - len(token)))
        else:
            mask = [1] * pad_size
            token_ids = token_ids[:pad_size]
            seq_len = pad_size
    contents.append((token_ids, 0, seq_len, mask))
    return contents

def predict(data):
    data_ = build_iterator(data, config)
    for texts, labels in data_:
        outputs = model(texts)
    predict_ = torch.max(outputs.data, 1)[1].cpu().numpy()
    return predict_

if __name__ == '__main__':
    while True:
        print("neg" if predict(read(input())) else "pos")


import redis
import hashlib
import time

pool = redis.ConnectionPool(host='localhost', port=6379)
redis_connection = redis.Redis(connection_pool=pool)

SESSION_EXPIRE = 600

class RedisSession:

    def __init__(self, session_id):
        if session_id and redis_connection.exists(session_id):
            self.session_id = session_id
        else:
            self.session_id = self._generate_random_str()
            redis_connection.hset(self.session_id, '', '')
            redis_connection.expire(self.session_id, SESSION_EXPIRE)

    def _generate_random_str(self):
        md5_obj = hashlib.md5()
        md5_obj.update(bytes(str(time.time()), encoding='utf-8'))
        return md5_obj.hexdigest()

    def __setitem__(self, key, value):
        redis_connection.hset(self.session_id, key, value)
        redis_connection.expire(self.session_id, SESSION_EXPIRE)

    def __getitem__(self, item):
        return redis_connection.hget(self.session_id, item)

    def __delitem__(self, key):
        redis_connection.hdel(self.session_id, key)
        redis_connection.expire(self.session_id, SESSION_EXPIRE)


import stringcase
import giturlparse

from jinja2.ext import Extension

def as_hostname(value):
    # Replace underscores with dash - to allow using the value as a hostname
    return value.replace("_", "-")

def snake_to_pascal_case(value):
    return stringcase.pascalcase(value)

def as_git_path(value):
    if not value or not value.strip():
        return value

    res = value.strip()

    # value is already a git ssh url. Return it as is
    if value.startswith("git@"):
        return value

    res = res.replace("http://", "")
    res = res.replace("https://", "")

    # Strip .git suffix from a git http/https url
    if res.endswith(".git"):
        res = res[:-4]

    domain, rest = res.split("/", maxsplit=1)

    return f"git@{domain}:{rest}.git"

def get_url_path(value):
    """Takes a gitlab repository url and returns its path component

    Example:
    >>> get_url_path("https://gitlab.com/thorgate-public/django-project-template")
    >>> "thorgate-public/django-project-template"
    """
    if not value or not value.strip():
        return value

    res = value.strip()

    if res.startswith("git@") or res.endswith(".git"):
        try:
            if not res.startswith("git@") and not res.startswith("http"):
                # add https:// prefix to make giturlparse happy with the uris
                res = f"https://{res}"

            parts = giturlparse.parse(res)

            return "/".join(
                x
                for x in [
                    parts.owner,
                    parts.name,
                ]
                if x
            )

        except giturlparse.ParserError:
            pass

    res = res.replace("http://", "")
    res = res.replace("https://", "")

    domain, rest = res.split("/", maxsplit=1)

    return f"{rest}"

class TGFiltersModule(Extension):
    def __init__(self, environment):
        super().__init__(environment)

        environment.filters["as_hostname"] = as_hostname
        environment.filters["snake_to_pascal_case"] = snake_to_pascal_case
        environment.filters["as_git_path"] = as_git_path
        environment.filters["get_url_path"] = get_url_path

        environment.globals.update(
            SPA="spa",
            WEBAPP="webapp",
            ALPINE="alpine",
            DEBIAN="debian",
            YES="yes",
            NO="no",
            S3="S3",
            GCS="GCS",
        )


import curses
import curses.textpad
import random
import time

board = [' '] * 10
# board[0] = "X"

z = 0  # for the 'playersymbol' function

def test(i, screen, dims):  # alternates between 'X' and 'O'
    global z
    if z == 0:
        s = 'X'
        board[i] = s
        z += 1
        # fills the list of moves with the players' corresponding symbols
        return "X"
    else:
        s = "O"
        board[i] = s
        z -= 1
        return "O"

def get_computer_move(screen, dims):
    i = random.randrange(1, 10)
    if i == 7:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6), int(dims[1]/6), test(i, screen, dims))
    elif i == 8:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6), int(dims[1]/2), test(i, screen, dims))
    elif i == 9:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6), int(dims[1]/6*5), test(i, screen, dims))
    elif i == 4:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*3), int(dims[1]/6), test(i, screen, dims))
    elif i == 5:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*3), int(dims[1]/2), test(i, screen, dims))
    elif i == 6:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*3), int(dims[1]/6*5), test(i, screen, dims))
    elif i == 1:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*5), int(dims[1]/6), test(i, screen, dims))
    elif i == 2:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*5), int(dims[1]/2), test(i, screen, dims))
    elif i == 3:
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*5), int(dims[1]/6*5), test(i, screen, dims))
    return()

def rewrite(i):  # to avoid overwriting existing steps
    global board
    global write
    global c
    if board[i] == " ":
        write = True
    else:
        write = False

def tie(screen, dims):
    check = 0
    for r in range(0, 10):
        if board[r] == "X":
            check += 1
    return(check > 4)

def getPlayerMove(c, screen, dims):
    # prints the players' symbols on the space associated with the num keys
    if c == ord('7'):
        i = 7
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6), int(dims[1]/6), test(i, screen, dims))
    elif c == ord('8'):
        i = 8
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6), int(dims[1]/2), test(i, screen, dims))
    elif c == ord('9'):
        i = 9
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6), int(dims[1]/6*5), test(i, screen, dims))
    elif c == ord('4'):
        i = 4
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*3), int(dims[1]/6), test(i, screen, dims))
    elif c == ord('5'):
        i = 5
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*3), int(dims[1]/2), test(i, screen, dims))
    elif c == ord('6'):
        i = 6
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*3), int(dims[1]/6*5), test(i, screen, dims))
    elif c == ord('1'):
        i = 1
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*5), int(dims[1]/6), test(i, screen, dims))
    elif c == ord('2'):
        i = 2
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*5), int(dims[1]/2), test(i, screen, dims))
    elif c == ord('3'):
        i = 3
        rewrite(i)
        if write:
            screen.addstr(int(dims[0]/6*5), int(dims[1]/6*5), test(i, screen, dims))
    return()

player1 = "X"
player2 = "O"

def winner1():  # returns true if player1 wins

    global player1
    global board
    return ((board[7] == player1 and board[8] == player1 and board[9] == player1) or  # horizontal
            (board[4] == player1 and board[5] == player1 and board[6] == player1) or  # horizontal
            (board[1] == player1 and board[2] == player1 and board[3] == player1) or  # horizontal
            (board[7] == player1 and board[4] == player1 and board[1] == player1) or  # vertical
            (board[8] == player1 and board[5] == player1 and board[2] == player1) or  # vertical
            (board[9] == player1 and board[6] == player1 and board[3] == player1) or  # vertical
            (board[7] == player1 and board[5] == player1 and board[3] == player1) or  # diagonal
            (board[9] == player1 and board[5] == player1 and board[1] == player1))  # diagonal

def winner2():  # returns true if player2 wins
    global player2
    global board
    return ((board[7] == player2 and board[8] == player2 and board[9] == player2) or  # horizontal
            (board[4] == player2 and board[5] == player2 and board[6] == player2) or  # horizontal
            (board[1] == player2 and board[2] == player2 and board[3] == player2) or  # horizontal
            (board[7] == player2 and board[4] == player2 and board[1] == player2) or  # vertical
            (board[8] == player2 and board[5] == player2 and board[2] == player2) or  # vertical
            (board[9] == player2 and board[6] == player2 and board[3] == player2) or  # vertical
            (board[7] == player2 and board[5] == player2 and board[3] == player2) or  # diagonal
            (board[9] == player2 and board[5] == player2 and board[1] == player2))  # diagonal

def winning(screen, dims, name1, name2):  # whether one of the players won, prints this text
    if winner1():
        screen.addstr(int(dims[0]/2-2), int(dims[1]/2-7), name1 + " won!")  # X
    elif winner2():
        screen.addstr(int(dims[0]/2-2), int(dims[1]/2-7), name2 + " won!")  # O
    return()

def one_player(screen, name1, name2):
    global z
    dims = screen.getmaxyx()
    screen = curses.newwin(int(dims[0]), int(dims[1]), 0, 0)
    screen.box()
    for i in range(0, int(dims[0])):
        screen.addstr(i, int(dims[1]/3), '|')  # vertical
        screen.addstr(i, int(dims[1]/3*2), '|')
    for i in range(0, int(dims[1])):
        screen.addstr(int(dims[0]/3), i, '_')  # horizontal
        screen.addstr(int(dims[0]/3*2), i, '_')
    screen.addstr(0, 1, "TIC-TAC-TOE ------ Use number keys to place 'X' and 'O', press 'q' to quit.")
    curses.noecho()
    curses.cbreak()
    screen.keypad(True)
    curses.curs_set(0)

    while True:
        if winner1() or winner2():  # if one of the players won
            winning(screen, dims, name1, name2)
            screen.refresh()
            c = screen.getch()
            if c == ord('q'):  # if 'q' pressed the game exits
                break  # Exit the while loop
        elif tie(screen, dims):
            screen.addstr(int(dims[0]/2-2), int(dims[1]/2-7), "It's tie!")
            c = screen.getch()
            if c == ord('q'):  # if 'q' pressed the game exits
                break  # Exit the while loop
        elif z == 0:
            c = screen.getch()
            if c == ord('q'):  # if 'q' pressed the game exits
                break  # Exit the while loop
            elif z == 0:  # if 'q' is not pressed the game continues
                getPlayerMove(c, screen, dims)
                screen.refresh()
        else:
            get_computer_move(screen, dims)
            time.sleep(1)
    curses.endwin()

def multi_player(screen, name1, name2):
    global z
    dims = screen.getmaxyx()
    screen = curses.newwin(int(dims[0]), int(dims[1]), 0, 0)
    screen.box()
    for i in range(0, int(dims[0])):
        screen.addstr(i, int(dims[1]/3), '|')  # vertical
        screen.addstr(i, int(dims[1]/3*2), '|')
    for i in range(0, int(dims[1])):
        screen.addstr(int(dims[0]/3), i, '_')  # horizontal
        screen.addstr(int(dims[0]/3*2), i, '_')
    screen.addstr(0, 1, "TIC-TAC-TOE ------ Use number keys to place 'X' and 'O', press 'q' to quit.")
    curses.noecho()
    curses.cbreak()
    screen.keypad(True)
    curses.curs_set(0)

    while True:
        if winner1() or winner2():  # if one of the players won
            winning(screen, dims, name1, name2)
            screen.refresh()
            c = screen.getch()
            if c == ord('q'):  # if 'q' pressed the game exits
                break  # Exit the while loop
        elif tie(screen, dims):
            screen.addstr(int(dims[0]/2-2), int(dims[1]/2-7), "It's tie!")
            c = screen.getch()
            if c == ord('q'):  # if 'q' pressed the game exits
                break  # Exit the while loop
        else:
            c = screen.getch()
            if c == ord('q'):  # if 'q' pressed the game exits
                break  # Exit the while loop
            else:  # if 'q' is not pressed the game continues
                getPlayerMove(c, screen, dims)
                screen.refresh()
    curses.endwin()

def menu(screen, name1, name2, multi_single):
    dims = screen.getmaxyx()
    screen = curses.newwin(int(dims[0]), int(dims[1]), 0, 0)
    screen.box()
    screen.addstr(0, int(dims[1]/2-4), "MENU")
    screen.addstr(5, 3, 'Welcome ' + name1 + " and " + name2 + ' in this TicTacToe Game!')
    screen.addstr(10, 3, 'If you need help, press "h" or press "s" to start a game!')
    while True:
        c = screen.getch()
        screen.refresh()
        if c == ord('h'):
            screen.addstr(15, 3, 'You can navigate the symbols with num keys.'
                          ' Goal is collecting 3 same symbols next to each other.')
        elif c == ord('s'):
            if multi_single == "1":
                one_player(screen, name1, name2)
            else:
                multi_player(screen, name1, name2)
            break
        elif c == ord('q'):  # if 'q' pressed the game exits
            break
        else:
            screen.addstr(12, 3, 'Please chose from "h" to help or "s" to start!')
            screen.refresh()
    curses.endwin()
    exit()

def main():

    name1 = input('Type your name! ')
    name2 = "Computer"
    multi_single = input(name1 + " ,if you want to play against computer press 1 or play multi press 2!")
    while True:
        if multi_single == "1":
            screen = curses.initscr()
            menu(screen, name1, name2, multi_single)
        elif multi_single == "2":
            name2 = input('Type your friend name! ')
            screen = curses.initscr()
            menu(screen, name1, name2, multi_single)
        else:
            multi_single = input(name1 + " ,if you want to play against computer press 1 or play multi press 2!")

main()


import importlib
import os
from pathlib import Path
from unittest import mock

import setuptools
from google.cloud import storage
from google.auth import compute_engine

import boto3
from botocore.exceptions import ClientError
from com.griddynamics.dsl.ml.settings.description import Platform

class Helper:

    @staticmethod
    def get_file(file):
        if file:
            f = open(file, "rb")
            return f, Helper.get_file_name(file)

    @staticmethod
    def get_file_name(path):
        return os.path.basename(path)

    @staticmethod
    def get_setup_params(setup_path):
        spec = importlib.util.spec_from_file_location("setup", setup_path)
        setup = importlib.util.module_from_spec(spec)

        with mock.patch.object(setuptools, 'setup') as mock_setup:
            spec.loader.exec_module(setup)  # This is setup.py which calls setuptools.setup

        # called arguments are in `mock_setup.call_args`
        args, kwargs = mock_setup.call_args
        return kwargs

    @staticmethod
    def build_package_name_from_params(kwargs, extension):
        return f"{kwargs['name']}-{kwargs['version']}.{extension}"

    @staticmethod
    def construct_path(filename: str, base_path, platform=Platform.GCP):
        if platform ==Platform.GCP:
            return filename if filename.startswith('gs://') else Path(base_path) / filename
        else:
            return filename if filename.startswith('s3://') else Path(base_path) / filename

class GCPHelper(Helper):

    @staticmethod
    def delete_path_from_storage(bucket_name, path, use_cloud_engine_credentials=False):
        credentials = None
        if use_cloud_engine_credentials:
            credentials = compute_engine.Credentials()

        storage_client = storage.Client(credentials=credentials)
        bucket = storage_client.get_bucket(bucket_name)
        blobs = bucket.list_blobs(prefix=path)
        for blob in blobs:
            blob.delete()

    @staticmethod
    def copy_folder_on_storage(bucket_name, path_from: str, path_to: str, use_cloud_engine_credentials=False):
        path_from = path_from.replace(f'gs://{bucket_name}/', "")
        path_to = path_to.replace(f'gs://{bucket_name}/', "")
        credentials = None
        if use_cloud_engine_credentials:
            credentials = compute_engine.Credentials()

        storage_client = storage.Client(credentials=credentials)
        bucket = storage_client.get_bucket(bucket_name)
        blobs = bucket.list_blobs(prefix=path_from)
        for blob in blobs:
            bucket.copy_blob(blob, bucket, path_to + blob.name.replace(path_from, path_to))

    @staticmethod
    def download_folder_from_storage(bucket_name, path_from, path_to, use_cloud_engine_credentials=False):
        path_from = path_from.replace(f'gs://{bucket_name}/', "")
        credentials = None
        if use_cloud_engine_credentials:
            credentials = compute_engine.Credentials()
        storage_client = storage.Client(credentials=credentials)
        bucket = storage_client.get_bucket(bucket_name)
        blobs = bucket.list_blobs(prefix=path_from, delimiter="/")

        for blob in blobs:
            destination_uri = f'{path_to} / {blob.name}'
            blob.download_to_filename(destination_uri)

    @staticmethod
    def upload_file_to_storage(project_id, bucket, file_path: str, gs_path, use_cloud_engine_credentials=False):
        gs_path = gs_path.replace(f'gs://{bucket}/', "")
        credentials = None
        if use_cloud_engine_credentials:
            credentials = compute_engine.Credentials()
        client = storage.Client(project=project_id, credentials=credentials)
        bucket = client.get_bucket(bucket)
        blob = bucket.blob('{}/{}'.format(gs_path, file_path.split('/')[-1]))
        blob.upload_from_filename(file_path)

    @staticmethod
    def copy_file_on_storage(bucket_name, blob_name, new_blob_name, new_bucket_name=None,
                             use_cloud_engine_credentials=False):
        """Copies a blob from one bucket to another with a new name."""
        if new_bucket_name is None:
            new_bucket_name = bucket_name

        credentials = None
        if use_cloud_engine_credentials:
            credentials = compute_engine.Credentials()
        storage_client = storage.Client(credentials=credentials)
        source_bucket = storage_client.get_bucket(bucket_name)
        source_blob = source_bucket.blob(blob_name.replace(f'gs://{bucket_name}/', ""))
        destination_bucket = storage_client.get_bucket(new_bucket_name)

        source_bucket.copy_blob(
            source_blob, destination_bucket, new_blob_name.replace(f'gs://{new_bucket_name}/', ""))

class AWSHelper(Helper):

    @staticmethod
    def delete_path_from_storage(bucket_name, path):
        s3 = boto3.resource('s3')
        bucket = s3.Bucket(bucket_name)
        bucket.objects.filter(Prefix=path).delete()

    @staticmethod
    def copy_folder_on_storage(bucket_name, path_from: str, path_to: str):
        s3 = boto3.client("s3")
        objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=path_from)['Contents']
        for obj in objects:
            dest = obj['Key'].replace(path_from, path_to)
            copy_source = {'Bucket': bucket_name, 'Key': obj['Key']}
            s3.copy_object(CopySource=copy_source, Bucket=bucket_name, Key=dest)

    @staticmethod
    def download_folder_from_storage(bucket_name, path_from, path_to):
        s3 = boto3.client("s3")
        objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=path_from)['Contents']
        for obj in objects:
            dest = obj['Key'].replace(path_from, path_to)
            if not os.path.exists(os.path.dirname(dest)):
                os.makedirs(os.path.dirname(dest))
            try:
                s3.download_file(bucket_name, obj['Key'], dest)
            except IsADirectoryError:
                pass

    @staticmethod
    def upload_file_to_storage(bucket, file_name, object_name):
        if object_name is None:
            object_name = file_name
        # Upload the file
        s3_client = boto3.client('s3')
        try:
            s3_client.upload_file(file_name, bucket, object_name)
        except ClientError as e:
            raise e

    @staticmethod
    def copy_object(source_bucket, source_object_name, dest_bucket=None, dest_object_name=None):
        copy_source = {'Bucket': source_bucket, 'Key': source_object_name}
        if dest_object_name is None:
            dest_object_name = source_object_name
        if dest_bucket is None:
            dest_bucket = source_bucket
        # Copy the object
        s3 = boto3.client('s3')
        try:
            s3.copy_object(CopySource=copy_source, Bucket=dest_bucket, Key=dest_object_name)
        except ClientError as e:
            raise e

    @staticmethod
    def upload_object_to_storage(obj, bucket, object_name):
        client = boto3.client('s3')
        client.put_object(Body=obj, Bucket=bucket, Key=object_name)


from django.urls import path
from . import views, api

app_name = 'sales'

urlpatterns = [
    path('', views.index, name='site_index'),
    path('add', views.create_view, name='site_index'),
    path('register', views.register, name='site_index'),
    path('login', views.login, name='login'),
    path('logout', views.logout, name='logout'),
    path('list', views.list_view, name='Cars'),
    path('particular/<id>', views.detail_view, name='particular'),
    path('update/<id>', views.update_view, name='particular'),
    path('delete/<id>', views.delete_view, name='delete'),
    ]