#!/usr/bin/env python

"""Display a subsection of input, starting with a matching pattern.

With no input file, or when file is -, read standard input.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import click
import gzip
import bz2
import re
import signal
import sys

@click.command()
@click.argument('pattern')
@click.argument('filenames', nargs=-1,
                type=click.Path(exists=True, dir_okay=False))
@click.option('-F', '--fixed-string',
                    default=False, is_flag=True,
                    help='fixed string match (default is regex)')
@click.option('-i', '--ignore-case',
                    default=False, is_flag=True,
                    help='ignore case distinctions')
@click.option('-m', '--max-count',
                    default=1,
                    help='start after number of matches.')
@click.option('-n', '--line-number',
                    default=False, is_flag=True,
                    help='print line number with output lines.')
@click.option('-x', '--exclude-match',
                    default=False, is_flag=True,
                    help='exclude matching line.')
def main(pattern, filenames, fixed_string, ignore_case,
         exclude_match, line_number, max_count):
    """Display input starting with a line matching PATTERN."""
    global exit_code
    exit_code = 1
    is_match = pattern_matcher(pattern, fixed_string, ignore_case)
    lines = text_file_input(filenames)
    if line_number:
        lines = (':'.join((str(i), l)) for i, l in enumerate(lines, 1))
    selected_lines = fromthis(is_match, lines, exclude_match, max_count)
    for line in selected_lines:
        print(line)
    return exit_code

def fromthis(is_match, input_stream, exclude_match, max_count):
    """Display input starting with a matching pattern."""
    global exit_code
    for line in input_stream:
        if is_match(line, max_count):
            exit_code = 0
            if not exclude_match:
                yield line
            break
    for line in input_stream:
        yield line

def count_matches(match_function):
    """Return True only if the required number of matches is reached."""
    def counter(line, max_count, persistent={'count': 0}):
        result = match_function(line)
        if result:
            persistent['count'] += 1
        return True if persistent['count'] == max_count else False
    return counter

def pattern_matcher(pattern, fixed_string, ignore_case):
    """Return a custom function for pattern matching."""
    regex_flags = re.IGNORECASE if ignore_case else 0

    @count_matches
    def match_regex(line, regex=re.compile(pattern, regex_flags)):
        return bool(regex.search(line))

    @count_matches
    def match_fixed_string_ignore_case(line, pattern=pattern):
        return pattern.upper() in line.upper()

    @count_matches
    def match_fixed_string_match_case(line, pattern=pattern):
        return pattern in line

    if not fixed_string:
        match_function = match_regex
    elif ignore_case:
        match_function = match_fixed_string_ignore_case
    else:
        match_function = match_fixed_string_match_case
    return match_function

def text_file_input(filenames):
    """Generate input from files or standard input."""
    if filenames:
        file_handles = open_files(filenames)
    else:
        file_handles = (iter(sys.stdin.readline, ''), )
    for file_handle in file_handles:
        for line in file_handle:
            yield line.rstrip()

def open_files(filenames):
    """Return a filehandle for each provided filename."""
    for filename in filenames:
        file_open = file_opener(filename)
        with file_open(filename, 'rt') as filehandle:
            yield filehandle

def file_opener(filename):
    """Return a file open function based on file extension."""
    if filename.endswith('.gz'):
        return gzip.open
    if filename.endswith('.bz2'):
        return bz2.open
    else:
        return open

if __name__ == '__main__':
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)  # IOError: Broken pipe
    signal.signal(signal.SIGINT, signal.SIG_DFL)  # KeyboardInterrupt: Ctrl-C
    sys.exit(main())


import numpy as np
from typing import Tuple
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont

from definitions import ROOT_DIR

# ==========================================================
#                        ROCKSAMPLE
# ==========================================================

def create_circular_mask(h: int, w: int, center: Tuple[int, int] = None, radius: int = None):
    if center is None: # use the middle of the image
        center = (int(w/2), int(h/2))
    if radius is None: # use the smallest distance between the center and image walls
        radius = min(center[0], center[1], w-center[0], h-center[1])

    Y, X = np.ogrid[:h, :w]
    dist_from_center = np.sqrt((X - center[0])**2 + (Y-center[1])**2)

    mask = dist_from_center <= radius
    return mask

def create_rocksample_agent(h: int, w: int, thickness: int = 3, radius: int = None):
    grid = create_circular_mask(h, w, radius=radius)
    grid_inner = create_circular_mask(h, w, radius=radius - thickness)
    return np.bitwise_xor(grid, grid_inner)

def generate_rock_agent_rgb(size: int, agent_color: np.ndarray) -> np.ndarray:
    agent_mask = create_rocksample_agent(size, size, radius=size // 3)
    rgb = np.repeat(agent_mask[..., np.newaxis], 3, axis=-1)
    rgb = np.ones_like(rgb) * 255
    rgb[agent_mask.astype(bool)] = agent_color
    return rgb

def create_rectangle(h: int, w: int, thickness: int = 2, length: int = None, width: int = None):
    grid = np.zeros((h, w))
    if length is None:
        length = h - 2
    if width is None:
        width = w - 2

    assert h > length and w > width
    h_space = (h - length) // 2
    w_space = (w - width) // 2

    # East/West
    w_range_pos = np.arange(thickness) + w_space
    w_range = np.concatenate([w_range_pos, -(w_range_pos + 1)])
    grid[h_space:-h_space, w_range] = 1

    # North/South
    h_range_pos = np.arange(thickness) + h_space
    h_range = np.concatenate([h_range_pos, -(h_range_pos + 1)])
    grid[h_range, w_space:-w_space] = 1
    return grid

def generate_rock_rgb(size: int, rock_color: np.ndarray, weight: float = None,
                      background_color: np.ndarray = None):
    if weight is not None and background_color is None:
        w_p = int(weight * 155) + 30 if weight > 0 else 0
        background_color = np.array([255 - w_p, 255 - w_p, 255])
    l = size - size // 4
    rock_mask = create_rectangle(size, size, thickness=5, length=l, width=l)
    rgb = np.repeat(rock_mask[..., np.newaxis], 3, axis=-1)
    if weight is not None:
        rgb[:, :] = background_color
    else:
        rgb = np.ones_like(rgb) * 255
    rgb[rock_mask.astype(bool)] = rock_color
    return rgb

def generate_label(h: int, w: int, str_label: str,
                   font_size: int = 12):
    """
    Generate an h x w x 3 RGB label with str_label in the centre printed.
    :param h: height
    :param w: width
    :param str_label: string to label
    :param color: color for font
    :param font_size: what size font do we use
    :return:
    """
    # Start with an all white label
    label = Image.new('RGB', (w, h), (255, 255, 255))
    d_actual = ImageDraw.Draw(label)
    # font = ImageFont.truetype("FreeMono.ttf", font_size)
    font_path = Path(ROOT_DIR, "scripts", "FreeMono.ttf")
    font = ImageFont.truetype(str(font_path), font_size)

    img_to_guide = Image.new('RGB', (w, h), (255, 255, 255))
    d = ImageDraw.Draw(img_to_guide)
    d.text((0, 0), str_label, (0, 0, 0), font=font)

    text_w, text_h = d.textsize(str_label, font)
    offset_x, offset_y = font.getoffset(str_label)
    text_w += offset_x
    text_h += offset_y

    pos = ((w - text_w) // 2, (h - text_h) // 2)

    d_actual.text(pos, str_label, fill=(0, 0, 0), font=font)
    return np.array(label)

def rocksample_arr_to_viz(arr: np.ndarray, scale: int = 10, grid_lines: bool = True,
                          background_weights: np.ndarray = None, greedy_actions: np.ndarray = None) -> np.ndarray:
    """
    Make a pixel representation of rock sample state/array
    :param arr: array representation of the environment
    :param scale: How large do we scale up?
    :param grid_lines: Do we show grid lines?
    :param background_weights: What are the weights for each grid?
    :param greedy_actions: Do we show optimal actions for each position?
    :return: array to plot
    """
    space_color = np.array([255, 255, 255], dtype=np.uint8)
    rock_color = np.array([255, 167, 0], dtype=np.uint8)
    goal_color = np.array([0, 150, 0])
    agent_color = np.array([0, 0, 0], dtype=np.uint8)
    grid_color = None

    size = arr.shape[0] * scale

    if grid_lines:
        size += arr.shape[0] + 1
        grid_color = np.array([150, 150, 150], dtype=np.uint8)

    final_viz_array = np.zeros((size, size, 3), dtype=np.uint8)

    if grid_lines:
        final_viz_array[::(scale + 1)] = grid_color
        final_viz_array[:, ::(scale + 1)] = grid_color

    for y, row in enumerate(arr):
        for x, val in enumerate(row):
            if val == 1:
                # AGENT
                to_fill = generate_rock_agent_rgb(scale, agent_color)
            elif val == 2:
                # ROCK
                to_fill = generate_rock_rgb(scale, rock_color, weight=background_weights[y, x] if background_weights is not None else None)
            elif val == 3:
                # AGENT + ROCK
                background = generate_rock_rgb(scale, rock_color, weight=background_weights[y, x] if background_weights is not None else None)
                agent = generate_rock_agent_rgb(scale, agent_color)
                background[(agent != 255).astype(bool)] = agent[(agent != 255).astype(bool)]
                to_fill = background
            elif val == 4:
                to_fill = np.zeros((scale, scale, 3))
                to_fill[:, :] = np.copy(goal_color)
            else:
                to_fill = np.zeros((scale, scale, 3))
                to_fill[:, :] = np.copy(space_color)

            if greedy_actions is not None and y < greedy_actions.shape[0] and x < greedy_actions.shape[1]:
                action_str = greedy_actions[y, x]
                label = generate_label(scale, scale, action_str)
                to_fill[(label != 255).astype(bool)] = label[(label != 255).astype(bool)]

            if grid_lines:
                final_viz_array[y * (scale + 1) + 1:(y + 1) * (scale + 1),
                x * (scale + 1) + 1:(x + 1) * (scale + 1)] = to_fill
            else:
                final_viz_array[y * scale:(y + 1) * scale,
                x * scale:(x + 1) * scale] = to_fill

    return final_viz_array

def generate_greedy_action_array(env, agent):
    """
    NOTE: for ROCKSAMPLE only.
    :return:
    """
    all_pos_states = env.sample_all_states()
    obses = []
    for state in all_pos_states:
        obses.append(env.get_obs(state))

    obses = np.stack(obses)
    qs = agent.Qs(obses, agent.network_params)
    actions = np.argmax(qs, axis=1)

    arr = np.zeros((env.size, env.size - 1), dtype=np.int16)
    for act, state in zip(actions, all_pos_states):
        pos = state[:2]
        arr[pos[0], pos[1]] = int(act)

    return arr


import cv2
import os
import sys
import numpy as np
class dark_detection:
    def __init__(self, car):
        self.car = car
        self.plate = None
        self.probability_minimum = 0.3
        self.threshold = 0.3

    def app_path(self):
        """Returns the base application path."""
        if hasattr(sys, 'frozen'):
            # Handles PyInstaller
            return os.path.dirname(sys.executable)  # 使用pyinstaller打包后的exe目录
        return os.path.dirname(__file__)  # 没打包前的py目录

    def darknet_detection(self):
        #self.car = cv2.resize(self.car, (600, 425))  # 尺寸300x225
        self.weights_file = self.app_path() + r'/darknet_model/carplate.weights'
        self.classes_file = self.app_path() + r'/darknet_model/classes.names'
        self.cfg_file = self.app_path() + r'/darknet_model/darknet-yolov3.cfg'
        self.weights_isfile = os.path.exists(self.weights_file)
        self.classes_isfile = os.path.exists(self.classes_file)
        self.cfg_isfile = os.path.exists(self.cfg_file)
        if  self.weights_isfile == False or self.classes_isfile == False or self.cfg_isfile == False:
            return
        network = cv2.dnn.readNetFromDarknet(self.cfg_file, self.weights_file)
        layers_names_all = network.getLayerNames()

        layers_names_output = [layers_names_all[i[0] - 1] for i in network.getUnconnectedOutLayers()]
        blob = cv2.dnn.blobFromImage(self.car, 1 / 255.0, (416, 416), swapRB=True, crop=False)
        blob_to_show = blob[0, :, :, :].transpose(1, 2, 0)
        network.setInput(blob)
        output_from_network = network.forward(layers_names_output)
        np.random.seed(42)
        labels = open( self.classes_file).read()
        #colours = np.random.randint(0, 255, size=(len(labels), 3), dtype='uint8')
        bounding_boxes = []
        confidences = []
        class_numbers = []
        h, w = self.car.shape[:2]

        for result in output_from_network:
            for detection in result:
                scores = detection[5:]
                class_current = np.argmax(scores)
                confidence_current = scores[class_current]
                if confidence_current > self.probability_minimum:
                    box_current = detection[0:4] * np.array([w, h, w, h])
                    x_center, y_center, box_width, box_height = box_current.astype('int')
                    x_min = int(x_center - (box_width / 2))
                    y_min = int(y_center - (box_height / 2))
                    bounding_boxes.append([x_min, y_min, int(box_width), int(box_height)])
                    confidences.append(float(confidence_current))
                    class_numbers.append(class_current)

        results = cv2.dnn.NMSBoxes(bounding_boxes, confidences, self.probability_minimum, self.threshold)
        if len(results) > 0:
            for i in results.flatten():
                x_min, y_min = bounding_boxes[i][0], bounding_boxes[i][1]
                box_width, box_height = bounding_boxes[i][2], bounding_boxes[i][3]
                #colour_box_current = [int(j) for j in colours[class_numbers[i]]]

                self.car = cv2.rectangle(self.car, (x_min-5, y_min-5), (x_min + box_width+5, y_min + box_height+5),
                              (0, 255, 0), 1)

               # text_box_current = '{}: {:.4f}'.format(labels[int(class_numbers[i])], confidences[i])
                #cv2.putText(image_input, text_box_current, (x_min, y_min - 7), cv2.FONT_HERSHEY_SIMPLEX,
                          #  1.5, colour_box_current, 5)
                self.plate = self.car[y_min:y_min + box_height, x_min:x_min + box_width]


import brainpy_datasets as bp_data
import matplotlib.pyplot as plt
import numpy as np

import brainpy as bp
import brainpy.math as bm

# data
ds = bp_data.cognitive.RatePerceptualDecisionMaking(
  dt=20.,
  t_fixation=lambda: np.random.choice((50, 100, 200, 400)),
  t_stimulus=lambda: np.random.choice((100, 200, 400, 800)),
  num_trial=64 * 100
)
loader = bp_data.cognitive.TaskLoader(ds,
                                      max_seq_len=100,
                                      batch_size=64,
                                      data_first_axis='T')

# EI RNN model
class EI_RNN(bp.DynamicalSystem):
  def __init__(
      self, num_input, num_hidden, num_output, dt,
      e_ratio=0.8, sigma_rec=0., seed=None,
      w_ir=bp.init.KaimingUniform(scale=1.),
      w_rr=bp.init.KaimingUniform(scale=1.),
      w_ro=bp.init.KaimingUniform(scale=1.)
  ):
    super().__init__()

    # parameters
    self.tau = 100
    self.num_input = num_input
    self.num_hidden = num_hidden
    self.num_output = num_output
    self.e_size = int(num_hidden * e_ratio)
    self.i_size = num_hidden - self.e_size
    self.alpha = dt / self.tau
    self.sigma_rec = (2 * self.alpha) ** 0.5 * sigma_rec  # Recurrent noise
    self.rng = bm.random.RandomState(seed=seed)

    # hidden mask
    mask = np.tile([1] * self.e_size + [-1] * self.i_size, (num_hidden, 1))
    np.fill_diagonal(mask, 0)
    self.mask = bm.asarray(mask, dtype=bm.float_)

    # input weight
    self.w_ir = bm.TrainVar(bp.init.parameter(w_ir, (num_input, num_hidden)))

    # recurrent weight
    bound = 1 / num_hidden ** 0.5
    self.w_rr = bm.TrainVar(bp.init.parameter(w_rr, (num_hidden, num_hidden)))
    self.w_rr[:, :self.e_size] /= (self.e_size / self.i_size)
    self.b_rr = bm.TrainVar(self.rng.uniform(-bound, bound, num_hidden))

    # readout weight
    bound = 1 / self.e_size ** 0.5
    self.w_ro = bm.TrainVar(bp.init.parameter(w_ro, (self.e_size, num_output)))
    self.b_ro = bm.TrainVar(self.rng.uniform(-bound, bound, num_output))

  def reset_state(self, batch_size):
    self.h = bm.Variable(bm.zeros((batch_size, self.num_hidden)), batch_axis=0)
    self.o = bm.Variable(bm.zeros((batch_size, self.num_output)), batch_axis=0)

  def cell(self, x, h):
    ins = x @ self.w_ir + h @ (bm.abs(self.w_rr) * self.mask) + self.b_rr
    state = h * (1 - self.alpha) + ins * self.alpha
    state += self.sigma_rec * self.rng.randn(self.num_hidden)
    return bm.relu(state)

  def readout(self, h):
    return h @ self.w_ro + self.b_ro

  def update(self, x):
    self.h.value = self.cell(x, self.h)
    self.o.value = self.readout(self.h[:, :self.e_size])
    return self.h.value, self.o.value

  def predict(self, xs):
    self.h[:] = 0.
    return bm.for_loop(self.update, xs)

  def loss(self, xs, ys):
    hs, os = self.predict(xs)
    l = bp.losses.cross_entropy_loss(os.reshape((-1, os.shape[-1])), ys.flatten())
    acc = bm.mean(bm.argmax(os, axis=-1) == ys)
    return l, acc

# Instantiate the network and print information
hidden_size = 50
net = EI_RNN(num_input=len(ds.input_features),
             num_hidden=hidden_size,
             num_output=len(ds.output_features),
             dt=ds.dt,
             sigma_rec=0.15)

# Adam optimizer
opt = bp.optim.Adam(lr=0.001, train_vars=net.train_vars().unique())

# gradient function
grad_f = bm.grad(net.loss,
                 grad_vars=net.train_vars().unique(),
                 return_value=True,
                 has_aux=True)

# training function
@bm.jit
def train(xs, ys):
  grads, loss, acc = grad_f(xs, ys)
  opt.update(grads)
  return loss, acc

# training
for epoch_i in range(30):
  losses = []
  accs = []
  for x, y in loader:
    net.reset_state(x.shape[1])
    l, a = train(x, y)
    losses.append(l)
    accs.append(a)
  print(f'Epoch {epoch_i}, loss {np.mean(losses)}, acc {np.mean(accs)}')

# testing
ds.t_fixation = 500.  # set the fixed time duration for fixation and stimulus
ds.t_stimulus = 500.
x, y = zip(*[ds[i] for i in range(50)])  # get 50 trials
x = np.asarray(x).transpose(1, 0, 2)
y = np.asarray(y).transpose(1, 0)
net.reset_state(x.shape[1])
rnn_activity, action_pred = net.predict(x)
rnn_activity = bm.as_numpy(rnn_activity)
choice = np.argmax(bm.as_numpy(action_pred[-1]), axis=1)
correct = choice == y[-1]
print('Average performance', np.mean(correct))

# plot activity
trial = 0
plt.figure(figsize=(8, 6))
_ = plt.plot(rnn_activity[:, trial, :net.e_size], color='blue', label='Excitatory')
_ = plt.plot(rnn_activity[:, trial, net.e_size:], color='red', label='Inhibitory')
plt.xlabel('Time step')
plt.ylabel('Activity')
plt.show()


import base64
import io
from PIL import Image
import pytesseract
import cv2
import numpy as np

def base64str_to_PILImage(base64str):
    """Convert a Base64 Image to a Pillow Image

    Args:
        base64str (str): Image in Base64 string

    Returns:
        Image: Pillow image (https://pillow.readthedocs.io/en/stable/reference/Image.html)
    """

    base64_img_bytes = base64str.encode('utf-8')
    base64bytes = base64.b64decode(base64_img_bytes)
    bytesObj = io.BytesIO(base64bytes)
    img = Image.open(bytesObj)
    return img

def PILImage_to_base64str(image):
    buffered = io.BytesIO()
    image.save(buffered, format="JPEG")
    img_str = base64.b64encode(buffered.getvalue())

    return img_str.decode()

def base64img_to_np_array(image: str):
    pillow_img = base64str_to_PILImage(image)
    return np.array(pillow_img)

def np_array_to_base64img(image):
    pillow_img = Image.fromarray(image)
    return PILImage_to_base64str(pillow_img)

def PILImage_to_np_array(image):
    # im is a PIL Image object
    im_arr = np.asarray(image)
    # convert rgb array to opencv's bgr format
    im_arr_bgr = cv2.cvtColor(im_arr, cv2.COLOR_RGB2BGR)
    return im_arr_bgr

def np_array_to_PILImage(image):
    # img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    im_pil = Image.fromarray(image)
    return im_pil

class OCRLine:
    def __init__(self, level, page_num, block_num, par_num, line_num, word_num, left, top, width, height, conf, text):
        self.level = level
        self.page_num = page_num
        self.block_num = block_num
        self.par_num = par_num
        self.line_num = line_num
        self.word_num = word_num
        self.left = left
        self.top = top
        self.width = width
        self.height = height
        self.conf = conf
        self.text = text

class OCRDataParser:
    def __init__(self):
        self.ocr_lines = []

    def parse(self, text: str, first_line_is_header: bool = True):
        lines = text.split('\n')
        for i, line in enumerate(lines):
            if i == 0 and first_line_is_header: continue
            data = line.split('\t')
            if len(data) == 12:
                level = data[0]
                page_num = data[1]
                block_num = data[2]
                par_num = data[3]
                line_num = data[4]
                word_num = data[5]
                left = data[6]
                top = data[7]
                width = data[8]
                height = data[9]
                conf = data[10]
                text = data[11]
                text = text.strip()

                if text != '':
                    ocr_line = OCRLine(level, page_num, block_num, par_num, line_num, word_num, left, top, width, height, conf, text)
                    self.ocr_lines.append(ocr_line)

    def get_coordenates_and_texts(self):
        items_list = []

        for line in self.ocr_lines:
            start_point = (int(line.left), int(line.top))
            end_point = (int(line.left) + int(line.width), int(line.top) + int(line.height))
            items_list.append((start_point, end_point, int(line.conf), line.text))

        return items_list

class OCRDetector:
    def __init__(self):
        pass

    def putText(self, text: str, location, fontScale = 3, color = (255,0,0), thickness = 3):
        cv2.putText(self.img, text, location, cv2.FONT_HERSHEY_COMPLEX, fontScale, color, thickness)

    def processed_img(self):
        return PILImage_to_base64str(self.img)

    def process_ocr(self, img_base64: str, confLevel: int):
        ocr_parser = OCRDataParser()
        self.texts = ""

        self.img = base64str_to_PILImage(img_base64)
        open_cv_img = base64img_to_np_array(img_base64)
        # print(pytesseract.image_to_string(self.img))
        # print(pytesseract.image_to_data(self.img))
        ocr_parser.parse(pytesseract.image_to_data(self.img))
        # print(ocr_parser.get_coordenates_and_texts())
        for item in ocr_parser.get_coordenates_and_texts():
            start_point = item[0]
            end_point = item[1]
            conf = item[2]
            text = item[3]

            if conf >= confLevel:
                cv2.rectangle(open_cv_img, start_point, end_point, (255,0,0), 3)
                self.texts = ''.join([self.texts, "|", text])

        self.img = np_array_to_PILImage(open_cv_img)


import frida

def on_message(message, data):
    print("[on_message] message:", message, "data:", data)

session = frida.attach("winmine.exe")

script = session.create_script("""
rpc.exports.test1 = function () {
    return Process.enumerateModules();
};
rpc.exports.test2 = function (message) {
    console.log("Test 2 From JS ,", message);
};
""")

script.on("message", on_message)
script.load()

# print([m["name"] for m in script.exports.enumerate_modules()])
for m in script.exports.test1():
    print(m)

script.exports.test2("123")


from functools import reduce

def knot_hash(input):
  processed_input = list(map(ord, input)) + [17, 31, 73, 47, 23]
  lengths = processed_input

  numbers = [x for x in range(256)]

  current_position = 0
  skip_size = 0

  for i in range(64):
    for length in lengths:
      if length <= len(numbers):
        tmp = {}
        start = current_position
        end = current_position + length - 1

        for i in range(length):
          tmp[(end-i) % len(numbers)] = numbers[(start+i) % len(numbers)]

        for i in range(len(numbers)):
          if i in tmp.keys():
            numbers[i] = tmp[i]

        current_position = (current_position + length + skip_size) % len(numbers)
        skip_size = skip_size + 1

  blocks = []
  for i in range(16):
    tmp = []
    for n in range(len(numbers)):
      if n // 16 == i:
        tmp.append(numbers[n])
    blocks.append(tmp)

  dense_hash = [reduce(lambda x, y: x ^ y, block) for block in blocks]

  # return ''.join(list(map(lambda x: format(x, '02x'), dense_hash)))
  return list(map(bin, dense_hash))

input = 'ffayrhll'

def get_disk_grid(input):
  disk_grid = []
  for i in range(128):
    tmp = []
    for j in range(128):
      tmp.append(0)
    disk_grid.append(tmp)

  for i in range(128):
    string_to_hash = input + '-' + str(i)
    current_hash = knot_hash(string_to_hash)
    current_hash = ''.join(map(lambda x: x[2:], current_hash))
    for j in range(len(current_hash)):
      disk_grid[i][j] = 1 * (current_hash[j] == '1')

  return disk_grid

print(sum(map(sum, get_disk_grid(input))))


#!/usr/bin/env python
import argparse
import os.path
import re
import shutil
import subprocess
import sys

def main():
    parser = argparse.ArgumentParser(prog="workflow_preparation")
    parser.add_argument("workflow_listing", help="A file of workflows to prepare")
    args = parser.parse_args()

    if not os.path.exists(args.workflow_listing):
        sys.exit(1)

    with open(args.workflow_listing) as f:
        workflows = f.readlines()

    current_dir = os.getcwd()
    if not os.path.exists('workflows'):
        os.mkdir('workflows')

    os.chdir('workflows')
    default_workflow_set = False
    for workflow_info in workflows:
        parts = workflow_info.split()
        if len(parts) == 2:
            url = parts[0]
            tag = parts[1]
            if not default_workflow_set:
                default_workflow_set = True
                with open('default_workflow.txt', 'w') as f:
                    f.write(os.path.basename(url))

            result = subprocess.run(["git", "-c", "advice.detachedHead=false", "clone", "--depth", "1", url, "-b", tag])
            print(' == result git:', result.returncode, flush=True)
            try:
                result.check_returncode()
            except subprocess.CalledProcessError as e:
                if e.returncode == 128:
                    print(' == skipping existing expecting version:', tag[1:], flush=True)
                else:
                    sys.exit(e.returncode)

    os.chdir(current_dir)
    if len(workflows):
        shutil.make_archive('internal_workflows', 'zip', './workflows')

if __name__ == "__main__":
    main()


import os
import pandas as pd
import numpy as np
import pickle
from imblearn.over_sampling import SMOTE
from sklearn.preprocessing import LabelEncoder
from collections import Counter
import typing as typing
from sklearn.impute import KNNImputer
from thyroid.utils.exception import customException
from thyroid.logging import logger
from thyroid.entity.config_entity import DataTransformationConfig
from thyroid.components.clustering import DataClustering
from pathlib import Path
from thyroid.config.configuration import ConfigurationManager

class DataTransformation:
    def __init__(self, config: DataTransformationConfig):
        self.config = config

    def convert_to_numerics(self,df):
        df[self.config.numerical_columns] = df[self.config.numerical_columns].apply(pd.to_numeric, errors='coerce')
        return(df)

    def case_normalization(self):
        """
        Method: this method will normalize all values to small case, will convert 'y' to 't' and 'n' to 'f'
        , it will replace '?' with nan and will convert columns to numerics
        Input: dataframe
        Outputs: normalized dataframe
        """

        file_path = self.config.merged_file
        df = pd.read_csv(os.path.join(self.config.validation_dir,file_path))

        # Convert all object columns to lowercase
        df = df.applymap(lambda x: x.lower() if isinstance(x, str) else x)

        # Replace '?' with NaN
        df.replace('?', np.nan, inplace=True)

        # Replace 'y' with 't' and 'n' with 'f'
        df.replace({'y': 't', 'n': 'f'}, inplace=True)

        for column in self.config.categorical_columns:
            df[column].replace({'0': 'f', '1': 't'}, inplace=True)

        # convert "age","TSH","T3","TT4","T4U","FTI" to numeric
        df = self.convert_to_numerics(df)

        return(df)

    def impute_column(self,column):
        imputer = KNNImputer(n_neighbors=3, weights="uniform", missing_values=np.nan)
        imputed_values = imputer.fit_transform(column.values.reshape(-1, 1))

        return np.round(imputed_values, 2)

    def handle_missing_values(self, df):
        """
        Method: This method will eliminate columns with more than 30% missing values, handle missing values and drop unnecessary columns
        Inputs: data frame with missing values
        Output: data frame with no missing values and useless columns
        """
        # Drop columns with more missing values than the threshold
        logger.info("Drop columns with more missing values than the threshold = {self.config.threshold}")
        column_threshold = int(self.config.threshold * len(df))
        df = df.dropna(axis=1, thresh=column_threshold)

        # Impute missing values
        logger.info("Impute missing values in numerical column")
        columns_to_impute = self.config.numerical_columns[1:]
        for column in columns_to_impute:
            df[column] = self.impute_column(df[column])

        # Mode Impute missing values for remaining categorical columns
        logger.info("Impute other columns missing values using mode")
        mode_columns = ['hypopituitary', 'I131_treatment', 'psych','sex']
        for column in mode_columns:
            mode_value = df[column].mode()[0]  # Get the mode value of the column
            df[column].fillna(mode_value, inplace=True)

        # median_imputation for age column
        logger.info("Impute age column using median")
        median_value = df['age'].median()  # Calculate the median value of the age column
        df['age'].fillna(median_value, inplace=True)

        # drop the measured columns
        df.drop(columns=self.config.drop_columns, inplace=True)

        return (df)

    def calculate_age_iqr(self,df):
        """_summary_

        Args:
            df (_type_): _description_

        Returns:
            _type_: _description_
        """

        logger.info("removing outliers from age column")
        age = self.config.numerical_columns[0]
        # Calculate IQR for the 'age' column
        Q1 = df[age].quantile(0.25)
        Q3 = df[age].quantile(0.75)
        IQR = Q3 - Q1

        # Define the upper and lower bounds for outlier removal
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR

        # Remove outliers from the 'age' column
        df = df[(df['age'] >= lower_bound) & (df['age'] <= upper_bound)]

        return (df)

    def calculate_z_score(self,df):
        # Define a threshold for identifying outliers (e.g., Z-score > 2)
        logger.info("removing outliers using z-score")
        # Iterate over the columns and remove outliers
        non_outliers_data_frame = df.copy()  # Create a copy to store non-outliers
        for column in self.config.numerical_columns[1:]:
            z_scores = np.abs((non_outliers_data_frame[column] - non_outliers_data_frame[column].mean()) / non_outliers_data_frame[column].std())
            non_outliers_data_frame = non_outliers_data_frame[(z_scores <= self.config.z_threshold)]

        return non_outliers_data_frame

    def no_duplicates_and_binary_convert(self, df):
        # map sex column values to 0 and 1
        logger.info("converting sex column -> 'f' to 0 and 'm' to 1")
        sex = self.config.categorical_columns_to_convert[0]
        df[sex] = df[sex].map({'f': 0, 'm': 1})

        # map all othere categorical function to 0 and 1
        logger.info("mapping 'f' to 0 and 't' to 1")
        for column in self.config.categorical_columns_to_convert[1:]:
            if len(df[column].unique()) == 2:
                df[column] = df[column].map({'f': 0, 't': 1})

        # remove duplicate rows
        df = df.drop_duplicates()
        logger.info("duplicate rows removed")

        return(df)

    def outlier_removal(self,df):
        """_summary_
        Method:
            remove outliers from numerical columns
        Args:
            df (_type_): _description_
        """
        logger.info("Outliers removal started")
        # compute IQR on age column to remove outliers
        df = self.calculate_age_iqr(df)
        logger.info("IQR on age column completed")
        # compute Z-score on remaining numerical columns to remove outliers
        df = self.calculate_z_score(df)
        logger.info("Z-score on remaining numerical columns")
        df = self.no_duplicates_and_binary_convert(df)
        logger.info("remvoed duplicates and converted to binary")

        return df

    def imbalance_handling(self, df):
        """_summary_

        Args:
            df (_type_): imbalanced data frame

        Returns:
            _type_: dataframe with balanced data points
        """
        X = df.drop(columns=['class'])
        y = df['class']

        prev_count = Counter(y)
        logger.info("Class distribution before SMOTE: {prev_count}")

        smote = SMOTE(sampling_strategy='auto', random_state=42)

        X_resampled, y_resampled = smote.fit_resample(X, y)

        resampled_count = Counter(y_resampled)
        logger.info("Class distribution after SMOTE: {resampled_count})} ")

        resampled_data = pd.concat([X_resampled, y_resampled], axis=1)

        return resampled_data

    def labelencoding_and_save(self, df):
        encode = LabelEncoder().fit(df['class'])

        df['class'] = encode.transform(df['class'])

        # we will save the encoder as pickle to use when we do the prediction. We will need to decode the predcited values
        # back to original
        with open(os.path.join(self.config.encoding_dir, self.config.encoder_file), 'wb') as file:
            pickle.dump(encode, file)
        logger.info("label encoding successfull")

        # save the data
        df.to_csv(os.path.join(self.config.data_dir, self.config.data_file), index=False)
        logger.info(f"file saved as {self.config.data_file}")

        return df

    def get_clustered_data(self,df):
        config = ConfigurationManager()
        data_clustering_config = config.get_data_clustering_config()
        data_clustering = DataClustering(df,config=data_clustering_config)
        optimal_clusters = data_clustering.plot_knee()
        data_clustering.create_clusters(optimal_clusters)


from django.http import HttpResponse, HttpResponseServerError
from django.utils.deprecation import MiddlewareMixin
from wagtail.core.models import Site
import logging

LOGGER = logging.getLogger("healthcheck")

try:
    from django.conf import settings
    XS_SHARING_ALLOWED_ORIGINS = settings.XS_SHARING_ALLOWED_ORIGINS
    XS_SHARING_ALLOWED_METHODS = settings.XS_SHARING_ALLOWED_METHODS
    XS_SHARING_ALLOWED_HEADERS = settings.XS_SHARING_ALLOWED_HEADERS
    XS_SHARING_ALLOWED_CREDENTIALS = settings.XS_SHARING_ALLOWED_CREDENTIALS
except AttributeError:
    XS_SHARING_ALLOWED_ORIGINS = '*'
    XS_SHARING_ALLOWED_METHODS = ['POST', 'GET', 'OPTIONS', 'PUT', 'DELETE']
    XS_SHARING_ALLOWED_HEADERS = ['Content-Type', '*']
    XS_SHARING_ALLOWED_CREDENTIALS = 'true'

class SiteMiddleware(MiddlewareMixin):

    def process_request(self, request):
        """
        Set request.site to contain the Site object responsible for handling this request.
        """
        try:
            request.site = Site.find_for_request(request)
        except Site.DoesNotExist:
            request.site = None

class XsSharing(object):
    """
    This middleware allows cross-domain XHR using the HTML5 postMessage API.

    Access-Control-Allow-Origin: http://foo.example
    Access-Control-Allow-Methods: POST, GET, OPTIONS, PUT, DELETE

    Based off https://gist.github.com/426829
    """
    def process_request(self, request):
        if 'HTTP_ACCESS_CONTROL_REQUEST_METHOD' in request.META:
            response = HttpResponse()
            response['Access-Control-Allow-Origin'] = XS_SHARING_ALLOWED_ORIGINS
            response['Access-Control-Allow-Methods'] = ",".join(XS_SHARING_ALLOWED_METHODS)
            response['Access-Control-Allow-Headers'] = ",".join(XS_SHARING_ALLOWED_HEADERS)
            response['Access-Control-Allow-Credentials'] = XS_SHARING_ALLOWED_CREDENTIALS
            return response

        return None

    def process_response(self, request, response):
        response['Access-Control-Allow-Origin'] = XS_SHARING_ALLOWED_ORIGINS
        response['Access-Control-Allow-Methods'] = ",".join(XS_SHARING_ALLOWED_METHODS)
        response['Access-Control-Allow-Headers'] = ",".join(XS_SHARING_ALLOWED_HEADERS)
        response['Access-Control-Allow-Credentials'] = XS_SHARING_ALLOWED_CREDENTIALS

        return response

class HealthCheckMiddleware(object):

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        if request.method == "GET":
            if request.path == "/readiness":
                return self.readiness(request)
            elif request.path == "/liveness":
                return self.liveness(request)
        return self.get_response(request)

    def liveness(self, request):
        """Returns that the server is alive.
        """
        return HttpResponse("OK")

    def readiness(self, request):
        """Connect to each database and do a generic standard SQL query
        that doesn't write any data and doesn't depend on any tables
        being present.
        """
        try:
            from django.db import connections
            for name in connections:
                cursor = connections[name].cursor()
                cursor.execute("SELECT 1;")
                row = cursor.fetchone()
                if row is None:
                    return HttpResponseServerError("db: invalid response")
        except Exception as e:
            LOGGER.exception(e)
            return HttpResponseServerError("db: cannot connect to database.")

        return HttpResponse("OK")


import matplotlib.pyplot as plt
from numpy import (sin, pi, linspace)
from numpy.testing import (assert_almost_equal)
from pymarine.utils.plotting import sub_plot_axis_to_2d

def test_sub_plot_axis_to_2d():
    def test_plot(nr, nc, x, y):
        # create a plot with nr x nc subplots
        fig, axis = plt.subplots(nrows=nr, ncols=nc)
        # turn the axis into a 2D list
        axis_2d = sub_plot_axis_to_2d(axis, n_rows=nr, n_cols=nc)
        # for all the plots, add a line and check if the plot line data equals the original data
        for i_row in range(nr):
            for j_col in range(nc):
                line, = axis_2d[i_row][j_col].plot(x, y)
                x_line, y_line = line.get_xydata().T
                assert_almost_equal(x_line, x)
                assert_almost_equal(y_line, y)

    x = linspace(0, 2 * pi, num=100)
    y = sin(x)
    # loop over a number of rows and columns ranging in between 1,2,3
    for nr in range(1, 4):
        for nc in range(1, 4):
            test_plot(nr, nc, x, y)


from flask import Flask, render_template, request, url_for
from keras.models import Model, load_model
import tensorflow as tf
import numpy as np
import pickle
import os

# Constants used in execution
APP_ROOT = os.path.dirname(os.path.abspath(__file__))   # refers to application_top
APP_STATIC = os.path.join(APP_ROOT, 'static')
ENCODER_MODEL_FILE = APP_ROOT + '/gridsai-qahumor-encoder-model.h5'
DECODER_MODEL_FILE = APP_ROOT + '/gridsai-qahumor-decoder-model.h5'
ENCODER_PICKLE_FILE = APP_ROOT + '/gridsai-qahumor-encoder-pickle.pckl'
DECODER_PICKLE_FILE = APP_ROOT + '/gridsai-qahumor-decoder-pickle.pckl'
CHAR_TOKENS = ['\n', '\t', ' ', '.', ',', '!', '?', ':', ';', '$', '#', '@', '%', '^', '&', '*', '(', ')', '-', '_',
               '=', '+', '\\', '/', '|', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
               'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
               'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4', '5',
               '6', '7', '8', '9', '0', '\"', '\'', '[', ']', '{', '}', '<', '>', '`', '~', '’', 'ø', 'è', '£', 'é',
               '∫', '—', '͡', '°', '͜', 'ʖ', '']

# Global variables
encoder_model = None
decoder_model = None
graph = None
max_encoder_seq_length = None
max_decoder_seq_length = None
num_tokens = None
token_index = None
reverse_char_index = None

application = Flask(__name__)

def setup():
  global encoder_model
  encoder_model = load_model(ENCODER_MODEL_FILE, compile=False)
  global decoder_model
  decoder_model = load_model(DECODER_MODEL_FILE, compile=False)
  global graph
  graph = tf.get_default_graph()

  global max_encoder_seq_length
  encoder_pickle_file = open(ENCODER_PICKLE_FILE, 'rb')
  max_encoder_seq_length = pickle.load(encoder_pickle_file)
  encoder_pickle_file.close()
  global max_decoder_seq_length
  #decoder_pickle_file = open(DECODER_PICKLE_FILE, 'rb')
  #max_decoder_seq_length = pickle.load(decoder_pickle_file)
  max_decoder_seq_length = 50
  #decoder_pickle_file.close()

  # Reverse-lookup token index to decode sequences back to something readable.
  global num_tokens
  global token_index
  global reverse_char_index
  characters = sorted(list(CHAR_TOKENS))
  num_tokens = len(characters)
  token_index = dict([(char, i) for i, char in enumerate(characters)])
  reverse_char_index = dict((i, char) for char, i in token_index.items())

setup()

@application.route('/', methods=['POST', 'GET'])
def generate_joke():
    if request.method == 'GET':
        return render_template('jokes.html')
    elif request.method == 'POST':
        joke = [('\t' + request.form['userJoke'], request.form['userJoke'])]
        answer = forward_pass(joke)
        return render_template('jokes.html', userJoke=request.form['userJoke'], punchline=answer[0][1][1])

def decode_sequence(input_seq, seed_char, num_decoder_tokens, target_token_index, reverse_target_char_index, max_decoder_seq_length):
    # Encode the input as state vectors.
    with graph.as_default():
      states_value = encoder_model.predict(input_seq)

    # Generate empty target sequence of length 1.
    target_seq = np.zeros((1, 1, num_decoder_tokens))
    # Populate the first character of target sequence with the start character.
    target_seq[0, 0, target_token_index[seed_char]] = 1.

    # Sampling loop for a batch of sequences
    # (to simplify, here we assume a batch of size 1).
    stop_condition = False
    decoded_sentence = seed_char
    while not stop_condition:
      with graph.as_default():
        output_tokens, h, c = decoder_model.predict(
            [target_seq] + states_value)

        # Sample a token
        sampled_token_index = np.argmax(output_tokens[0, -1, :])
        sampled_char = reverse_target_char_index[sampled_token_index]
        decoded_sentence += sampled_char

        # Exit condition: either hit max length
        # or find stop character.
        if (sampled_char == '\n' or
            sampled_char == '.' or
            sampled_char == '!' or
            sampled_char == '?' or
	          len(decoded_sentence) > max_decoder_seq_length):
          stop_condition = True

        # Update the target sequence (of length 1).
        target_seq = np.zeros((1, 1, num_decoder_tokens))
        target_seq[0, 0, sampled_token_index] = 1.

        # Update states
        states_value = [h, c]

    return decoded_sentence

def forward_pass(jokes_needing_punchlines):
    """
    Generates
    :param jokes_needing_punchlines: List of joke setup strings.
    :return: List of 2-tuples containing joke setup/generated punchline pairs.
    """
    decoded_sentences = []
    encoder_input_data = np.zeros(
        (len(jokes_needing_punchlines), max_encoder_seq_length, num_tokens),
        dtype='float32')
    for i, input_text in enumerate(jokes_needing_punchlines):
        for t, char in enumerate(input_text[0]):
            encoder_input_data[i, t, token_index[char]] = 1.
    for seq_index in range(len(jokes_needing_punchlines)):
        # Take one sequence for trying out decoding.
        input_seq = encoder_input_data[seq_index: seq_index + 1]
        decoded_sentence = decode_sequence(input_seq, jokes_needing_punchlines[seq_index][1][0], num_tokens, token_index, reverse_char_index, max_decoder_seq_length)
        decoded_sentences.append((jokes_needing_punchlines[seq_index][1][0], (jokes_needing_punchlines[seq_index][0], decoded_sentence[1:].strip())))

    return decoded_sentences

if __name__ == '__main__':
  application.run()


from dcext.oanda.api import OandaAPI, CANDLESV3, TOKEN
from datetime import datetime, timedelta, timezone
import pandas as pd
from pymongo import MongoClient
from pymongo.errors import DuplicateKeyError
from itertools import product
import logging
import json

EXCHANGE = "oanda"

def get_dt(date, tz=None):
    if isinstance(date, int):
        return get_dt(str(date), tz)
    elif isinstance(date, str):
        return datetime.strptime(date.replace("-", ""), "%Y%m%d").replace(tzinfo=tz)
    elif isinstance(date, datetime):
        return date.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=tz)
    else:
        raise TypeError("Not supported type: %s" % type(date))

class API(OandaAPI):

    def bar(self, instrument, granularity, start, end):
        if isinstance(start, datetime):
            start = start.timestamp()
        if isinstance(end, datetime):
            end = end.timestamp()
        query = {
            "granularity": granularity,
            "from": start,
            "to": end
        }
        content = self.get(CANDLESV3, query, instrument=instrument)
        data = json.loads(content)

        result = []
        for bar in data["candles"]:
            result.append(self.generate(bar))
        return  result

    MAPPER = {"o": "open", "h": "high", "c": "close", "l": "low"}

    def generate(self, bar):
        doc = bar.copy()
        mid = doc.pop("mid")
        for o, t in self.MAPPER.items():
            doc[t] = float(mid[o])
        return doc

class MongodbStorage(object):

    INSTRUMENT = "_i"
    START = "_s"
    END = "_e"
    DATE = "_d"
    COUNT = "_c"
    FILL = "_f"
    MODIFY = "_m"

    def __init__(self, host=None, db="OANDA_M1", log="log.oanda", tz=0):
        self.client = MongoClient(host)
        self.db = self.client[db]
        ldb, lcol = log.split(".", 1)
        self.log = self.client[ldb][lcol]
        self.tz = timezone(timedelta(hours=tz))
        self.init_log_collection()

    def ensure_table(self, instrument):
        collection = self.get_collection(instrument)
        info = collection.index_information()
        unique = info.get("datetime_1", {}).get("unique", False)
        if not unique:
            logging.warning("ensure table | %s | index datetime not unique", instrument)
            self.drop_dups(collection)
            collection.create_index("datetime", unique=1, background=True)
            logging.warning("ensure table | %s | unique index datetime created", instrument)
        else:
            logging.warning("ensure table | %s | unique index datetime exists", instrument)
        if "date_1" not in info:
            collection.create_index("date", background=True)
            logging.warning("ensure table | %s | index date created", instrument)

    @staticmethod
    def drop_dups(collection):
        dts = set()
        cursor = collection.find(None, ["datetime"])
        _id = None
        while True:
            try:
                doc = next(cursor)
                _id = doc["_id"]
                dt = doc["datetime"]
            except StopIteration:
                break
            except KeyError:
                continue
            except:
                cursor = collection.find({"_id": {"$gte": _id}}, ["datetime"])
            else:
                if dt in dts:
                    collection.delete_one({"_id": _id})
                    logging.warning("drop dups | %s | %s" % (collection, dt))
                else:
                    dts.add(dt)

    def init_log_collection(self):
        self.log.create_index([
            (self.INSTRUMENT, 1),
            (self.DATE, 1)
        ], unique=True, background=True)

    def create(self, instrument, date):
        dt = get_dt(date, self.tz)
        filters = {
            self.INSTRUMENT: instrument,
            self.DATE: date,
        }
        doc = {
            self.START: dt,
            self.END: dt+timedelta(days=1),
            self.COUNT: 0,
            self.FILL: 0,
            self.MODIFY: datetime.now()
        }
        doc.update(filters)
        return self.log.update_one(filters, {"$setOnInsert": doc}, upsert=True).upserted_id

    def fill(self, instrument, date, count, fill):
        filters = {
            self.INSTRUMENT: instrument,
            self.DATE: date
        }
        doc = {self.COUNT: count, self.FILL: fill, self.MODIFY: datetime.now()}
        self.log.update_one(filters, {"$set": doc})

    def find(self, instruments=None, start=None, end=None, filled=False):
        filters = {}
        if instruments:
            filters[self.INSTRUMENT] = {"$in": instruments}
        if start:
            filters[self.DATE] = {"$gte": start}
        if end:
            filters.setdefault(self.DATE, {})["$lte"] = end
        if filled:
            filters[self.FILL] = {"$gte": 0}
        elif filled is not None:
            filters[self.FILL] = 0
        print(filters)
        cursor = self.log.find(filters, [self.INSTRUMENT, self.DATE, self.START, self.END])
        for doc in list(cursor):
            yield doc[self.INSTRUMENT], doc[self.DATE], doc[self.START].replace(tzinfo=self.tz), doc[self.END].replace(tzinfo=self.tz)

    def time(self, date):
        if isinstance(date, int):
            return self.time(str(date))
        elif isinstance(date, str):
            return datetime.strptime(date.replace("-", ""), "%Y%m%d").replace(tzinfo=self.tz)
        elif isinstance(datetime, date):
            return date.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=self.tz)

    def write(self, instrument, data):
        count = 0
        collection = self.get_collection(instrument)
        for bar in data:
            doc = self.vnpy_format(bar, instrument)
            count += self.append(collection, doc)
        return count

    def get_collection(self, instrumet):
        return self.db["%s:%s" % (instrumet, EXCHANGE)]

    @staticmethod
    def append(collection, bar):
        try:
            collection.insert_one(bar)
        except DuplicateKeyError:
            return 0
        else:
            return 1

    def vnpy_format(self, bar, symbol):
        bar.pop("complete", None)
        bar["symbol"] = symbol
        bar["exchange"] = EXCHANGE
        bar["vtSymbol"] = "%s:%s" % (symbol, EXCHANGE)
        dt = datetime.strptime(bar.pop("time").split(".")[0], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=self.tz)
        bar["datetime"] = dt
        bar["date"] = dt.strftime("%Y%m%d")
        bar["time"] = dt.strftime("%H:%M:%S.000000")
        bar["rawData"] = None
        bar["openInterest"] = 0
        return bar

    def get_last_date(self):
        doc = self.log.find_one(sort=[(self.DATE, -1)])
        if doc:
            return doc[self.DATE]
        else:
            return None

class Framework(object):

    def __init__(self, api, storage, ltz=8):
        assert isinstance(api, API)
        assert isinstance(storage, MongodbStorage)
        self.api = api
        self.storage = storage
        self.ltz = timezone(timedelta(hours=ltz))

    def create(self, instruments, start, end):
        dates = pd.date_range(get_dt(start), get_dt(end)).map(lambda t: t.year*10000+t.month*100+t.day)
        for i, d in product(instruments, dates):
            r = self.storage.create(i, int(d))
            logging.warning("create log | %s | %s | %s", i, d, r)

    def ensure(self, instruments):
        for i in instruments:
            self.storage.ensure_table(i)

    def publish(self, instruments=None, start=None, end=None, filled=False, redo=3):
        logging.warning("publish cycle start| %s | %s | %s | %s | %s", instruments, start, end, filled, redo)
        now = datetime.now(self.ltz)
        missions = list(self.storage.find(instruments, start, end, filled))
        total = len(missions)
        accomplish = 0
        for i, d, s, e in missions:
            if e >= now:
                logging.warning("publish | %s | %s | end: %s is future", i, d, e)
                accomplish += 1
                continue
            accomplish += self.download(i, d, s, e)
        logging.warning("publish cycle done | total: %s | accomplished: %s", total, accomplish)
        if redo:
            if accomplish < total:
                self.publish(instruments, start, end, False, redo-1)

    def download(self, instrument, date, start, end):
        try:
            data = self.api.bar(instrument, "M1", start, end)
            count = len(data)
        except Exception as e:
            logging.error("req bar | %s | %s | %s", instrument, date, e)
            return 0

        try:
            if count:
                fill = self.storage.write(instrument, data)
            else:
                fill = -1
        except Exception as e:
            logging.error("write bar | %s | %s | %s", instrument, date, e)
            return 0

        try:
            self.storage.fill(instrument, date, count, fill)
        except Exception as e:
            logging.error("fill log | %s | %s | %s", instrument, date, e)
        else:
            logging.warning("download bar | %s | %s | fill: %s, count: %s", instrument, date, fill, count)
            return 1

    def last_date(self):
        return self.storage.get_last_date()

def init_from_config(filename):
    with open(filename) as f:
        conf = json.load(f)
    api = API(conf.get("token", None))
    storage = MongodbStorage(**conf.get("storage", {}))
    fw = Framework(api, storage, conf.get("ltz", 8))
    return fw

def run(command, instruments, start, end, filled=False, redo=3, config_file="oanda_m1.json"):
    fw = init_from_config(config_file)
    if command == "create":
        assert isinstance(instruments, list)
        assert isinstance(start, int)
        assert isinstance(end, int)
        fw.create(instruments, start, end)
    elif command == "publish":
        fw.publish((instruments, start, end, filled, redo))

import click

@click.command()
@click.option("-i", "--instruments", default="")
@click.option("-s", "--start", default=None, type=click.INT)
@click.option("-e", "--end", default=None, type=click.INT)
@click.option("-f", "--filled", default=False, is_flag=True)
@click.option("-r", "--redo", default=3, type=click.INT)
@click.option('-n', "--filename", default="oanda_m1.json", type=click.STRING)
@click.argument("command", nargs=-1)
def command(command, instruments, start, end, filled=False, redo=3, filename="oanda_m1.json"):
    with open(filename) as f:
        conf = json.load(f)
    api = API(conf.get("token", None))
    storage = MongodbStorage(**conf.get("storage", {}))
    fw = Framework(api, storage, conf.get("ltz", 8))
    if instruments:
        instruments = instruments.split(",")
    else:
        instruments = conf["instruments"]

    if len(command) == 0:
        command = ["publish"]
    for cmd in command:
        if cmd == "create":
            if not start:
                start = fw.last_date() or conf.get("start", None)
            if not end:
                end = conf.get("end", int(datetime.now().strftime("%Y%m%d")))
            assert isinstance(start, int)
            assert isinstance(end, int)
            fw.create(instruments, start, end)
        elif cmd == "publish":
            fw.publish(instruments, start, end, filled, redo)
        elif cmd == "ensure":
            fw.ensure(instruments)

if __name__ == '__main__':
    command()


r"""We deal with the 196884-dimensional representation of the monster.

Here class ``MMSpaceCRT`` is an analogue of class ``MMSpace`` that
allows a limited set of operations on the real
``196884``-dimensional representation of the monster with vectors of
a small norm. Typical examples of such vectors are unit vectors, or
so called *axes*, as described in :cite:`Con85`.

"""
# References in the __docstr__ see file docs/source/references.bib

from __future__ import absolute_import, division, print_function
from __future__ import  unicode_literals

import sys
import os
import re
import numpy as np
from numbers import Integral
import warnings
from collections import defaultdict, OrderedDict
import math
from functools import partial

from mmgroup.structures.abstract_group import singleton
from mmgroup.structures.abstract_mm_rep_space import AbstractMmRepVector
from mmgroup.structures.abstract_mm_rep_space import AbstractMmRepSpace
from mmgroup.structures.mm_space_indices import tuple_to_sparse
from mmgroup.structures.mm_space_indices import numeric_index_to_sparse
from mmgroup.structures.mm_space_indices import sparse_from_indices
from mmgroup.structures.abstract_group import AbstractGroupWord
from mmgroup.structures.abstract_mm_group import AbstractMMGroupWord
from mmgroup.mm_space  import MMSpace, MMVector
from mmgroup.mm_space  import standard_mm_group
from mmgroup.structures.parse_atoms import AtomDict
from mmgroup.structures.parse_atoms import eval_atom_expression, ihex

from mmgroup.mm_op import mm_aux_mmv_size
from mmgroup.mm_op import mm_vector, mm_aux_random_mmv
from mmgroup.mm_op import mm_aux_zero_mmv, mm_aux_reduce_mmv
from mmgroup.mm_op import mm_aux_mmv_to_sparse
from mmgroup.mm_op import mm_aux_mmv_set_sparse
from mmgroup.mm_op import mm_crt_combine, mm_crt_check_v2
from mmgroup.mm_op import mm_crt_combine_bytes
from mmgroup.mm_op import mm_crt_check_g
from mmgroup.mm_op import mm_crt_norm_int32
from mmgroup.generators import mm_group_n_clear
from mmgroup.generators import mm_group_n_mul_word_scan
from mmgroup.generators import mm_group_n_to_word
from mmgroup.mm_op import mm_op_word
from mmgroup.mm_op import mm_op_compare

PRECISION = math.log(7 * 31 * 127 * 255) / math.log(2.0) - 4

######################################################################
# Auxiliary class vsparse representing a sparse vector
######################################################################

ERR_CRT_TYPE = "Connot construct MMVectorCRT object from type '%s'"

class vsparse:
    def __init__(self, *data):
        self.d = defaultdict(int)
        if len(data) == 0 or not data[0]:
            return
        scalar = 1
        if isinstance(data[0], vsparse):
            self.d.update(data[0].d)
            return
        if isinstance(data[0], Integral):
            scalar, data = data[0],  data[1:]
        if scalar == 0 or len(data) == 0:
            return
        if isinstance(data[0], str):
            if len(data[0]) != 1 or not data[0] in "ABCTXZYIDE0":
                err = "Illegal tag '%s' in tuple for class MMSpaceCRT"
                raise ValueError(err %  data[0])
        else:
            raise TypeError(ERR_CRT_TYPE % type(data[0]))
        for t in tuple_to_sparse(255, *data):
            scalar1, t =  t & 0xff, t & 0xffffff00
            scalar1 = scalar1 if scalar1 < 128 else scalar1 - 255
            self.d[t] += scalar * scalar1

    def __imul__(self, other):
        assert isinstance(other, Integral)
        if other == 0:
            self.d.clear()
        else:
            for tag in self.d.keys():
                 self.d[tag] *= other
        return self

    def __mul__(self, other):
        return vsparse(self).__imul__(other)

    __rmul__ = __mul__

    def __neg__(self):
        return self.__mul__(-1)

    def __pos__(self):
        return self

    def __iadd__(self, other):
        if other == 0:
            return
        assert isinstance(other, vsparse)
        for tag, value in other.d.items():
            self.d[tag] += value
        return self

    def __add__(self, other):
        return vsparse(self).__iadd__(other)

    def __isub__(self, other):
        if other == 0:
            return
        assert isinstance(other, vsparse)
        for tag, value in other.d.items():
            self.d[tag] -= value
        return self

    def __sub__(self, other):
        return vsparse(self).__isub__(other)

    def reduce(self):
        for tag, value in self.d.items():
            if value == 0:
                del self.d[tag]
        return self

    def norm(self):
        norm = 0
        self.reduce()
        for tag, value in self.d.items():
            factor = 1
            if tag & 0xE000000 == 0x2000000:
                i0, i1 = (tag >> 14) & 0x7ff, (tag >> 8) & 0x3f
                factor += i0 != i1
            norm += factor * value * value
        return norm

    def sparse_array(self, p, shift = 0):
        assert p & 1 and p < 256
        self.reduce()
        a = np.zeros(len(self.d), dtype = np.uint32)
        for i, (tag, value) in enumerate(self.d.items()):
            a[i] = tag + (value << shift) % p
        return a

######################################################################
# Convert string to instance of class vsparse
######################################################################

FRAME = re.compile(r"^([A-Za-z_])+\<([0-9]+;)?(.+)\>$")

def vsparse_from_str(s):
    string = s
    m = FRAME.match(s)
    if m:
        _, p_str, string = m[1], m[2], m[3]
        if p_str:
            err = "Vector is defined module an integer only"
            raise ValueError(err)
    f = AtomDict(vsparse)
    return eval_atom_expression(string, f)

######################################################################
# Auxiliary functions
######################################################################

def _err_tag(*args, **kwds):
    err = "Bad entry in monster group element"
    raise ValueError(err)

def _iter_group(g):
    start, data, len_ = 0, g.mmdata, len(g.mmdata)
    while start < len_:
        tag = data[start] & 0x70000000
        if  tag >= 0x50000000:
            yield  data[start : start+1], True
            start += 1
        else:
            end_ = start
            while end_ < len_ and data[end_] & 0x70000000 < 0x50000000:
               end_ += 1
            yield data[start : end_], False
            start = end_

def compress_data(data):
    out = np.zeros(196884, dtype = np.int32)
    out[0:24] = data[0:768:33]
    k = 24
    for i in range(1,24):
        out[k:k+i] = data[32*i:32*i+i]
        out[276+k:276+k+i] = data[768+32*i:768+32*i+i]
        out[552+k:552+k+i] = data[1536+32*i:1536+32*i+i]
        k += i
    assert k == 300
    out[852:49428] = data[2304:50880]
    t = data[50880:247488].reshape((3*2048,32))[:,:24]
    out[49428:196884] = t.ravel()
    assert len(out) == 196884
    return out

######################################################################
# If check_MMVectorCRT is True then we check a vector of type
# MMVectorCRT for overflow or underflow after each operation
######################################################################

check_MMVectorCRT = True

######################################################################
# Modelling a vector of the 196884-dimensional rep of the monster
######################################################################

MODULUS = 7*31*125*255
MAX_CRT_NORM = MODULUS**2 // 4
ERR_OVERFLOW = "Overflow in class MMVectorCRT"
ERR_UNDERFLOW = "Underflow in class MMVectorCRT"
ERR_UNDERFLOW_G = "Underflow at group operation in class MMVectorCRT"

class MMVectorCRT(AbstractMmRepVector):
    """Models a vector in a space of type ``MMSpaceCRT``.

    Such a vector should be constructed by calling an instance ``V``
    of class ``MMSpaceCRT`` which models a real representation of
    the monster group. Calculations in this space are exact in
    fixed-point arithmetic with a precision of about %.2f bits.

    ValueError is raised in case of overflow or underflow.

    The functionality of this class is a subset of the functionality
    of class ``MMVector``. See class ``MMSpaceCRT`` for details.

    A vector may also be reduced modulo ``p = 7, 31, 127, 255`` with
    the modulo operator ``%%``. Then the result is a vector in the
    vector space ``MMSpace(p)``.

    :var space:
        This attribute contains the space to which the vector
        belongs. That space is an instance of class |MMSpaceCRT|.

    .. warning::
       The constructor of this class is not for public use! You
       may call an instance ``V`` of class  |MMSpaceCRT| for
       constructing vectors in the real representation space
       ``V`` of the monster group.
    """ % PRECISION
    p = MODULUS
    MAX_CRT_NORM = MAX_CRT_NORM

    def __init__(self, shift, tag = 0, i0 = None, i1 = None):
        self.shift = shift
        self.factor = 1.0 / (1 << shift)
        if not 3 <= self.shift <= 21:
            raise ValueError("Bad shift factor for class MMVectorCRT")
        self.data = OrderedDict()
        self.data_int = np.zeros(247488, dtype = np.int32)
        self.expanded = False
        d = vsparse()
        if isinstance(tag, MMVectorCRT):
            for p in (7, 31, 127, 255):
                self.data[p] = tag.data[p].copy()
            self.shl(self.shift - tag.shift)
            return
        elif isinstance(tag, Integral) and not tag:
            for p in (7, 31, 127, 255):
                self.data[p] = MMVector(p)
            return
        elif isinstance(tag, str) and len(tag) == 1:
            d += vsparse(tag, i0, i1)
        elif isinstance(tag, str) and tag == "Axis":
            g = None
            if isinstance(i0, AbstractMMGroupWord):
               i0, g = MMSpace._mm_element_to_axis(i0)
               i1 = None
            for p in (7, 31, 127, 255):
                self.data[p]  = MMVector(p, tag, i0, i1) << self.shift
            if g is not None:
                self *= g
            return
        elif isinstance(tag, str):
            d += vsparse_from_str(tag)
        elif isinstance(tag, list):
            for x in tag:
                if isinstance(x,tuple):
                    d += vsparse(*x)
                elif isinstance(x, str):
                    d += vsparse_from_str(x)
        else:
            err = "Connot construct MMVectorCRT object from type '%s'"
            raise ValueError(err % type(tag))
        for p in (7, 31, 127, 255):
            self.data[p] = v = MMVector(p)
            ind = d.sparse_array(p, shift)
            mm_aux_mmv_set_sparse(p, v.data, ind, len(ind))
        if check_MMVectorCRT:
            self.expand()
            if self._inorm > MAX_CRT_NORM:
                raise ValueError(ERR_OVERFLOW)

    def expand(self):
        """Expand data of vector ``v`` with CRT

        In the array ``v.data_int`` the ``i``-th entry is computed
        form the ``i``-th entries of the vector modulo 7, 31, 127,
        and 255.
        """
        if not self.expanded:
            v2 = mm_crt_combine(self.data[7].data,
                self.data[31].data, self.data[127].data,
                self.data[255].data, self.data_int)
            self._v2 = v2 - self.shift if v2 < 24 else 24
            self._inorm = mm_crt_norm_int32(self.data_int)
            self.expanded = True

    def shl(self, sh):
        assert isinstance(sh, Integral)
        if sh == 0:
            return self
        if sh > 0:
            if check_MMVectorCRT:
                self.expand()
                if sh > 24 or self._inorm * 4**sh > MAX_CRT_NORM:
                    raise ValueError(ERR_OVERFLOW)
                self.data_int <<= sh
                self._inorm *= 4**sh
            for d in self.data.values:
                d <<= sh
            self._v2 += sh
        elif sh < 0:
            nsh = -sh
            if check_MMVectorCRT:
                self.expand()
                if nsh > self.v2:
                    raise ValueError(ERR_UNDERFLOW)
                self.data_int >>= nsh
                self._inorm *= 4**sh
            for d in self.data.values:
                d >>= nsh
            self._v2 += sh
        return self

    def check(self):
        """Check if the vector is correct

        Raise ValueError if the vector is erroneous.
        """
        return True

    def __ilshift__(self, other):
        return self.shl(other)

    def __lshift__(self, other):
        return self.copy().shl(other)

    def __irshift__(self, other):
        return self.shl(-other)

    def __lrhift__(self, other):
        return self.copy().shil(-other)

    def __mod__(self, p):
        """Return the vector modulo ``p``.

        ``p``  must be in (7, 31, 127, 255). Actually, we divide
        the vector by the *scaling factor* before reducing it
        modulo ``p``.

        This method is mainly for testing.
        """
        if  p in (7, 31, 127, 255):
            return MMVector(p, self.data[p]) >> self.shift
        elif p in (3, 15):
            v0 = self % 255
            return MMVector(p, v0)
        elif isinstance(p, Integral):
            err = "Cannot reduce MMVectorCRT object modulo %d"
            raise ValueError(err % p)
        else:
            err = "Modulus for reducing MMVectorCRT object must be int"
            raise TypeError(err)

    @property
    def v2(self):
        """Return the ``2``-adic value of the vector.

        If "a, b" are odd integers and ``k`` is an integer then
        the  ``2``-adic value  ``a * 2**k / b`` is ``k``.

        The  ``2``-adic value of a vector is the minimum of the
         ``2``-adic values of its entries, ignoring zero entries.

        The function raises ZeroDivisionError if ``v ==  0``
        """
        v2 = self._v2
        if self._v2 >= 24:
            err = "The zero vector has infinite 2-adic value"
            raise ZeroDivisionError(err)
        return self._v2

    @property
    def inorm(self):
        """Return a scaled norm of the vector as an integer

        For a vector ``v`` we have

            ``v.fnorm() = v.inorm() * v.factor**2``.

        Where ``v.fnorm()`` is the real norm of ``v``.
        """
        self.expand()
        return self._inorm

    def norm(self):
        """Return norm of vector as a floating point number.

        The norm of a vector in the representation of the monster
        is the squared sum of is entries. Here the squares of all
        entries with index ``("A", i0, i1)`` must be doubled in
        case ``i0 != i1``.

        The returned norm is exact.
        """
        return self.inorm * self.factor**2

######################################################################
# class MMSpace
######################################################################

@singleton
class MMSpaceCRT(AbstractMmRepSpace):
    """Models a ``196884``-dimensional representation of the monster group

    This class models a real representation of the monster group
    with fixed-point arithmetic. Calculations are done by combining
    vectors modulo ``p = 7, 31, 127, 255`` with Chinese remaindering.
    This way we achieve about %.2f bit precision.

    The construction of a vector in this space and the computation
    with such vectors works in the same way as in class |MMSpace|.
    But there are som limitations:

      * Vectors may be constructed as in class |MMSpace|, but
        the arguments of the constructor may be tuples only.
        Here randomized scalars are illegal.

      * The only operations allowed for vectors are copying,
        multiplication with a group element, and testing for
        equality. Vector addition and scalar multiplication
        are illegal.

      * A vector may be reduced modulo one of the primes
        ``p = 7, 31, 127, 255`` using the modulo operator ``%%``.
        The result is a vector in the space ``MMSpace(p)``.

      * Changing entries of a vector via item assignment is
        illegal.

      * Entries and subarrays of a vector may be obtained as
        in class |MMSpace|. Here subarrays are returned as
        integers or numpy  arrays with ``dtype = np.int32``.
        On output, each number is multiplied with ``2**k``
        in order to obtain an integer. Here ``k`` is the
        argument given in the constructor by parameter ``shift``.

        You may multiply such an integer value with the attribute
        ``factor`` of the space for obtaining an exact floating
        point value. But floating point data may lead to imprecise
        results when processing the with a computer algebra system.

    :var shift:
        If shift is ``k`` then a vector ``v`` can be represented if
        all entries of ``v`` are integer multiples of ``2**(-k)``.
        The absolute value of an entry may be at most
        ``3513772 / 2**k``. In case ``shift == 20`` (default)
        this is about  ``3.35``.

    :var group:
        Contains the group operating on the vector
        space by right multiplication.
        That group must be an instance of class |MMGroup|.

    Caution!

        The norm of an input vector must not exceed
        ``1.234 * 10**13 * 4.0 ** (-k)``. Otherwise *overflow*
        occurs. In case ``k = 20`` (default) this quantity is about
        ``11.229``. Here ``k`` is given by parameter ``shift``.

        The norm of a vector is the sum of the squares of its
        entries, where for all entries with index ``("A", i0, i1)`` ,
        ``i0 != i1``, the doubled square of the entry must be
        taken instead.

    Caution!

        When multiplying a vector ``v`` with a generator of the group
        with tag ``t`` or ``l``, and ``v`` has an entry that is not
        an integer multiple of ``8 * 2**(-k)``, then a
        *precision error* may occur. We raise ``ValueError`` in
        case of a precision error.

    """ % PRECISION
    vector_type = MMVectorCRT
    _max_norm = (7*31*125*255)**2 // 4

    def __init__(self):
        """Create a 196884-dimensional representation of the monster

        All calculations are done modulo the odd number p
        """
        super(MMSpaceCRT, self).__init__()

    #######################################################################
    # Conversion to a list of tuples
    #######################################################################

    def _not_supported(self, *args, **kwds):
        err = "Method not supported in class MMSpaceCRT"
        raise NotImplementedError(err)

    as_tuples = _not_supported

    #######################################################################
    # Creating vectors
    #######################################################################

    def zero(self, shift):
        """Return the zero vector"""
        err = "Cannot create zero vector in space of class MMSpaceCRT"
        return MMVectorCRT(shift)

    def copy_vector(self, v1):
        assert v1.space == self
        v = MMVectorCRT(v1.shift, 0)
        for p in (7, 31, 127, 255):
            np.copyto(v.data[p].data, v1.data[p].data)
        if v1.expanded:
            np.copyto(v.data_int, v1.data_int)
            v._v2 = v1._v2
            v._inorm = v1._inorm
        v.expanded = v1.expanded
        return v

    def __call__(self, *args):
       return self.from_tuples(*args)

    def set_rand_uniform(self, *args, **kwds):
        err = "Cannot create random vector in space of class MMSpaceCRT"
        raise NotImplementedError(err)

    parse = _not_supported

    #######################################################################
    # Obtaining and setting components via sparse vectors
    #######################################################################

    def getitems_sparse(self, v, a_sparse):
        raise NotImplementedError(err)

    def additems_sparse(self, *args, **kwds):
        err = "Sparse representation not supported in class MMSpaceCRT"
        raise NotImplementedError(err)

    setitems_sparse = additems_sparse

    #######################################################################
    # Conversion from and to to sparse representation
    #######################################################################

    as_sparse = additems_sparse

    #######################################################################
    # Vector operations
    #######################################################################

    def iadd(self, v1, v2):
        err = "Vector addition not supported in class MMSpaceCRT"
        raise NotImplementedError(err)

    def imul_scalar(self, v1, a):
        err = "Scalar multiplication not supported in class MMSpaceCRT"
        raise NotImplementedError(err)

    #######################################################################
    # Group operation
    #######################################################################

    def _imul_word(self, v1, g_word, buf):
        if len(g_word):
            vectors = list(v1.data.values())
            if check_MMVectorCRT:
                if mm_crt_check_g(g_word[0], *[v.data for v in vectors]):
                    print("MMVectorCRT: shift = %d, v2 = %d, tag = %s" %
                        (v1.shift, v1.v2, ((g_word[0] >> 28) & 7)))
                    raise ValueError(ERR_UNDERFLOW_G)
            for v in vectors:
                mm_op_word(v.p, v.data, g_word, len(g_word), 1, buf)

    def imul_group_word(self, v1, g):
        """Return product v1 * g of vector v1 and group word g.

        v1 is replaced by v1 * g.

        This method is called for elements v1 of the space
        'self' and for elements g of the group 'self.group' only.
        """
        assert isinstance(g, AbstractGroupWord) and g.group.is_mmgroup
        a = g.mmdata
        nn = np.zeros(5, dtype = np.uint32)
        nnw = np.zeros(5, dtype = np.uint32)
        buf = np.zeros(mm_aux_mmv_size(255), dtype = np.uint64)
        while len(a):
            mm_group_n_clear(nn)
            i = mm_group_n_mul_word_scan(nn, a, len(a))
            length =  mm_group_n_to_word(nn, nnw)
            self._imul_word(v1, nnw[:length], buf)
            a = a[i:]
            if len(a):
                self._imul_word(v1, a[:1], buf)
                a = a[1:]
        del buf
        v1.expanded = False
        return v1

    vector_mul_exp = _not_supported

    #######################################################################
    # Checking equality
    #######################################################################

    def equal_vectors(self, v1, v2):
        """Return True iff vectors v1 and v2 are equal

        This method is called for elements v1 and v2 of the space
        'self' only.
        """
        v1.expand()
        v2.expand()
        v1_2, v2_2 = v1._v2, v2._v2
        if v1_2 == v2_2 == 24:
             return True   # then v1 = v1 = zero
        if v1_2 != v2_2:
             return False
        data1, data2 = v1.data_int, v2.data_int
        if v1.shift > v2.shift:
             data1 = data1 >> (v1.shift - v2.shift)
        if v2.shift > v1.shift:
             data2 = data2 >> (v2.shift - v1.shift)
        return (data1 == data2).all()

    #######################################################################
    # Conversion from and to byte format
    #######################################################################

    as_bytes =  _not_supported

    from_bytes = _not_supported

    #######################################################################
    #  Checking and reducing a vector
    #######################################################################

    def check(self, v1):
        return True

    def reduce(self, v1):
        return v1

    #######################################################################
    # getitem and setitem
    #######################################################################

    def vector_get_item(self, v, item):
        assert v.space == self
        if not isinstance(item, tuple):
            item = (item,)
        shape, a_sparse = sparse_from_indices(255, *item)
        d = {}
        for p in (7, 31, 127, 255):
            vdata = v.data[p]
            asp = a_sparse.copy()
            vdata.space.getitems_sparse(vdata, asp)
            d[p] = (asp & p).astype(np.uint8)
        l = len(d[7])
        a = np.zeros(l, dtype = np.int32)
        mm_crt_combine_bytes(d[7], d[31], d[127], d[255], l, a)
        af = np.array(a * v.factor,  dtype = float)
        return af.reshape(shape) if len(shape) else float(af)

    def vector_set_item(*args, **kwd):
        err = "Item assigment not supported in space of type MMSpaceCRT"
        raise NotImplementedError(err)

    #######################################################################
    # Formatting a vector
    #######################################################################

    def str_vector(self, v1):
        return "<vector in space of type MMSpaceCRT>"

    #######################################################################
    # Properties
    #######################################################################

    @property
    def factor(self):
        """Factor of type ``float`` for vector entries.

        When reading entries from a vector then integers or arrays
        of integers are returned. Here each entry must be multiplied
        with this ``factor`` to obtain an exact floating point value
        of that entry.
        """
        return self._factor

    @property
    def max_norm(self):
        """Maximum feasible norm of an input vector.

        The norm of a vector is the sum of the squares of its
        entries, where for all entries with index ``("A", i0, i1)`` ,
        ``i0 != i1``, the doubled square of the entry must be
        taken instead.
        """
        return (self._max_norm * self._factor)**2

StdMMSpaceCRT = MMSpaceCRT()
MMVectorCRT.space = StdMMSpaceCRT

def MMV_CRT(shift):
    return partial(MMVectorCRT, shift)


from ..utils import Utils
from ..authentication import Authentication
from rest_framework.authtoken.models import Token
import json
from ..error_class import CustomException
from ..models import User, Topic
from django.http import HttpResponse

class Decorators():

    def validateHeaders(self, function):
        def innerFunction(*args, **kwargs):
            utils = Utils()
            if utils.contentTypeValid(args[1].content_type):
                return function(*args, **kwargs)
            else:
                raise CustomException("The Content Type is not valid")
        return innerFunction

    def validateIfTopicExists(self, function):
        def innerFunction(*args, **kwargs):
            params = args[1].body.decode("utf-8")
            params = json.loads(params)
            if "topicName" in params:
                if Topic.objects.filter(topicName=params["topicName"].strip()).exists():
                    return function(*args, **kwargs)
                else:
                    raise CustomException("Topic does not exist !")
            else:
                raise CustomException("topicName not present in request")
        return innerFunction

    def validateToken(self, function):
        def innerFunction(*args, **kwargs):
            authentication = Authentication()
            allArgs = []
            if "Authorization" in args[1].headers:
                token = args[1].headers["Authorization"].split(" ")[1]
                if authentication.checkIfTokenExists(token):
                    tokenObject = Token.objects.get(key=token)
                    if not authentication.checkIfTokenExpired(tokenObject):
                        resp = function(*args, **kwargs)
                        return resp
                    else:
                        raise CustomException("Auth Token already expired")
                else:
                    raise CustomException("Token does not exist !")
            else:
                raise CustomException("Invalid Headers")
        return innerFunction

    def containsAllKeys(self, function):
        def innerFunction(*args, **kwargs):
            params = args[1].body.decode("utf-8")
            params = json.loads(params)
            if "userId" in params and "topicId" in params:
                return function(*args, **kwargs)
            else:
                raise CustomException("One or more parameters are missing !")
        return innerFunction

    def checkIfContainsUserId(self, function):
        def innerFunction(*args, **kwargs):
            if args[1].GET.get("userId") is None:
                 raise CustomException("UserId query param is not present")
            else:
                return function(*args, **kwargs)
        return innerFunction

    # def validateParamsLength(function):
    #     def outerFunction(self, outerDecorator):
    #         def innerFunction(*args, **kwargs):
    #             params = args[1].GET.get("eventIdOrName")
    #             if params is None:
    #                 raise CustomException("More than one params passed")
    #             else:
    #                 return function(*args, **kwargs)
                # params = json.loads(params)
                # if len(params) == 1:
                #     return function(*args, **kwargs)
                # else:
                #     raise CustomException("More than one params passed")
        #     return innerFunction
        # return outerFunction

    # WIP; Second order decorator
    # @validateParamsLength
    def validateIfParamIsValid(self, function):
        def innerFunction(*args, **kwargs):
            params = args[1].body.decode("utf-8")
            params = json.loads(params)
            for key, _ in params.keys():
                if key == "eventIdOrName":
                    return function(*args, **kwargs)
                else:
                    return HttpResponse(
                        json.dumps(
                            {
                                "msg": "Invalid Args passed"
                            }
                        )
                    )
        return innerFunction

    def validateEventParams(self, function):
        def innerFunction(*args, **kwargs):
            params = args[1].body.decode("utf-8")
            params = json.loads(params)
            possibleKeys = [
                "eventDescription",
                "eventName",
                "eventType",
                "eventDate",
                "eventDuration",
                "eventHost",
                "eventLocation"
            ]
            for key, value in params.items():
                if key not in possibleKeys:
                    raise CustomException("One or more keys are missing !")
                else:
                    pass
            return function(*args, **kwargs)
        return innerFunction

    def validateIfUserIsAdmin(self, function):
        utils = Utils()
        def innerFunction(referenceToCurrentObj, request):
            params = request.GET.get("params")
            if params is None:
                params = utils.getParamsFromRequest(request)
            else:
                params = json.loads(params)
            userObject = User.objects.get(emailId=params["emailId"])
            if userObject.isAdmin:
                return function(referenceToCurrentObj, request)
            else:
                return HttpResponse(
                    json.dumps(
                        utils.getBadResponse(
                            "User is not an Admin. Hence cannot access this API"
                        )
                    ),
                    status=500
                )
        return innerFunction


import numpy as np
import glob
from PIL import Image
from skimage.metrics import structural_similarity as ssim
from skimage.metrics import peak_signal_noise_ratio as psnr

im_dir = 'results/predict/J/*.jpg'
label_dir = '../Dataset/UIE/UIEBD/test/label/'
p = 0
s = 0
k = 0
for item in sorted(glob.glob(im_dir)):

    k += 1
    name = item.split('/')[-1]
    im1 = Image.open(item).convert('RGB')
    im2 = Image.open(label_dir + name).convert('RGB')

    (h, w) = im2.size
    im1 = im1.resize((h, w))

    im1 = np.array(im1)
    im2 = np.array(im2)

    psnr_score = psnr(im1, im2)
    ssim_score = ssim(im1, im2, multichannel=True)
    print(item, ssim_score)
    p += psnr_score
    s += ssim_score
print(p/k)
print(s/k)


import time
from collections import defaultdict
from multiprocessing import Pool
from typing import AbstractSet, Dict, List, Optional, Union

import numpy as np
from PIL import Image
from scalabel.common.parallel import NPROC
from scalabel.common.typing import NDArrayU8
from scalabel.eval.result import OVERALL, Result, Scores, ScoresList
from scalabel.label.coco_typing import PanopticCatType
from tqdm import tqdm

from bdd100k.common.utils import reorder_preds

from ..common.bitmask import (
    bitmask_intersection_rate,
    gen_blank_bitmask,
    parse_bitmask,
)
from ..common.logger import logger
from ..label.label import labels

STUFF = "STUFF"
THING = "THING"

class PanSegResult(Result):
    """The class for panoptic segmentation evaluation results."""

    PQ: List[Dict[str, float]]
    SQ: List[Dict[str, float]]
    RQ: List[Dict[str, float]]
    N: List[Dict[str, int]]  # pylint: disable=invalid-name

    # pylint: disable=useless-super-delegation
    def __eq__(self, other: "PanSegResult") -> bool:  # type: ignore
        """Check whether two instances are equal."""
        return super().__eq__(other)

    def summary(
        self,
        include: Optional[AbstractSet[str]] = None,
        exclude: Optional[AbstractSet[str]] = None,
    ) -> Scores:
        """Convert the pan_seg data into a flattened dict as the summary."""
        summary_dict: Dict[str, Union[int, float]] = {}
        for metric, scores_list in self.dict(
            include=include, exclude=exclude
        ).items():
            summary_dict[f"{metric}/{STUFF}"] = scores_list[1][STUFF]
            summary_dict[f"{metric}/{THING}"] = scores_list[1][THING]
            summary_dict[metric] = scores_list[-1][OVERALL]
        return summary_dict

class PQStatCat:
    """PQ statistics for each category."""

    def __init__(self) -> None:
        """Initialize method."""
        self.iou: float = 0.0
        self.tp: int = 0  # pylint: disable=invalid-name
        self.fp: int = 0  # pylint: disable=invalid-name
        self.fn: int = 0  # pylint: disable=invalid-name

    def __iadd__(self, pq_stat_cat: "PQStatCat") -> "PQStatCat":
        """Adding definition."""
        self.iou += pq_stat_cat.iou
        self.tp += pq_stat_cat.tp
        self.fp += pq_stat_cat.fp
        self.fn += pq_stat_cat.fn
        return self

class PQStat:
    """PQ statistics for an image of the whole dataset."""

    def __init__(self) -> None:
        """Initialize the PQStatCat dict."""
        self.pq_per_cats: Dict[int, PQStatCat] = defaultdict(PQStatCat)

    def __getitem__(self, category_id: int) -> PQStatCat:
        """Get a PQStatCat object given category."""
        return self.pq_per_cats[category_id]

    def __iadd__(self, pq_stat: "PQStat") -> "PQStat":
        """Adding definition."""
        for category_id, pq_stat_cat in pq_stat.pq_per_cats.items():
            self.pq_per_cats[category_id] += pq_stat_cat
        return self

    def pq_average(
        self, categories: List[PanopticCatType]
    ) -> Dict[str, float]:
        """Calculate averatge metrics over categories."""
        pq, sq, rq, n = 0.0, 0.0, 0.0, 0
        for category in categories:
            category_id = category["id"]
            iou = self.pq_per_cats[category_id].iou
            tp = self.pq_per_cats[category_id].tp
            fp = self.pq_per_cats[category_id].fp
            fn = self.pq_per_cats[category_id].fn

            if tp + fp + fn == 0:
                continue
            pq += (iou / (tp + 0.5 * fp + 0.5 * fn)) * 100
            sq += (iou / tp if tp != 0 else 0) * 100
            rq += (tp / (tp + 0.5 * fp + 0.5 * fn)) * 100
            n += 1

        if n > 0:
            return {"PQ": pq / n, "SQ": sq / n, "RQ": rq / n, "N": n}
        return {"PQ": 0, "SQ": 0, "RQ": 0, "N": 0}

def pq_per_image(gt_path: str, pred_path: str = "") -> PQStat:
    """Calculate PQStar for each image."""
    gt_bitmask: NDArrayU8 = np.asarray(Image.open(gt_path), dtype=np.uint8)
    if not pred_path:
        pred_bitmask = gen_blank_bitmask(gt_bitmask.shape)
    else:
        pred_bitmask = np.asarray(Image.open(pred_path), dtype=np.uint8)

    gt_masks, gt_ids, gt_attrs, gt_cats = parse_bitmask(gt_bitmask)
    pred_masks, pred_ids, pred_attrs, pred_cats = parse_bitmask(pred_bitmask)

    gt_valids = np.logical_not(np.bitwise_and(gt_attrs, 3).astype(bool))
    pred_valids = np.logical_not(np.bitwise_and(pred_attrs, 3).astype(bool))

    ious, iofs = bitmask_intersection_rate(gt_masks, pred_masks)
    cat_equals = gt_cats.reshape(-1, 1) == pred_cats.reshape(1, -1)
    ious *= cat_equals

    max_ious = ious.max(axis=1)
    max_idxs = ious.argmax(axis=1)
    inv_iofs = 1 - iofs[gt_valids].sum(axis=0)

    pq_stat = PQStat()
    pred_matched = set()
    for i in range(len(gt_ids)):
        if not gt_valids[i]:
            continue
        cat_i = gt_cats[i]
        if max_ious[i] <= 0.5 or not pred_valids[max_idxs[i]]:
            pq_stat[cat_i].fn += 1
        else:
            pq_stat[cat_i].tp += 1
            pq_stat[cat_i].iou += max_ious[i]
            pred_matched.add(max_idxs[i])

    for j in range(len(pred_ids)):
        if not pred_valids[j] or j in pred_matched or inv_iofs[j] > 0.5:
            continue
        pq_stat[pred_cats[j]].fp += 1
    return pq_stat

def evaluate_pan_seg(
    gt_paths: List[str],
    pred_paths: List[str],
    nproc: int = NPROC,
    with_logs: bool = True,
) -> PanSegResult:
    """Evaluate panoptic segmentation with BDD100K format."""
    start_time = time.time()
    if with_logs:
        logger.info("evaluating...")
    pred_paths = reorder_preds(gt_paths, pred_paths)
    if nproc > 1:
        with Pool(nproc) as pool:
            pq_stats = pool.starmap(
                pq_per_image,
                tqdm(zip(gt_paths, pred_paths), total=len(gt_paths)),
            )
    else:
        pq_stats = [
            pq_per_image(gt_path, pred_path)
            for gt_path, pred_path in tqdm(
                zip(gt_paths, pred_paths), total=len(gt_paths)
            )
        ]
    pq_stat = PQStat()
    for pq_stat_ in pq_stats:
        pq_stat += pq_stat_

    if with_logs:
        logger.info("accumulating...")
    categories: List[PanopticCatType] = [
        PanopticCatType(
            id=label.id,
            name=label.name,
            supercategory=label.category,
            isthing=label.hasInstances,
            color=label.color,
        )
        for label in labels
    ]
    categories = categories[1:]
    categories_stuff = [
        category for category in categories if not category["isthing"]
    ]
    categories_thing = [
        category for category in categories if category["isthing"]
    ]
    basic_category_names = [category["name"] for category in categories]

    res_dict: Dict[str, ScoresList] = {}
    for category_name, category in zip(basic_category_names, categories):
        result = pq_stat.pq_average([category])
        for metric, score in result.items():
            if metric not in res_dict:
                res_dict[metric] = [{}, {}, {}]
            res_dict[metric][0][category_name] = score

    result = pq_stat.pq_average(categories_stuff)
    for metric, score in result.items():
        res_dict[metric][1][STUFF] = score
    result = pq_stat.pq_average(categories_thing)
    for metric, score in result.items():
        res_dict[metric][1][THING] = score
    result = pq_stat.pq_average(categories)
    for metric, score in result.items():
        res_dict[metric][2][OVERALL] = score

    t_delta = time.time() - start_time
    if with_logs:
        logger.info("Time elapsed: %0.2f seconds", t_delta)

    return PanSegResult(**res_dict)  # type: ignore