#!/usr/bin/env python
# coding: utf-8

# # Remarks
#
# * Data normalization
#     * Mobilenet expects data from -1 to 1
#         * Normalize Input Data or Include in Model
#         * TFLite Conversion must fit according to decision
#     * Ground Truth Data: for better inspection Data multiplied by 80. Undo the change in the Data Input Pipeline
# * Overview in Tutorials:
#     * tf.function
#     * Repeat addapted Version of using Build in methods for training, ...
#     * Save models using keras
#         * CaseNet first real model: check in implementation of Frey if a Layer needs to be written
#         * other Example: depth seperable dilated convolution,
# * Idea
#     * Loss
#         * Focal Loss: for imbalanced Data
#         * In general Loss: just now weight in each dependent on number of Edge Pixels

# # Libraries

# In[1]:

#!for a in /sys/bus/pci/devices/*; do echo 0 | sudo tee -a $a/numa_node; done

import tensorflow as tf
import numpy as np
import os
import time
from datetime import datetime
import sys
import matplotlib.pyplot as plt
import argparse

import DataProcessing.data_processing as data_processing
import Nets.backbones as backbones
import Nets.features as features
import Nets.losses as losses
import Nets.metrics as metrics
import Nets.visualize as visualize
import Nets.tools as tools

os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'

#np.set_printoptions(threshold=sys.maxsize)

# # Parser

# In[2]:

parser = argparse.ArgumentParser()

parser.add_argument('--model', type=str, required=False, default=None)
parser.add_argument('--data', type=str, required=False, default=None)

parser.add_argument('--bs', type=int, required=False, default=None)
parser.add_argument('--idx', type=int, required=False, default=None)
parser.add_argument('--epoch', type=int, required=False, default=None)
parser.add_argument('--noise', type=float, required=False, default=None)

parser.add_argument('--train_model', action='store_true', default=False)
parser.add_argument('--cache', action='store_true', default=False)
parser.add_argument('--save', action='store_true', default=False)
parser.add_argument('--sigmoid', action='store_true', default=False)
parser.add_argument('--focal', action='store_true', default=False)

parser.add_argument('--beta_upper', type=float, required=False, default=None)
parser.add_argument('--gamma', type=float, required=False, default=None)
parser.add_argument('--alpha', type=float, required=False, default=None)

file_name = None
try:
    file_name = __file__
except:
    print("Jupyter Notebook")

if file_name is None:
    args = parser.parse_args("")
    args.train_model = True
    args.cache = True
    #args.save = True
    args.save = False
    args.sigmoid = False
    args.focal = True
else:
    args = parser.parse_args()

# # Options

# In[3]:

# Generall Parameters
MODEL= 'CASENET_FOCAL_LOSS_0.7_g2_a2_random' if args.model is None else args.model
DATA= 'SceneNetFloorTiledTextureRandom' if args.data is None else args.data
#DATA = 'RealRed'
TRAIN_DS ='Train'
TEST_DS = 'Test'
HALF = True

# Dataset Loading Parameters
IMG_SIZE_HEIGHT = 1280
IMG_SIZE_WIDTH = 720
NUM_CLASSES = 3
MAX_IMG_TRAIN = 500
MAX_IMG_TEST = 300
SEED = None
BATCH_SIZE = 3 if args.bs is None else args.bs
CACHE = args.cache
NOISE_STD = 0.0 if args.noise is None else args.noise

# Model Parameters
BACKBONE = "RESNet50"
BACKBONE_OUTPUT = [0,1,2,4]
BACKBONE_WEIGHTS = "imagenet"
ALPHA = 1
FINE_TUNING = False
FINE_TUNE_EPOCHS = 10
TRAINABLE_IDX = 0 if args.idx is None else args.idx # (3-1), as indexing starts at 0
EPOCHS = 80 if args.epoch is None else args.epoch
SAVE = args.save
TRAIN_MODEL = args.train_model

#Model Callback
MODEL_SAVE_EPOCH_FREQ = 5
DEL_OLD_CHECKPOINTS = False
TENSORBOARD = False
DEL_OLD_TENSORBOARD = True

# LOSS
weighted_multi_label_sigmoid_edge_loss = args.sigmoid
focal_loss = args.focal

beta_upper = 0.7 if args.beta_upper is None else args.beta_upper
beta_lower = 1.0 - beta_upper
gamma=2.0 if args.gamma is None else args.gamma
alpha=2.0 if args.alpha is None else args.alpha
class_weighted = True
weighted_beta=True

# All Pixels have been labeled correctly and thus we don't need to account shifted labels
# and a protection band around the labels for the calculation of the metrics

# In the work of Frey he mentioned that state of the Art ? is 2% of diagonal.
# He takes 1%, I sugest to take a threshold of 3 Pixels.
#I don't think that I made more then 3 Pixel mistake in labeling and tracking. Thus this is 0.4%
THRESHOLD_EDGE_WIDTH_REAL = 2

# Data Augmentation:
aug_param = {"contrast_factor": 0.8, "brightness": 0.2, "hue": 0.05, "saturation": 0.8, "gaussian_value": 0.015,
            "value": 0.1, "strength_spot": 0.5, "blur": False, "sigma": 1.0}

#TESTING
test = False
if test:
    EPOCHS = 10
    MAX_IMG_TRAIN = 18
    MAX_IMG_TEST = 3

# # Load Dataset, Preprocess Images and Dataset

# In[5]:

tf.random.set_seed(SEED)

paths, files = data_processing.path_definitions(HALF, MODEL, DATA, TRAIN_DS, TEST_DS, make_dirs=True)

data_processing.clean_model_directories(paths, DEL_OLD_CHECKPOINTS, DEL_OLD_TENSORBOARD)

if TRAIN_MODEL:

    rng = tf.random.Generator.from_seed(123, alg='philox')

    train_ds, img_count_train = data_processing.load_dataset(paths,"TRAIN", IMG_SIZE_HEIGHT, IMG_SIZE_WIDTH, HALF,
                                                             MAX_IMG_TRAIN, noise_std=NOISE_STD)
    train_ds = data_processing.dataset_processing(train_ds, cache=CACHE, shuffle=True, batch_size=BATCH_SIZE, prefetch=True,
                                                  img_count=img_count_train, augment=True, rng=rng, aug_param=aug_param)

test_ds, img_count_test = data_processing.load_dataset(paths,"TEST", IMG_SIZE_HEIGHT, IMG_SIZE_WIDTH, HALF,
                                                       MAX_IMG_TEST, noise_std=None)
test_ds = data_processing.dataset_processing(test_ds, cache=CACHE, shuffle=False, batch_size=BATCH_SIZE, prefetch=True,
                                             img_count=img_count_test)

# In[6]:

DATA_REAL = 'RealRed'
TRAIN_REAL = 'Train'
TEST_REAL = 'Test'
TEST_HARD_REAL = 'Test Hard'
IMG_ONLY_REAL = 'Img Only'
BS_REAL = 8

paths_real, files_real = data_processing.path_definitions(HALF, MODEL, DATA_REAL, TRAIN_REAL, TEST_REAL, TEST_HARD_REAL, IMG_ONLY_REAL, make_dirs=False)

test_real_ds, img_count_test_real = data_processing.load_dataset(paths_real,"TEST", IMG_SIZE_HEIGHT, IMG_SIZE_WIDTH, HALF, MAX_IMG_TEST)
test_real_ds = data_processing.dataset_processing(test_real_ds, cache=False, shuffle=False, batch_size=BS_REAL, prefetch=False, img_count = img_count_test_real)

# In[8]:

#for image,label in train_ds.take(1):
#        image, mask = image,label
#plt.figure(figsize=(10,20))
#plt.imshow(tf.keras.preprocessing.image.array_to_img(image[0, :, :, :]))

#for image,label in train_ds.take(1):
#    sample_image, sample_mask = image,label
#
#visualize.plot_images(images=sample_image, labels=sample_mask, predictions=None, batch_size=3)

# # Model

# In[7]:

if weighted_multi_label_sigmoid_edge_loss:
    loss = lambda y_true, y_pred : losses.weighted_multi_label_sigmoid_loss(y_true,y_pred,beta_lower=beta_lower,beta_upper=beta_upper, class_weighted=class_weighted)
elif focal_loss:
    loss = lambda y_true, y_pred : losses.focal_loss_edges(y_true, y_pred, gamma=gamma, alpha=alpha, weighted_beta=weighted_beta,beta_lower=beta_lower,beta_upper=beta_upper, class_weighted=class_weighted)
else:
    raise ValueError("either FocalLoss or WeightedMultiLabelSigmoidLoss must be True")

# In[8]:

if TRAIN_MODEL:
    backbone, output_names = backbones.get_backbone(name=BACKBONE,weights=BACKBONE_WEIGHTS,
                                              height=IMG_SIZE_HEIGHT,width=IMG_SIZE_WIDTH,
                                              alpha=ALPHA, output_layer = BACKBONE_OUTPUT,
                                                            trainable_idx = TRAINABLE_IDX)

    upsample_side_1 = tf.keras.layers.Conv2D(1, kernel_size=1, strides=(1, 1), padding='same')(backbone.output[0])
    upsample_side_2 = features.side_feature_casenet(backbone.output[1],channels=1,kernel_size_transpose=4,stride_transpose=2)
    upsample_side_3 = features.side_feature_casenet(backbone.output[2],channels=1,kernel_size_transpose=8,stride_transpose=4)
    #upsample_side_5 = tf.image.resize(backbone.output[3],(int(IMG_SIZE_HEIGHT/16),int(IMG_SIZE_WIDTH/16)))
    upsample_side_5 = features.side_feature_casenet(backbone.output[3],channels=NUM_CLASSES,kernel_size_transpose=16,stride_transpose=8,name='side5')

    side_outputs = [upsample_side_1,upsample_side_2,upsample_side_3,upsample_side_5]
    concat = features.shared_concatenation(side_outputs,NUM_CLASSES)

    output = features.fused_classification(concat,NUM_CLASSES,name="output")

    model = tf.keras.Model(inputs = backbone.input, outputs = [output,upsample_side_5])

    # tf.keras.utils.plot_model(model,show_shapes = True,to_file = 'h.png')

# # Compile and Train Model

# In[9]:

if TENSORBOARD:
    get_ipython().run_line_magic('load_ext', 'tensorboard')
    get_ipython().run_line_magic('tensorboard', '--logdir /home/david/SemesterProject/Models/CASENet/logs')

# In[10]:

if TRAIN_MODEL:
    # learning rate schedule
    base_learning_rate = 0.0005
    end_learning_rate = 0.0001
    decay_step = np.ceil(img_count_train / BATCH_SIZE)*EPOCHS
    lr_schedule = tf.keras.optimizers.schedules.PolynomialDecay(base_learning_rate,decay_steps = decay_step,end_learning_rate = end_learning_rate, power = 0.9)

    frequency = int(np.ceil(img_count_train / BATCH_SIZE)*MODEL_SAVE_EPOCH_FREQ)+1

    logdir = os.path.join(paths['TBLOGS'], datetime.now().strftime("%Y%m%d-%H%M%S"))
    callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath = paths["CKPT"]+ "/ckpt-loss={val_loss:.2f}-epoch={epoch:.2f}-f1={val_f1:.4f}",
                                                    save_weights_only=False,save_best_only=False,monitor="val_f1",verbose=1,save_freq= 'epoch', period=5),
                tf.keras.callbacks.TensorBoard(log_dir=logdir,histogram_freq=1)]

    # compile model
    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),
                  loss=loss,
                  metrics={'output': [metrics.BinaryAccuracyEdges(threshold_prediction=0),
                                      metrics.F1Edges(threshold_prediction=0, threshold_edge_width=0)]})

    history = model.fit(train_ds, epochs=EPOCHS, validation_data=test_real_ds, callbacks=callbacks)

# In[16]:

model_ckpt = os.listdir(paths['CKPT'])

f1_max = 0
for ckpt_name in model_ckpt:
    if float(ckpt_name[-4:]) > f1_max:
        f1_max = float(ckpt_name[-4:])
        model_path = paths['CKPT']+"/"+ckpt_name

        print(model_path)

custom_objects = {"BinaryAccuracyEdges": metrics.BinaryAccuracyEdges,
                  "F1Edges": metrics.F1Edges,
                  "<lambda>":loss}

model = tf.keras.models.load_model(model_path, custom_objects=custom_objects)

# # Plot Results

# In[17]:

if TRAIN_MODEL:
    plot_losses = ["loss", "output_loss"]
    plot_metrics = ["output_accuracy_edges", "f1", "recall", "precision"]

    path = os.path.join(paths["FIGURES"],"training.svg")

    visualize.plot_training_results(res=history.history, losses=plot_losses, metrics=plot_metrics, save=SAVE, path=path)

# In[18]:

### Maximum F1 Score:
if not TRAIN_MODEL:
    step_width = 0.05
    threshold_range = [0.05,0.95]
    threshold_array = np.arange(threshold_range[0],threshold_range[1]+step_width,step_width)
    threshold_array = np.array([0.025, 0.1, 0.2,0.3,0.4,0.45,0.5,0.55,0.6,0.7,0.8, 0.9, 0.975])

    path_metrics_evaluation_plot = os.path.join(paths["FIGURES"],"threshold_metrics_evaluation_test_ds.svg")
    threshold_f1_max = visualize.plot_threshold_metrics_evaluation_class(model=model,
                                                                         ds=test_ds,
                                                                         num_classes=NUM_CLASSES,
                                                                         threshold_array=threshold_array,
                                                                         threshold_edge_width=0, save=SAVE,
                                                                         path=path_metrics_evaluation_plot)

# In[19]:

if not TRAIN_MODEL:
    i = 0
    for img, label in test_ds.take(4):
        img, label = img, label

        threshold = 0.5

        predictions = model.predict(img)
        predictions = tools.predict_class_postprocessing(predictions[0], threshold=threshold)

        path = os.path.join(paths["FIGURES"],"img_test_threshold_{}_{}".format(threshold,i))
        visualize.plot_images(images=img, labels=label, predictions=predictions, save=SAVE, path=path, batch_size=3)

        threshold = threshold_f1_max
        path = os.path.join(paths["FIGURES"],"img_test_ods_{}".format(i))
        visualize.plot_images(images=img, labels=label, predictions=predictions, save=SAVE, path=path, batch_size=3)

        i += 1

# # Fine Tuning

# In[ ]:

if FINE_TUNING and TRAIN_MODEL:

    # Fine-tune from this layer onwards
    fine_tune_output = output_names[1-1]

    model.trainable = True

    # Freeze all the layers before the `fine_tune_at` layer:
    for submodel in model.layers:
        if submodel.name == "base_model":
            for layer in submodel.layers:
                layer.trainable = False
                if layer.name == fine_tune_output:
                    break

    total_epochs =  EPOCHS + FINE_TUNE_EPOCHS

    base_learning_rate = 0.00001
    end_learning_rate =  0.00001
    decay_step = np.floor(img_count_train / BATCH_SIZE)*FINE_TUNE_EPOCHS
    lr_schedule = tf.keras.optimizers.schedules.PolynomialDecay(
        base_learning_rate,decay_steps = decay_step,end_learning_rate = end_learning_rate, power = 0.9)

    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),
                  loss=loss,
                  metrics={'output': [metrics.BinaryAccuracyEdges(threshold_prediction=0),
                                      metrics.F1Edges(threshold_prediction=0, threshold_edge_width=0)]})

    history_fine = model.fit(train_ds, epochs=total_epochs,
                               initial_epoch=history.epoch[-1]+1,validation_data=train_ds.take(1),
                               callbacks=callbacks)

    plot_losses = ["loss", "output_loss"]
    plot_metrics = ["output_accuracy_edges", "f1", "recall", "precision"]

    path = os.path.join(paths["FIGURES"],"fine_tuning_training.svg")

    visualize.plot_training_results(res=history.history, res_fine = history_fine.history,
                                losses=plot_losses, metrics=plot_metrics, save=SAVE, path=path, epochs=EPOCHS)

    path_metrics_evaluation_plot = os.path.join(paths["FIGURES"],"threshold_metrics_evaluation__fine_tune_test_ds.svg")
    visualize.plot_threshold_metrics_evaluation(model=model, ds=test_ds, threshold_array=threshold_array,
                                        threshold_edge_width=0, save=SAVE, path=path_metrics_evaluation_plot,
                                        accuracy_y_lim_min = 0.9)

    for img, label in test_ds.take(1):
        img, label = img, label

    predictions = model.predict(img)
    predictions = tools.predict_class_postprocessing(predictions[0], threshold = 0.5)

    path = os.path.join(paths["FIGURES"],"fine_tuning_images_0,5")
    visualize.plot_images(images=img, labels=label, predictions=predictions, save=SAVE, path=path, batch_size=3)

# # Evaluate on Test DS of Real Images

# In[ ]:

DATA_REAL = 'RealRed'
TRAIN_REAL = 'Train'
TEST_REAL = 'Test'
TEST_HARD_REAL = 'Test Hard'
IMG_ONLY_REAL = 'Img Only'
BS_REAL = 8

paths_real, files_real = data_processing.path_definitions(HALF, MODEL, DATA_REAL, TRAIN_REAL, TEST_REAL, TEST_HARD_REAL, IMG_ONLY_REAL, make_dirs=False)

test_real_ds, img_count_test_real = data_processing.load_dataset(paths_real,"TEST", IMG_SIZE_HEIGHT, IMG_SIZE_WIDTH, HALF, MAX_IMG_TEST)
test_real_ds = data_processing.dataset_processing(test_real_ds, cache=False, shuffle=False, batch_size=BS_REAL, prefetch=False, img_count = img_count_test_real)

# ## Metrics Evaluation

# In[ ]:

if not TRAIN_MODEL:
    step_width = 0.025
    threshold_range = [0.025, 0.975]
    threshold_array = np.arange(threshold_range[0],threshold_range[1]+step_width,step_width)

    path_metrics_evaluation_plot = os.path.join(paths["FIGURES"],"threshold_metrics_evaluation_test_real_edge_threshold_{:.1f}.svg".format(0))
    threshold_f1_max = visualize.plot_threshold_metrics_evaluation_class(model=model, ds=test_real_ds,
                                                                   num_classes = NUM_CLASSES,
                                                                   threshold_array=threshold_array,
                                                                   threshold_edge_width=0, save=SAVE,
                                                                   path=path_metrics_evaluation_plot)

# ## Visual Results

# In[ ]:

if not TRAIN_MODEL:
    for img, label in test_real_ds.take(1):
        img, label = img, label

    threshold = 0.5

    predictions = model.predict(img)
    predictions = tools.predict_class_postprocessing(predictions[0], threshold=threshold)

    path = os.path.join(paths["FIGURES"],"images_test_real_threshold_{:.2f}".format(threshold))
    visualize.plot_images(images=img, labels=label, predictions=predictions, save=SAVE, path=path, batch_size=8)

    threshold = threshold_f1_max

    predictions = model.predict(img)
    predictions = tools.predict_class_postprocessing(predictions[0], threshold=threshold)

    path = os.path.join(paths["FIGURES"],"images_test_real_threshold_ods")
    visualize.plot_images(images=img, labels=label, predictions=predictions, save=SAVE, path=path, batch_size=8)

# # Save Model

# In[20]:

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),
                  loss=loss,
                  metrics={'output': [metrics.BinaryAccuracyEdges(threshold_prediction=0),
                                      metrics.F1Edges(threshold_prediction=0, threshold_edge_width=0)]})

if SAVE:
    model.save(paths["MODEL"])

    custom_objects = {"BinaryAccuracyEdges": metrics.BinaryAccuracyEdges,
                      "F1Edges": metrics.F1Edges,
                      "<lambda>":loss}

    model = tf.keras.models.load_model(paths["MODEL"], custom_objects=custom_objects)

# # Plot Other, Additional Data.

# # Addtional Elements to Consider in other Projects
#
# * Data augmentation for small datasets

# In[ ]:


from django.contrib.auth.views import LoginView
from django.urls import path

from cinema.views import (
    IndexView,
    MovieDetailView,
    SessionDetailView,
    CreateUserView,
    TicketListView,
    LogoutView,
)

urlpatterns = [
    path("", IndexView.as_view(), name="index"),
    path("accounts/login/", LoginView.as_view(), name="login"),
    path("accounts/logout/", LogoutView.as_view(), name="logout"),
    path("accounts/register/", CreateUserView.as_view(), name="register"),
    path("movie/<int:pk>/", MovieDetailView.as_view(), name="movie-detail"),
    path(
        "movie/session/<int:pk>/",
        SessionDetailView.as_view(),
        name="movie-session-detail",
    ),
    path("orders/", TicketListView.as_view(), name="ticket-listview"),
]

app_name = "cinema"


import unittest
import data

class TestData(unittest.TestCase):
    def test_batch(self):
        sequences = [el.split() for el in \
                     ['I love it', 'My name is Earl', 'Whose line is it anyway', 'As you like it', 'Everybody']]
        words = list(set([el for lis in sequences for el in lis]))
        word_index = {wd: ix for ix, wd in enumerate(words)}
        word_index[data.PAD_TOKEN] = 170
        word_index[data.START_OF_VERSE_TOKEN] = 171
        word_index[data.END_OF_VERSE_TOKEN] = 172
        dataset, original_sequence_lengths = data.batch(sequences, 2, word_index)
        self.assertEqual(3, len(dataset))
        self.assertTrue(all([len(el) <= 2 for el in dataset]))
        self.assertTrue(all([len(seq) == 6 for seq in dataset[0]]))
        self.assertTrue(all([len(seq) == 7 for seq in dataset[1]]))
        self.assertTrue(all([all([seq[0] == 171 for seq in b]) for b in dataset]))
        self.assertTrue(all([all([seq[-1] in (170, 172) for seq in b]) for b in dataset]))
        self.assertEqual(172, dataset[0][1][-2])
        self.assertEqual(6, original_sequence_lengths[1][1])

    def test_batch_sorted(self):
        sequences = [el.split() for el in \
                     ['I love it', 'My name is Earl', 'Whose line is it anyway', 'As you like it', 'Everybody']]
        words = list(set([el for lis in sequences for el in lis]))
        word_index = {wd: ix for ix, wd in enumerate(words)}
        index_word = {ix: wd for wd, ix in word_index.items()}
        word_index[data.PAD_TOKEN] = 170
        word_index[data.START_OF_VERSE_TOKEN] = 171
        word_index[data.END_OF_VERSE_TOKEN] = 172
        dataset, _ = data.batch(sequences, 2, word_index)
        self.assertEqual('My', index_word[dataset[0][0][1]])

    def test_pad_batch(self):
        sequences = [el.split() for el in ['I love it', 'My name is Earl']]
        padded = data.pad_batch(sequences)
        self.assertEqual(2, len(padded))
        self.assertTrue(all([len(el) == 4 for el in padded]))
        self.assertEqual(data.PAD_TOKEN, padded[0][-1])

    def test_join_texts(self):
        texts = ['something', 'else', 'is', 'here']
        self.assertEqual(
            '\n\nsomething else is here EOS',
            data.join_texts(texts, prompt='\n\n', separator=' ', eot_token='EOS')
        )

    def test_join_by_hierarchy(self):
        comments = """# language_name:        English
# closest_ISO_639-3:    eng
# ISO_15924:            Latn
# year_short:           1997
# year_long:
# vernacular_title:
# english_title:        World English Bible
# URL:                  http://biblehub.com/web/matthew/1.htm
# copyright_short:      Public Domain 1997
# copyright_long:
# notes:
"""
        comment_lines = comments.split('\n')
        lines = comment_lines + ['40001001\tFirst verse', '40001002\tSecond verse', '40002001\tNext chapter', '41001001\tAnother book',
                 '67001001\tAnother testament']
        bible = data.parse_pbc_bible_lines(lines, True, 'eng')
        _, by_testament, by_book, by_chapter, _ = bible.join_by_toc()
        self.assertTrue('old' not in by_testament)
        self.assertEqual('First verse_Second verse_Next chapter_Another book'.split('_'),
                         by_testament['new'])
        self.assertEqual(['Another testament'],
                         by_testament['apocryphal'])
        self.assertEqual('First verse_Second verse_Next chapter'.split('_'), by_book[40])
        self.assertEqual(['Another book'], by_book[41])
        self.assertEqual('Another testament'.split('_'), by_book[67])
        self.assertEqual('First verse_Second verse'.split('_'), by_chapter[40001])
        self.assertEqual('Next chapter'.split('_'), by_chapter[40002])
        self.assertEqual('Another book'.split('_'), by_chapter[41001])
        self.assertEqual('Another testament'.split('_'), by_chapter[67001])

    def test_to_dictionaries_repeated_commented_out_lines(self):
        comment_lines = []
        content_lines = [(1, "some lines", False),
                         (2, "some other line", False),
                         (3, "this line is commented out", True),
                         (3, "again but with the same ID", True),
                         (4, "some other line", False)]
        _, _, hidden_content = data.PbcBible.to_dictionaries(comment_lines, content_lines)
        self.assertEqual({3: 'again but with the same ID'}, hidden_content)

if __name__ == "__main__":
    unittest.main()


import numpy as np
import pytest

import torch
from torch.utils.data import DataLoader

from ml_recon.utils.collate_function import collate_fn
from ml_recon.dataset.fastMRI_dataset import FastMRIDataset
from ml_recon.dataset.self_supervised_decorator import UndersampleDecorator

ACS_LINES = 10
@pytest.fixture
def dataset(get_data_dir, scope='session') -> UndersampleDecorator:
    torch.manual_seed(0)
    dataset = FastMRIDataset(get_data_dir, build_new_header=True)
    undersample_dataset = UndersampleDecorator(dataset, acs_lines=ACS_LINES)
    return undersample_dataset

@pytest.fixture
def get_data_dir() -> str:
    return '/home/kadotab/projects/def-mchiew/kadotab/Datasets/t1_fastMRI/16_chans/multicoil_train/'

def test_slice_load(dataset):
    data = next(iter(dataset))
    assert len(data) == 4

def test_undersampled_slice(dataset):
    data = next(iter(dataset))

    phase_encode_size = data[0].shape[-1]
    center = np.floor(phase_encode_size/2).astype(int)
    acs = data[0][..., center - ACS_LINES//2:center + ACS_LINES//2]
    assert (acs != 0).all()
    assert acs.shape[-1] == 10

# Test if we are able to batch slices.
def test_undersampled_slice_batching(dataset):
    dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=5)
    data = next(iter(dataloader))
    assert data[0].shape[0] == 5
    assert data[0].ndim == 4

def test_non_deterministic(dataset):

    data1 = dataset[0]
    data2 = dataset[0]

    assert ((data1[0] != 0) != (data2[0] != 0)).any()

def test_non_deterministic_between_slices(dataset):

    data1 = dataset[0]
    data2 = dataset[1]

    assert ((data1[1] != 0) != (data2[1] != 0)).any()

def test_undersampling(dataset):

    doub_under, under, k_space, _ = dataset[0]

    assert (doub_under == 0).sum() > (under == 0).sum()
    assert (under == 0).sum() > (k_space == 0).sum()


from api_yamdb.settings import REGEX_USERNAME
from django.core.exceptions import ValidationError

class ValidateUsername:
    """
    Валидатор для username.
    """
    def validate_username(self, username):
        if not REGEX_USERNAME.fullmatch(username):
            raise ValidationError(
                f'Некорректные символы в username: {REGEX_USERNAME}'
            )
        if username == 'me':
            raise ValidationError('Ник "me" нельзя регистрировать!')
        return username


import json

from tqdm import tqdm

from anlp_project.datasets.europarl import EuroParlRaw

def main():
    """
    Converts dataset to jsonlines format

    needed for baseline++ training
    """
    dataset = EuroParlRaw()
    with open("dataset.jsonl", "w") as f:
        for de, en in tqdm(dataset):
            f.write(json.dumps({"translation": {"en": en, "de": de}}) + "\n")

if __name__ == "__main__":
    main()


import nidaqmx
import torch
import numpy as np
import time
from nidaqmx.constants import AcquisitionType, TerminalConfiguration
from collections import deque
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from LSTMmodel import GaitLSTM
from utils import buffer_data

# The number of IMU (It can only 1, 2, 4)
num_IMU = 1

Gyro = True
Acc = False

# Define ML model parameter
input_size = num_IMU*(Gyro+Acc)*3
hidden_size = 128
num_layers = 3
output_size = 1
dropout_prob = 0.4

# Load the trained LSTM model
model = GaitLSTM(input_size, hidden_size, num_layers, output_size, dropout_prob)
model.load_state_dict(torch.load('LSTMmodel_'+str(num_IMU)+'IMU_gyro.pth'))
model.eval()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)

# Buffer to store the incoming data
window_size = 200
buffer = deque(maxlen=window_size)

# Replace device_name is the name of your device, and channel_name is the name of your input channel.
device_name = "Dev1"
channel_names = ["ai7", "ai8", "ai9"]

ylabel = ['Gyro X', 'Gyro Y', 'Gyro Z']

gyro_range = 2000 # Choose the range you are using: 250 dps, 500 dps, 1000 dps, 2000 dps
analog_output_range = 5
gyro_sensitivity = gyro_range / (2 * analog_output_range)

# Create a task
with nidaqmx.Task() as task:
    # Configure the analog input channel
    for channel_name in channel_names:
        task.ai_channels.add_ai_voltage_chan(f'{device_name}/{channel_name}',
                                             terminal_config=TerminalConfiguration.RSE)

    # Set the sample rate and number of samples
    sample_rate = 1000  # 1000 samples per second
    num_samples = 100  # Acquire 10 samples
    task.timing.cfg_samp_clk_timing(sample_rate, samps_per_chan=num_samples,
                                     sample_mode=AcquisitionType.CONTINUOUS)

    while True:
        try:
            start = time.time()

            data = np.array(task.read(number_of_samples_per_channel=num_samples, timeout=1)) * gyro_sensitivity

            # Preprocess the incoming data
            scaler = StandardScaler()
            preprocessed_data = scaler.fit_transform(data)

            # Buffer the preprocessed data and create segments when the buffer is full
            segment = buffer_data(buffer, preprocessed_data, window_size)

            if segment is not None:
                # Reshape the segment to match the input shape of the LSTM model
                segment = segment.reshape(1, window_size, -1)

                # Convert the segment to a PyTorch tensor
                input_data = torch.tensor(segment, dtype=torch.float32).to(device)
                # Estimate gait speed using the LSTM model
                with torch.no_grad():
                    estimated_speed = model(input_data).item()

                print(f"Estimated gait speed: {estimated_speed}")

        except KeyboardInterrupt:
            print("Stopping data acquisition...")
            task.stop()
            break


import numpy as np
import scipy.sparse as sps
try:
    from petsc4py import PETSc
except ImportError:
    print("No PETSc. Bicstab solver wont work")

try:
    import pyamg
except ImportError:
    print("No pyamg. amg solver wont work")

try:
    import pypardiso
except ImportError:
    print("No pypardis. pardiso solver won't work")

def amg(A, b, tol=1e-10):
    """Solve with AMG."""
    B = None  # no near-null spaces guesses for SA

    # use AMG based on Smoothed Aggregation (SA) and display info
    mls = pyamg.smoothed_aggregation_solver(A, B=B)
    print(mls)
    # # Solve Ax=b with no acceleration ('standalone' solver)
    # standalone_residuals = []
    # x = mls.solve(b, tol=1e-10, accel=None, residuals=standalone_residuals)

    # Solve Ax=b with Conjugate Gradient (AMG as a preconditioner to CG)
    residuals = []
    x = mls.solve(b, tol=tol, accel="cg", residuals=residuals)
    if residuals[-1] > 100 * tol:
        Warning("Iterative solver failed. Falling back to direct solver")
        return umfpack(A, b)
    print("Solved linear system with AMG. Residual is: {}".format(residuals[-1]))
    return x

def pardiso(A, b):
    "Solve with pardiso"
    return pypardiso.spsolve(A, b)

def gmres(A, b, x0, tol=1e-10):
    "Solve with gmres"
    M_iLU = sps.linalg.spilu(A, fill_factor=20, drop_tol=1e-5)
    M = sps.linalg.LinearOperator(A.shape, M_iLU.solve)

    def callback(res):
        print("Gmres residual: {}".format(res))

    x, info = sps.linalg.gmres(
        A, b, x0=x0, M=M, tol=tol, maxiter=200, callback=callback
    )
    if info != 0:
        Warning("Iterative solver failed. Falling back to direct solver")
        return umfpack(A, b)
    return x

def bicstab(A, b, tol=1e-10, gb=None):
    "Solve with bicstab"
    if gb is None:
        P = sps.identity(A.shape[0])
    else:
        nc = gb.num_cells()
        p_idx = np.arange(0, nc, 1)
        c_idx = np.arange(nc, 2 * nc, 1)
        col = np.ravel(np.vstack((p_idx, c_idx)), order='F')
        row = np.arange(0, 2 * nc)
        data = np.ones(2 * nc, dtype=int)
        P = sps.coo_matrix((data, (row, col)))

#    Ao = A.copy()
    A = P * A * P.T
    if A.format != "csr":
        A = A.tocsr()

    M = PETSc.Mat().createAIJ(size=A.shape,
                          csr=(A.indptr, A.indices,
                               A.data))
    x, rhs = M.getVecs()
    ksp = PETSc.KSP()
    ksp.create(PETSc.COMM_WORLD)
    ksp.setType('bcgs')
    ksp.getPC().setType('ilu')
    rhs.setArray(P * b)
    ksp.setOperators(M)
    ksp.setTolerances(1e-4, tol)
    ksp.solve(rhs, x)

#    print("linear it: ", ksp.getIterationNumber())
#    print("converged reason: ", ksp.getConvergedReason())
    return P.T * x.getArray()

def umfpack(A, b):
    "Solve with umfpack"
    if A.nnz > 500000:
        A.indices = A.indices.astype(np.int64)
        A.indptr = A.indptr.astype(np.int64)
    return sps.linalg.spsolve(A, b, use_umfpack=True)

def superlu(A, b):
    "Solve with superlu"
    return sps.linalg.spsolve(A, b)


# coding=utf-8
import codecs
import os
import pymssql
import sys
import uuid
from com.database.ConnectDataBase import ConnectionDatabase

# 设置默认编码
from xml.etree import ElementTree

import xlrd

'''
    数据库连接
'''
def getConnect_new():
    __conn = ConnectionDatabase("localhost", "sa", "11111", "EpointOATest3")
    return __conn

def getConnect_old():
    __conn = ConnectionDatabase("localhost", "sa", "11111", "oa_old")
    return __conn

##########################################################
#############遍历目录，读取xml文件，导入附件信息
##########################################################

# 遍历指定目录
def eachfile(filepath, sql, outputfile, state, count, conn):
    pathdir = os.listdir(filepath)
    for dir in pathdir:
        child = os.path.join('%s\%s' % (filepath, dir))
        if os.path.isfile(child):
            if child.find("basicInfo") != -1:
                readfile(child, sql, outputfile, count, state, conn)
            continue
        eachfile(child, sql, outputfile, state, count, conn)

# 遍历结果
def readfile(filenames, sql, filepath, count, state, conn):
    with codecs.open(filenames, 'r', encoding='gbk') as fp:
        text = fp.read().replace('<?xml version="1.0" encoding="GBK"?>', '<?xml version="1.0" encoding="UTF-8"?>')

    element = ElementTree.fromstring(text.encode('utf-8'))

    for doc in element:
        # 定义参数
        clientguid = ""
        for key in doc.attrib:
            clientguid = doc.attrib[key]
            filepath += (clientguid + "/")
        for items in doc:
            # 定义参数
            attachguid = uuid.uuid1()
            attachname = ""
            contenttype = ""
            # 附件
            storagetype = "NasShareDirectory"
            # attachtag = "leaderApprove_feedback"
            attachtag = "wd25_attach"

            if parse_xml_node(items):
                attachname = parse_xml_node(items)

            if os.path.splitext(attachname)[-1]:
                contenttype = os.path.splitext(attachname)[-1]
            # 插入附件表
            params = (attachguid, attachname, contenttype, clientguid, attachtag, filepath, storagetype, clientguid)
            # 过滤参数为空的数据
            if params is None:
                continue
            # 执行sql语句
            try:
                effect = conn.mssql_exe_sql(sql, params)
                if effect:
                    count += 1
                    print("导入第" + str(count) + "条成功*********" + attachname)
                else:
                    print("sql语句执行失败")

            except Exception as e:
                print(e)
                print(sql%params)
                return

# 解析节点
def parse_xml_node(node):
    if len(node.getchildren()) == 0:
        return node.text if node.text is not None else ''
    else:
        node_dict = {}
        for child in node.getchildren():
            if child.tag in node_dict.keys():
                if not isinstance(node_dict[child.tag], list):
                    node_dict[child.tag] = [node_dict[child.tag]]
                node_dict[child.tag].append(parse_xml_node(child))
            else:
                node_dict[child.tag] = parse_xml_node(child)
        return node_dict

def handleFile():
    conn = getConnect_old()
    count = 0
    # 通知
    type="leaderApprove"
    # filenames = "F:\松江OA\OA数据解析\老OA数据\附件\领导批示反馈"
    filenames = "F:\松江OA\OA数据解析\收文数据\收文管理"
    oupputfile = "D:\OA9Attach\BigFileUpLoadStorage/wd25/"

    # 发文
    # type="fawen"
    # filenames = "E:\\work\\project\\2018\\importdata\\fawen\\BigFileUpLoadStorage"
    # oupputfile="D:\OA9Attach\BigFileUpLoadStorage/fawen/"

    sql = '''
        insert into Frame_AttachInfo_wd25(attachguid,attachfilename,contenttype,CliengGuid,CliengTag,filepath,storagetype,attachstorageguid)
            values(%s,%s,%s,%s,%s,%s,%s,%s)
    '''
    eachfile(filenames, sql, oupputfile, type, count, conn)

    # 关闭连接
    conn.commitData()
    conn.closeConn()

if __name__ == '__main__':
    handleFile()


import sys
sys.path.append('/Users/me/_Nutbox/UniversalMolecularSystem')

# The main function of this module, TopologyMatching, seeks to solve the problem of matching atoms between two
# identical molecules with different atom ordering.
from UniversalMolecularSystem import *
from collections import deque

class SolvingStatus:
    nAtoms: []   # number of atoms in each molecule, an array of length 2
    visited_order: [[]]   # 2 lists recording visited atoms and their order in each mole
                    # for example, visited = [ [0,3],[1,2] ] means that atoms 0,3 and 1,2 are visited in each molecule
                    # Here the word 'visited' also means that the atoms in two molecules are matched with each other
                    # In the above example, it means 0-3, 1-2 matching in these two molecules.
    to_visit: [deque] # 2 deques recording the to-be visited atoms in both atoms. Used for the BFS in each mole
    visited_or_in_queue: [[]]  # 2*NAtoms bool matrix indicating whether each atom is found ( either visited or in queue)

    def __init__(self,nAtoms1, nAtoms2=None):
        self.nAtoms = [nAtoms1, nAtoms2] if nAtoms2!=None else [nAtoms1,nAtoms1]
        self.visited_order = [[],[]]
        self.visited_or_in_queue = [None,None]
        for i in range(2):
            self.visited_or_in_queue[i] = [False for _ in range(self.nAtoms[i])]
        self.to_visit = [deque(),deque()]

    def Copy(self):
        ns = SolvingStatus(self.nAtoms[0],self.nAtoms[1])
        import copy
        ns.visited_order = copy.deepcopy(self.visited_order)
        ns.to_visit = [self.to_visit[_].copy() for _ in range(2)]
        ns.visited_or_in_queue = copy.deepcopy(self.visited_or_in_queue)
        return ns

    def Show(self):
        import sys
        sys.stdout.write("Matching Order: ")
        for i in range(len(self.visited_order[0])):
            sys.stdout.write("{}-{} ".format(self.visited_order[0][i],self.visited_order[1][i]))
        print("")
        sys.stdout.write("Deque of each molecule: (A) ")
        for i in range(len(self.to_visit[0])):
            sys.stdout.write("{} ".format(self.to_visit[0][i]))
        sys.stdout.write("(B) ")
        for i in range(len(self.to_visit[1])):
            sys.stdout.write("{} ".format(self.to_visit[1][i]))
        print("")

global_iter_counter = 0

def __Recursive__(mols,bondedMaps,status,level=0,__exit_on_first_match__ = True, __debugging__ = False):

    total_results = []

    if __debugging__:
        print("Level = {}".format(level))
        status.Show()
        global global_iter_counter
        global_iter_counter += 1
        print("Global Iter = {}".format(global_iter_counter))

    # Do a DFS on both molecules. In cases there are multiple choices, perform and branch-and-cut algorithm
    while len(status.to_visit[0]) > 0:  # Until the queue is empty
        candidates = [[], []]
        head = [None, None]
        for iMol in range(2):
            # Get the first node from the queue. popleft() is BFS by using a queue, while pop() is DFS by using a stack,
            # Testing shows DFS is faster
#            head[iMol] = status.to_visit[iMol].popleft()
            head[iMol] = status.to_visit[iMol].pop()

            status.visited_order[iMol].append(head[iMol])  # visit the node
            for next_node in bondedMaps[iMol][head[iMol]]:  # among its neighbors, add those un-visited and un-queued node to the queue
                if status.visited_or_in_queue[iMol][next_node] == False:
                    candidates[iMol].append(next_node)
                    status.visited_or_in_queue[iMol][next_node] = True  # Just flag those atoms, don't really add them into the queue,yet.

        if __debugging__:
            for iMol in range(2):
                print("Head For Mol{}: {}, Candidates = {}".format(iMol,head[iMol],candidates[iMol]))

        # Based on the visited node and the number of candidates, we should either branch, or cut:
        if mols[0].atoms[head[0]].element != mols[1].atoms[head[1]].element:
            # Cut because element type mismatch
            return []
        elif status.nAtoms[0] == status.nAtoms[1] and len(candidates[0]) != len(candidates[1]):
            # Cut because bonded atoms count mismatch in equivalent molecules
            return []
        elif status.nAtoms[0] < status.nAtoms[1] and len(candidates[0]) > len(candidates[1]):
            # Cut because bonded atoms count mismatch in fragmental match
            return []
        elif len(candidates[0]) == 0:
            # If there is no bonded atoms in mol1, the visit is finished
            pass
        # elif len(candidates[0]) == 1:
        #     for iMol in region(2):
        #         status.to_visit[iMol].append(candidates[iMol][0])
        else:
            # Branch!  Do a full permutation of mol[1]'s candidates
            # Keep in mind that since A may be a fragment of B, the candidates of A and B may be of different length
            import itertools
            candidates_permutated = list(itertools.permutations(candidates[1]))
            # For each possible permutation, there is a branch:
            candidates_of_mol0 = candidates[0]
            for candidates_of_mol1 in candidates_permutated:
                newStatus = status.Copy()
                pre_test_flag = True
                for iPerm in range(min(len(candidates_of_mol0),len(candidates_of_mol1))):

                    a0 = candidates_of_mol0[iPerm]
                    a1 = candidates_of_mol1[iPerm]

                    # Some simple tests to exclude impossible matches
                    if mols[0].atoms[a0].element != mols[1].atoms[a1].element:
                        pre_test_flag = False
                        break

                    # if two mols have identical # of atoms, we require the # of bonds to be equal
                    if status.nAtoms[0] == status.nAtoms[1] and len(bondedMaps[0][a0]) != len(bondedMaps[1][a1]):
                        pre_test_flag = False
                        break
                    # if mol1 is a possibly a fragment of mol2, we require differently
                    if status.nAtoms[0] < status.nAtoms[1] and len(bondedMaps[0][a0]) > len(bondedMaps[1][a1]):
                        pre_test_flag = False
                        break

                    newStatus.to_visit[0].append(a0)
                    newStatus.to_visit[1].append(a1)

                if not pre_test_flag:
                    continue

                results = __Recursive__(mols,bondedMaps,newStatus,level=level+1,\
                                        __exit_on_first_match__ = __exit_on_first_match__,__debugging__ = __debugging__)
                if len(results) > 0:
                    total_results.extend(results)
                    if __exit_on_first_match__:
                        break

            return total_results   # The return point for recursive levels other than the last

    # After the while, all matched. End of recursion
    return [status]

def TopologyMatching(mols:[Molecule], first_node):
    # mols[0] and mols[1] are two molecules. They should be identical but the order of atoms may be totally different in these
    # two molecules. This algorithm works by identifying bondings between atoms, so bonds must be properly set in both
    # molecules
    # Modified Feb 2021: mols[0] can be a fragment of mols[1]. This function works for this senario
    # without any modification.

    # If successful, returns a list of lists, each of which represents a matching.
    # For example, the return value of [  [1,3,2,0], [1,2,0,3] ] means that atoms 0,1,2,3 in the first molecule
    # can be either be matched to atoms 1,3,2,0 or atoms 1,2,0,3 in the second molecule.
    # All indexes for atoms in this module starts from 0! Atom's serials are not used here!
    # There may be multiple ways (or no way) to match atoms in the first molecule to those in the second molecule,
    # therefore the length of the list is not known beforehand. An empty list [] as the return value means that it is
    # impossible to find a match.

    # The 'first_node' parameter is a hint given by the caller, suggesting which two atoms in these two molecules are
    # equivalent. The subsequent search shall begin on these two atoms. If not given, the program will try all combinations
    # and the function may run NAtoms times longer.. To mandate the user to give an initial hint, this function will
    # refuse to work without it! If really necessary, the caller can write a loop to try all intial matching point.

    if len(mols[0].atoms) > len(mols[1].atoms):
        error("In TopologyMatching(), either the two molecules are identical, or mols[0] is a fragment of mols[1]")
        return []

    initial_status = SolvingStatus(len(mols[0].atoms),len(mols[1].atoms))

    bondedMaps = [ mols[_].BondedMap() for _ in range(2) ]

    if first_node == None:
        error("The caller must give a 'first_node' parameter such as [0,3] to suggest that\n"
        "atom index 0 in mol1 is matched to atom index 3 in mol2")

    for iMol in range(2):
        initial_status.to_visit[iMol].append(first_node[iMol])          # Put first node in queue
        initial_status.visited_or_in_queue[iMol][first_node[iMol]] = True     # Set the proper flag

    results = __Recursive__(mols,bondedMaps,initial_status,level=0,__exit_on_first_match__ = True,__debugging__= False)

    if len(results) == 0:
        return None

    result = results[0]
    matching_map = [-1 for _ in range(result.nAtoms[0])]
    for iAtom in range(result.nAtoms[0]):
        matching_map[result.visited_order[0][iAtom]] = result.visited_order[1][iAtom]

    def __verify__(bondedMaps,matching_map):
        passed = True
        for iAtomInMol0 in range(len(matching_map)):
            iAtomInMol1 = matching_map[iAtomInMol0]
            for iBondedToInMol0 in bondedMaps[0][iAtomInMol0]:
                iBondedToInMol1 = matching_map[iBondedToInMol0]
                # verify that iBondedToInMol1 is in bondedMaps[1][iAtomInMol1]
                if iBondedToInMol1 not in bondedMaps[1][iAtomInMol1]:
                    print("Verify Failed: atom {} in (A) is matched to atom {} in (B), which should be bonded to "
                          "atom {} in (A). According to matching, this is {} in (B), but this is not.".format(iAtomInMol0,iAtomInMol1,iBondedToInMol0,iBondedToInMol1))
                    passed = False
            if not passed:
                break
        return passed

    if not __verify__(bondedMaps,matching_map):
        error("Internal error in TopologyMatching(): Verification Not Passed !",False)

    # for i in region(result.nAtoms[0]):
    #     print("{},{}".format(i,matching_map[i]))

    return matching_map

def TestCase(mol):
    # This is a testing case by randomly permutating atoms within a molecule, and match them
    # with the original molecule
    from UniversalMolecularSystem import MolecularSystem
    from BondDetection import DefaultBondRules
    from random import shuffle

    originalSystem = MolecularSystem()
    originalSystem.molecules = [mol]
    originalSystem.AutoDetectBonds(DefaultBondRules(),flushCurrentBonds = True)

    permSystem = originalSystem.Copy()
    shuffle(permSystem.molecules[0].atoms)  # Shuffle all but the first atom
    permSystem.RenumberAtomSerials()
    permSystem.AutoDetectBonds(DefaultBondRules(),flushCurrentBonds = True)

    originalSystem.Summary()
    permSystem.Summary()

    for i in range(len(originalSystem.molecules[0].atoms)):
        result = TopologyMatching([originalSystem.molecules[0],permSystem.molecules[0]],[0,i])
        if result != None:
            print(i)
            print(result)
        else:
            print("{}: No Match".format(i))

if __name__ == '__main__':

    TestCase()


from PIL import Image
import re
import os

path =  os.path.join(os.path.expanduser('~'),'images')
def img_fix(imglist):
    for img in imglist:
        savepath = os.path.join(os.path.expanduser('~'),'opt','icons')
        fullpath = os.path.join(path, img)
        open_image = Image.open(fullpath)
        new_name = re.sub('\.[a-z]*$', '.jpeg', img)
        print(new_name)
        new_path = os.path.join(savepath,new_name)
        open_image.rotate(-90).resize((128,128)).convert("RGB").save(new_path)
        print(open_image.size)

def main():
    print(path)
    imgfiles = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))]
    print(imgfiles)
    #img_fix(imgfiles)

if __name__ == "__main__":
    main()


import configparser, os

PATH = 'config.cfg'
DEFAULTS = {
    'main': {
        'debug': True,
        'interpreter': False,
    },
        'graphics': {
            'hwaccel': True,
    }
}
skip_file = False

config = configparser.ConfigParser()

if os.path.isfile(PATH) and not skip_file:
    config.read_file(open('config.cfg'))
else:
    config.read_dict(DEFAULTS)


# coding=utf-8
import json
import re
import requests,os

# TODO 可以用

dir = "H:\\11-快手单个视频\\"
def get(url: str) -> dict:
    """
    title、imgs、videos
    """
    data = {}
    failed = {'msg': 'failed...'}
    headers = {
        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25",
        "Cookie": "did=web_f3915064ee334c508642888137f27598;"
    }
    # rewrite desktop url
    temp = re.findall(r'live\.kuaishou\.com/u/\w+/(\w+)', url)
    # print(temp)
    if temp:
        url = 'https://c.kuaishou.com/fw/photo/{}'.format(temp[0])

    rep = requests.get(url, headers=headers, timeout=10)
    if rep.status_code != 200:
        return failed
    page_data = re.findall(r'<script type="text/javascript">window\.pageData= (\{.*?\})</script>', rep.text)
    # print(page_data)
    if not page_data:
        return failed
    try:
        page_data = json.loads(page_data[0])
    except Exception:
        print('kuaishou loads json failed')
        return failed

    video_info = page_data['video']
    data['title'] = video_info['caption']
    # 获取主播名字
    data['user'] = page_data['user']['name']
    # 时间
    data['time'] = page_data['rawPhoto']['timestamp']
    # 获取视频
    try:  # 如果出错，则可能是长图视频
        data['videos'] = [video_info['srcNoMark']]
    except Exception:
        pass
    else:
        data['videoName'] = data['title']
        data['msg'] = '如果快手视频下载出错请尝试更换网络'
    # 获取图片
    try:  # 如果出错，则可能是普通视频；
        images = video_info['images']
        imageCDN: str = video_info['imageCDN']
        # 如果是长图视频，则这几项一定存在
        assert images is not None
        assert imageCDN is not None
    except Exception:
        pass
    else:
        if not imageCDN.startswith('http'):
            imageCDN = 'http://' + imageCDN
        data['imgs'] = [imageCDN + i['path'] for i in images]
    return data

def readconfig():
    f = open(r'config.txt', "r", encoding='utf-8')
    for x in f:
        geturl(x)
    f.close()

def geturl(config:str)-> list:
    pattern = re.compile('(https:.*?) 复制此消息，打开【快手】直接观看！.*?',re.S)
    # pattern = re.compile('.*?发了一个快手作品，一起来看！(.*?) 复制此消息，打开【快手】直接观看！.*?',re.S)
    v_url = re.findall(pattern, config)
    print(v_url)
    if v_url!= [] :
        req = get(v_url[0])
        download(req)
def download(req):
    try:
        v_name = req.get('user') + str(req.get('time')) + ".mp4"
        video = dir + v_name
        print(video)
        print(req.get('videos'))
        if not os.path.exists(video):
            r = requests.get(req.get('videos')[0])
            r.raise_for_status()
            with open(video, "wb") as f:
                f.write(r.content)
            print("    视频 " + v_name + " 下载成功 √")
        else:
            print("    视频 " + v_name + " 已存在 √")
    except:
        print("  这里似乎有点小错误，已跳过")
if __name__ == "__main__":
    # print(get(url="https://live.kuaishou.com/u/kissyou696773/3x9vpmn3n4ihvg6"))
    # print(get(url="https://v.kuaishou.com/7x1fql"))
    readconfig()


import sys
from collections import deque

dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]

input = sys.stdin.readline

n, m = map(int, input().split())
# 치즈가 있는 부분 1, 치즈가 없는 부분 0
map = [list(map(int, input().split())) for _ in range(n)]

def melted():
    queue = deque()
    visited = [[0] * m for _ in range(n)]

    # 모눈종이의 맨 가장자리에는 치즈가 놓이지 않는 것으로 가정한다
    queue.append([0,0])
    visited[0][0] = 1

    while queue:
        x, y = queue.popleft()
        for i in range(4):
            nx, ny = x + dx[i], y + dy[i]

            if 0 <= nx < n and 0 <= ny < m:
                # 외부 공기
                if not map[nx][ny] and not visited[nx][ny]:
                    visited[nx][ny] = 1
                    queue.append([nx, ny])

                # 치즈
                else:
                    visited[nx][ny] += 1

    sum = 0
    for i in range(n):
        for j in range(m):
            if visited[i][j] >= 2:
                map[i][j] = 0

            sum += map[i][j]
    return sum

ans = 0
while True:
    result = melted()
    ans += 1

    if not result:
        break

print(ans)


#!/usr/bin/python

from gi.repository import Gtk, GObject, Gdk
import sys

class CommonAPI(Gtk.Window):

	def __init__ (self, Title, Length, Breadth):
		Gtk.Window.__init__(self, title = Title )
		self.set_size_request(Length, Breadth)
		self.fixed = Gtk.Fixed()
		self.grid = Gtk.Grid()
		self.radioButton1 = None
		#self.Show()

	def CreateButton(self, xPos, yPos, Title ):
		button = Gtk.Button( label = Title )
		self.fixed.put(button, xPos, yPos )

	def CreateRadioButton(self, xPos, yPos, Title ):
		radioButton = Gtk.RadioButton.new_with_label_from_widget(self.radioButton1, Title)
		self.radioButton1 = radioButton
		self.fixed.put(radioButton, xPos, yPos )

	def CreateCheckBox(self, xPos, yPos, Title ):
		checkButton = Gtk.CheckButton( label = Title )
		self.fixed.put(checkButton, xPos, yPos)

	def CreateTextBox(self, xPos, yPos):
		textField = Gtk.Entry()
		self.fixed.put(textField, xPos, yPos)

	def CreateList(self, xPos, yPos, List):
		#Creating a model from treeview
		store = Gtk.ListStore(str)
		for i in range(len(List)):
			listiter = store.append([List[i]])
		tree = Gtk.TreeView(store)

		vbox = Gtk.VBox(spacing = 1)
		label = Gtk.Label("Your list is here");
		vbox.pack_start(label, False, False, 0)
		vbox.pack_start(tree, False, False, 0)

		#column for name
		renderer = Gtk.CellRendererText()
		column = Gtk.TreeViewColumn(" ", renderer, text = 0)
		tree.append_column(column)
		self.fixed.put(vbox, xPos, yPos)

	def Show(self):
		self.add(self.fixed)
		self.connect("delete-event", Gtk.main_quit)
		self.show_all()
		Gtk.main()


import graph_tool as gt
import json
from random import sample

net = gt.load_graph('citenet0.out.gt')

core_pmap = net.vp['core']
core = [vertex for vertex in net.vertices() if core_pmap[vertex]]
downstream = core_pmap.copy()
gt.infect_vertex_property(net, downstream, vals = [True])

boundary_pmap = net.new_vp('bool',
							vals = [downstream[vertex] and not core_pmap[vertex]
									for vertex in net.vertices()])
boundary = [vertex for vertex in net.vertices() if boundary_pmap[vertex]]

core_dois = [net.vp['doi'][vertex] for vertex in core]
core_sids = [net.vp['sid'][vertex] for vertex in core]

core_refs = {}
for paper in boundary:
	core_ref_sids = [sid for sid in net.vp['references'][paper] if sid in core_sids]
	core_ref_papers = [paper for paper in core if net.vp['sid'][paper] in core_ref_sids]
	core_ref_dois = {net.vp['doi'][paper]: 0 for paper in core_ref_papers}

	core_refs[net.vp['doi'][paper]] = core_ref_dois

print('Total boundary items: ' + str(len(boundary)))

boundary_subset = sample(boundary, 25)
boundary_subset_dois = [net.vp['doi'][paper] for paper in boundary_subset]
core_refs_subset = {key:value for key, value in core_refs.items() if key in boundary_subset_dois}
print('Total subset items: ' + str(len(boundary_subset)))

with open('boundary.output.json', 'w') as outfile:
	json.dump(core_refs_subset, outfile, indent = 4)


import sys
import numpy as np
import pandas as pd

import data
from analysis import full_entropy_calculation_bpw

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print(f'USAGE: {sys.argv[0]} <filename>')
        exit(-1)
    # Variables related to the location of the data and the type of system
    bibles_path = '/home/pablo/Documents/GitHubRepos/paralleltext/bibles/corpus/'
    bible_filename = sys.argv[1]
    #bible_filename = 'eng-x-bible-world.txt'
    output_path = '/home/pablo/Documents/GitHubRepos/WordOrderBibles/output/MontemurroZanette/'
    # Variables related to the processing of text for GPT-2
    prompt = ''
    separator = ' '
    # Variables related to the processing of text for unigram entropies
    remove_punctuation = False
    lowercase = False

    bible = data.parse_pbc_bible(bibles_path + bible_filename)

    """For each of these hierarchical orders, we can compute the entropy per word and the unigram entropy."""
    by_bible, _, by_book, _, _ = bible.join_by_toc()
    by_level = {'bible': by_bible, 'book': by_book}

    eos_token = ''
    level_text = {level_name: data.join_texts_in_dict(id_texts, prompt, eos_token, separator) \
                  for level_name, id_texts in by_level.items()}

    raw_name = output_path + bible_filename
    level_entropies = {level_name: full_entropy_calculation_bpw(id_text,
                                                                remove_punctuation,
                                                                lowercase,
                                                            f'{raw_name}_{level_name}') \
                       for level_name, id_text in level_text.items()}

    level_avg_text_len = {level_name: np.mean([len(data.tokenize(text, remove_punctuation, lowercase)) \
                                               for text in id_text.values()]) \
                          for level_name, id_text in level_text.items()}

    # Save all these values to a Pandas dataframe that we can use to make histograms and compute statistics
    df = pd.DataFrame(columns=('level', 'n_tokens', 'H', 'H_s', 'H_r', 'id'))
    for level_name, section_entropies in level_entropies.items():
        for section_id, entropies in section_entropies.items():
            row = (level_name, len(data.tokenize(level_text[level_name][section_id], remove_punctuation, lowercase)),
                   entropies[0], entropies[1], entropies[2], str(section_id))
            df.loc[len(df)] = row

    # Compute the word-order entropies
    df['D_r'] = df['H_r'] - df['H']
    df['D_s'] = df['H_s'] - df['H']

    df.to_csv(output_path + bible_filename.replace('.txt', '_entropies.csv'), index=False)


# 게리맨더링
import sys
sys.stdin = open('input.txt', 'r')
from collections import deque

def dfs(cnt, s, end):
    global mn
    # 종료조건
    if cnt == end:  # 재귀횟수가 목표횟수에 도달
        group1, group2 = deque(), deque()   # 그룹1, 2로 나누기

        # 방문 한 지점 group1, # 방문 안 한 지점 group2
        for i in range(1, N+1):
            if visited[i]:
                group1.append(i)
            else:
                group2.append(i)

        # 연결되어 있는지 확인
        ans1 = bfs(group1)
        if not ans1:
            return
        ans2 = bfs(group2)
        if not ans2:
            return

        mn = min(mn, abs(ans1-ans2))
        return

    # end개수에 도달하지 못 했을 때는 그 다음 구역부터 조합 더하기
    for i in range(s, N+1):
        if not visited[i]:  # 방문하지 않은 지점
            visited[i] = 1
            dfs(cnt+1, i, end)
            visited[i] = 0

def bfs(group):
    q = deque([group[0]])   # 첫 지점 enqueue
    check = [0] * (N+1)
    check[group[0]] = 1     # 방문 표시

    cnt, answer = 1, 0      # cnt: 방문 지점 수, answer: 인구 수 합
    while q:
        t = q.popleft()
        answer += nums[t]
        for i in adjL[t]:
            # 인접하고 group 안에 속하며 방문한 적이 없으면 계속 진행
            if i in group and not check[i]:
                check[i] = 1
                cnt += 1
                q.append(i)

    if cnt == len(group):
        return answer
    else:
        return 0

N = int(input())
nums = [0] + list(map(int, input().split()))    # 인구수 배열
visited = [0] * (N+1)
adjL = [[] for _ in range(N+1)]     # 인접 리스트
for i in range(1, N+1):
    x, *lst = map(int, input().split())
    for c in lst:
        adjL[i].append(c)

mn = 1000
for i in range(1, N//2 + 1):    # 1개/N-1개, 2개/N-2개, ... N//2개/N//2개 -> 조합
    visited = [0] * (N+1)
    dfs(0, 1, i)                # 현재까지 재귀 횟수, 시작지점, target 재귀횟수

if mn == 1000:
    print(-1)
else:
    print(mn)


import pytest

from api.community import choices, factories, selectors

@pytest.mark.django_db
class TestCommunitySelectors:
    def test_filter_company_hashtags(self):
        h_1 = factories.HashtagFactory(slug="a")
        h_2 = factories.HashtagFactory(slug="b")

        _ = factories.CompanyFactory(slug="a", current_revision__hashtags=[h_1])
        company_2 = factories.CompanyFactory(slug="b", current_revision__hashtags=[h_2])
        company_3 = factories.CompanyFactory(
            slug="c", status=choices.ModerationStatus.REJECTED.name
        )

        qs = selectors.get_companies()

        rv = selectors.filter_companies(qs, None, ["b"], None)
        assert rv.count() == 1
        assert rv.first().slug == company_2.slug

        rv = selectors.filter_companies(qs, company_2.current_revision.name, None, None)
        assert rv.count() == 1
        assert rv.first().slug == company_2.slug

        rv = selectors.filter_companies(
            qs, None, None, choices.ModerationStatus.REJECTED.name
        )
        assert rv.count() == 1
        assert rv.first().slug == company_3.slug


import numpy as np
import copy
from scipy import sparse

file = "day_12_input.txt"
# file = "day_12_input_test.txt"

with open(file) as f:
    lines = f.readlines()
    f.close()

lines_letters = [l.replace("\n", "") for l in lines]
lines_elevation = [
    [ord(x) % 32 if x not in ["S", "E"] else 0 for x in l] for l in lines_letters
]

elevation = np.stack([np.array([*l], dtype=int) for l in lines_elevation], axis=1)
letters = np.stack([np.array([*l], dtype=str) for l in lines_letters], axis=1)

elevation = elevation.T
letters = letters.T

start = tuple(np.argwhere(letters == "S")[0])
end = tuple(np.argwhere(letters == "E")[0])

elevation[letters == "E"] = 26
elevation[letters == "S"] = 0
directions = ["u", "d", "l", "r"]

width = elevation.shape[1]
length = elevation.shape[0]

global directions
global elevation
global width
global length

def next_location(state, direction):
    if direction == "u":
        new = (state[0] - 1, state[1])
    if direction == "d":
        new = (state[0] + 1, state[1])
    if direction == "l":
        new = (state[0], state[1] - 1)
    if direction == "r":
        new = (state[0], state[1] + 1)
    return new

def is_in_map(state):
    max_x = elevation.shape[0]
    max_y = elevation.shape[1]
    x_valid = (state[0] >= 0) & (state[0] < max_x)
    y_valid = (state[1] >= 0) & (state[1] < max_y)
    return x_valid & y_valid

def is_possible(state, new):
    return elevation[new] - elevation[state] <= 1

def get_adjacent(state):
    adjacent = []
    for direction in directions:
        new = next_location(state, direction)
        if is_in_map(new):
            if is_possible(state, new):
                adjacent.append(new)
    return adjacent

def idx(position):
    return position[0] * width + position[1]

n = elevation.shape[0] * elevation.shape[1]
A = np.zeros((n, n), dtype=int)

for i in range(length):
    for j in range(width):
        adjacent = get_adjacent((i, j))
        for a in adjacent:
            A[idx((i, j)), idx(a)] = 1

B = A.copy()
A @ A
AA = sparse.csr_matrix(A)
BB = sparse.csr_matrix(B)

## test solution works
## Normal solution does not
## why?
## https://galaxyinferno.com/how-to-solve-advent-of-code-2022-day-12-with-python/

steps = 1
while BB.toarray()[idx(start), idx(end)] == 0 and steps < 1000:
    BB = BB @ AA
    steps += 1
    if steps % 50 == 0:
        print(steps)

steps


# do analysis on the layer

from fileIterables import allFilesInRecurseByType, getNameFromPath
from statistics import pstdev as stddev
from statistics import mean

Layer_DIR = "layerSampleSet"
LFILE_TYPE = "lyr"

class Stats:
    def __init__(self):
        self.segments = 0
        self.pointsMean = 0
        self.pointsDev = 0

statsList = []
#how many times is the most visited point visited?
mostVisitedPointQty = 0
mostVisitedPointPoint = (None, None)
mostVisitedPointFile = ""

for filePath in allFilesInRecurseByType(Layer_DIR, LFILE_TYPE):
    with open(filePath) as f:
        stats = Stats()

        segCounter = 0

        uniquePoints = 0
        pointsDict = {}

        for line in f:
            tokens = line.strip().split(" ")

            tokens_int = list(map(lambda x: round(float(x)*10000), tokens))

            for t in tokens_int:
                if t%10 != 0:
                    print(tokens)
                    raise ValueError(f"need more rounding precision: {t} {filePath}")

            p1 = (tokens_int[0], tokens_int[1])
            p2 = (tokens_int[2], tokens_int[3])

            for p in [p1,p2]:
                if p not in pointsDict:
                    pointsDict[p] = 1
                    uniquePoints += 1
                else:
                    pointsDict[p] += 1

            segCounter += 1

        for k in pointsDict:
            if(pointsDict[k] > mostVisitedPointQty):
                mostVisitedPointQty = pointsDict[k]
                mostVisitedPointFile = filePath
                mostVisitedPointPoint = k

        stats.segments = segCounter
        stats.pointsMean = mean(map(lambda x: pointsDict[x], pointsDict))
        stats.pointsDev = stddev(map(lambda x: pointsDict[x], pointsDict))
        statsList.append(stats)

segMean = mean(map(lambda x: x.segments, statsList))
segDev = stddev(map(lambda x: x.segments, statsList))
pointsMeanMean = mean(map(lambda x: x.pointsMean, statsList))
pointsMeanDev = stddev(map(lambda x: x.pointsMean, statsList))
pointsDevMean = mean(map(lambda x: x.pointsDev, statsList))
pointsDevDev = stddev(map(lambda x: x.pointsDev, statsList))

print(f"SM : {segMean}")
print(f"SD : {segDev}")
print(f"PMM: {pointsMeanMean}")
print(f"PMD: {pointsMeanDev}")
print(f"PDM: {pointsDevMean}")
print(f"PDD: {pointsDevDev}")
print("")
print(f"MVPQ: {mostVisitedPointQty}")
print(f"MVPP: {mostVisitedPointPoint}")
print(f"MVPF: {mostVisitedPointFile}")


import numpy as np
import cv2
import glob
import os
import PIL as Image
import os.path as osp

from tqdm import tqdm
current_dir = os.getcwd()
print("path", current_dir)
# path= ''.join([current_dir, folder])
count = 1
if __name__ == "__main__":
    # path_an = sorted(glob.glob('/home/airlab/Desktop/New_training/01_Annotation/ANNOTATION/test/*.png'))
    # path_im = sorted(glob.glob('/home/airlab/Desktop/New_training/01_Annotation/IMG/test/*.jpg'))
    name = 'train'
    path_im = sorted(glob.glob('C:\\Users\\ptthi\\OneDrive\\Desktop\\Image_processing\\DATA_PROCESS\\BACKGROUND\\*.png'))
    print("path", path_im)
    # tao folder moi
    if not os.path.exists(current_dir + '/' + "{}_new".format(name)):
        os.makedirs(current_dir + '/' + '{}_new'.format(name))
    for i,im in enumerate(path_im):
        base_an = osp.splitext(osp.basename(im))[0]
        print("name of image:", i)
        if i % 2 == 0:
            print("True")
            file_im = cv2.imread(im)
            cv2.imwrite(current_dir + '/' + '{}_new'.format(name) + '/' + '{}.jpg'.format(count),file_im)
            count += 1
        print("count",count)


from fabric.api import *
env.hosts = ['servername']

def _annotate_hosts_with_ssh_config_info():
    from os.path import expanduser
    from paramiko.config import SSHConfig

    def hostinfo(host, config):
        hive = config.lookup(host)
        if 'hostname' in hive:
            host = hive['hostname']
        if 'user' in hive:
            host = '%s@%s' % (hive['user'], host)
        if 'port' in hive:
            host = '%s:%s' % (host, hive['port'])
        return host

    try:
        config_file = file(expanduser('~/.ssh/config'))
    except IOError:
        pass
    else:
        config = SSHConfig()
        config.parse(config_file)
        keys = [config.lookup(host).get('identityfile', None)
            for host in env.hosts]
        env.key_filename = [expanduser(key) for key in keys if key is not None]
        env.hosts = [hostinfo(host, config) for host in env.hosts]

        for role, rolehosts in env.roledefs.items():
            env.roledefs[role] = [hostinfo(host, config) for host in rolehosts]

_annotate_hosts_with_ssh_config_info()


from flask import Flask, jsonify, request
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///test.db'
db = SQLAlchemy(app)

class User(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    firstname = db.Column(db.String(200), nullable=False)
    lastname = db.Column(db.String(200), nullable=False)
    email = db.Column(db.String(200), nullable=False)

    def __repr__(self):
        return '<User %r>' % self.id

# Create
@app.route('/users', methods=['POST'])
def postUser():

    user = User(firstname = request.json['firstname'],
                  lastname = request.json['lastname'],
                  email = request.json['email'])

    db.session.add(user)
    db.session.commit()

    return jsonify({'id': user.id, 'firstname':user.firstname,'lastname':user.lastname, 'email': user.email}), 201

# Read
@app.route('/users/<int:id>', methods=['GET'])
def getUserById(id):

    user = User.query.get_or_404(id)

    return jsonify({'id': user.id, 'firstname':user.firstname,'lastname':user.lastname, 'email': user.email})

# Update
@app.route('/users/<int:id>', methods=['PUT'])
def putUser(id):

    user = User.query.get_or_404(id)
    user.firstname = request.json['firstname']
    user.lastname = request.json['lastname']
    user.email = request.json['email']
    db.session.commit()
    return jsonify({'id': user.id, 'firstname':user.firstname,'lastname':user.lastname, 'email': user.email})

# Delete
@app.route('/users/<int:id>', methods=['DELETE'])
def deleteUser(id):

    user = User.query.get_or_404(id)
    db.session.delete(user)
    db.session.commit()
    return jsonify({'id': user.id, 'firstname':user.firstname,'lastname':user.lastname, 'email': user.email}), 200

# Get All
@app.route('/users', methods=['GET'])
def getUsers():
    users = User.query.all()
    all_users = [{'id': user.id, 'firstname':user.firstname,'lastname':user.lastname, 'email': user.email} for user in users]
    return jsonify(all_users)

if __name__ == "__main__":
    app.run(debug=True)


import torch
import torch.nn as nn
from nets.vgg16 import decom_vgg16
from nets.resnet50 import resnet50
from nets.rpn import RegionProposalNetwork
from nets.classifier import VGG16RoIHead,Resnet50RoIHead
import time
import numpy as np
class FasterRCNN(nn.Module):
    def __init__(self, num_classes,
                mode = "training",
                loc_normalize_mean = (0., 0., 0., 0.),
                loc_normalize_std = (0.1, 0.1, 0.2, 0.2),
                feat_stride = 16,
                anchor_scales = [8, 16, 32],
                ratios = [0.5, 1, 2],
                backbone = 'vgg'
    ):
        super(FasterRCNN, self).__init__()

        self.loc_normalize_mean = loc_normalize_mean
        self.loc_normalize_std = loc_normalize_std
        self.feat_stride = feat_stride
        if backbone == 'vgg':
            self.extractor, classifier = decom_vgg16()
            self.rpn = RegionProposalNetwork(
                512, 512,
                ratios=ratios,
                anchor_scales=anchor_scales,
                feat_stride=self.feat_stride,
                mode = mode
            )
            self.head = VGG16RoIHead(
                n_class=num_classes + 1,
                roi_size=7,
                spatial_scale=(1. / self.feat_stride),
                classifier=classifier
            )
        elif backbone == 'resnet50':
            self.extractor, classifier = resnet50()

            self.rpn = RegionProposalNetwork(
                1024, 512,
                ratios=ratios,
                anchor_scales=anchor_scales,
                feat_stride=self.feat_stride,
                mode = mode
            )
            self.head = Resnet50RoIHead(
                n_class=num_classes + 1,
                roi_size=14,
                spatial_scale=(1. / self.feat_stride),
                classifier=classifier
            )
    def forward(self, x, scale=1.):
        img_size = x.shape[2:]
        h = self.extractor(x)

        rpn_locs, rpn_scores, rois, roi_indices, anchor = \
            self.rpn.forward(h, img_size, scale)

        # print(np.shape(h))
        # print(np.shape(rois))
        # print(roi_indices)
        roi_cls_locs, roi_scores = self.head.forward(h, rois, roi_indices)
        return roi_cls_locs, roi_scores, rois, roi_indices

    def freeze_bn(self):
        for m in self.modules():
            if isinstance(m, nn.BatchNorm2d):
                m.eval()


import io
from google.oauth2 import service_account
from google.cloud import vision
from CREDENTIALS import VISION_KEY_PATH

credentials = service_account.Credentials.from_service_account_file(
    VISION_KEY_PATH)

client = vision.ImageAnnotatorClient(credentials=credentials)

def read_image(filepath):
    with io.open(filepath, 'rb') as image_file:
        content = image_file.read()
    image = vision.types.Image(content=content)
    response = client.document_text_detection(image=image)
    document = response.full_text_annotation.text

    if document == '':
        return "Oops, I didn't seem to find anything. Please try again."
    else:
        return document


from PyQt5.QtWidgets import QMainWindow, QApplication, QFileDialog, QMessageBox, QLabel, QDialog
from PyQt5 import uic
from PyQt5.QtGui import *
from PyQt5.QtCore import QRect, Qt, QPoint
import os, glob, cv2, sys

form_class = uic.loadUiType("Image_loader2.ui")[0]

class Label_Dialog(QDialog):
    def __init__(self, parent):
        super(Label_Dialog, self).__init__(parent)
        uic.loadUi('/Users/picardy/PycharmProjects/pythonProject/Label.ui', self)
        self.Green.setStyleSheet('background-color: #7cfc00')
        self.Yellow.setStyleSheet('background-color: yellow')
        self.Blue.setStyleSheet('background-color: #0066ff')
        self.parent = parent
        self.show()
        self.Green.clicked.connect(parent.greenColor)
        self.Yellow.clicked.connect(parent.yellowColor)
        self.Blue.clicked.connect(parent.blueColor)
        self.Green_line.textChanged.connect(self.func_G)
        self.Yellow_line.textChanged.connect(self.func_Y)
        self.Blue_line.textChanged.connect(self.func_B)
        self.Path_line.textChanged.connect(self.labelPath)
        self.toolButton.clicked.connect(self.path)

    def path(self):
        dir_path = QFileDialog.getExistingDirectory(self, 'open')
        for file in dir_path:
            exist = self.Path_line.text()
            self.Path_line.setText(exist + file)
        print(dir_path)

    def func_G(self):
        self.parent.Label_G = self.Green_line.text()

    def func_Y(self):
        self.parent.Label_Y = self.Yellow_line.text()

    def func_B(self):
        self.parent.Label_B = self.Blue_line.text()

    def labelPath(self):
        self.parent.path = self.Path_line.text()

class ViewerClass(QMainWindow, form_class):
    def __init__(self, parent=None):
        super().__init__()
        self.setupUi(self)
        self.qPixmapVar = QPixmap()
        self.Image_label = QLabel()
        self.actionFile.triggered.connect(self.fileSelect)
        self.actionDirectory.triggered.connect(self.folderSelect)
        self.NextButton.clicked.connect(self.NextClick)
        self.PreviousButton.clicked.connect(self.PreviousClick)
        self.actionClose.triggered.connect(self.close)
        self.actionOpen_Tool.triggered.connect(self.openTool)

        self.idx = 0
        self.flag = 0
        self.start, self.end = QPoint(), QPoint()
        self.Label_G = ()
        self.Label_Y = ()
        self.Label_B = ()
        self.path = ()
        self.image = None

    # def imageSave(self):
    #     filename = QFileDialog.getSaveFileName(filter="JPG(*.jpg);;PNG(*.png);;TIFF(*.tiff);;BMP(*.bmp)")[0]
    #     cv2.imwrite(filename, self.image)

    def fileSelect(self):
        filename = QFileDialog.getOpenFileName(self, 'open', '/Users/picardy/Downloads/Seungkyu/copied/')[0]
        if filename:
            self.image = cv2.imread(filename)
            self.dir_h, self.dir_w, channel = self.image.shape
            self.qPixmapVar.load(filename)
            self.Image_label.setPixmap(self.qPixmapVar)
            QMainWindow.resize(self, self.dir_w, self.dir_h)
            print(self.dir_w, self.dir_h)

            # folderName = self.image
            self.name = filename[filename.rfind("/"): - 4]
            self.savename = str(self.path) + str(self.name) + '.txt'
            try:
                if str(self.path) == self.savename:
                    print(self.savename)
                    self.write = open(self.savename, 'w', encoding='utf8')
                else:
                    os.mkdir(str(self.path))
                    self.savename = str(self.path) + str(self.name) + '.txt'
                    self.write = open(self.savename, 'w', encoding='utf8')
            except:
                self.Path_error()

    def folderSelect(self):
        dirName = QFileDialog.getExistingDirectory(self, 'open', '/Users/picardy/Downloads/Seungkyu/copied/')
        self.files = []
        if dirName:
            for file in glob.glob(os.path.join(dirName, '*.png')):
                self.files.append(file)
            for file in glob.glob(os.path.join(dirName, '*.jpg')):
                self.files.append(file)
            self.image = cv2.imread(self.files[self.idx])
            self.dir_h, self.dir_w, channel = self.image.shape
            self.qPixmapVar.load(self.files[0])
            self.Image_label.setPixmap(self.qPixmapVar)
            QMainWindow.resize(self, self.dir_w, self.dir_h)

            folderName = self.files[self.idx]
            self.name = folderName[folderName.rfind("/"): - 4]
            self.savename = str(self.path) + str(self.name) + '.txt'
            try:
                if str(self.path) == self.savename:
                    print(self.savename)
                    self.write = open(self.savename, 'w', encoding='utf8')
                else:
                    os.mkdir(str(self.path))
                    self.savename = str(self.path) + str(self.name) + '.txt'
                    self.write = open(self.savename, 'w', encoding='utf8')
            except:
                self.Path_error()

    def NextClick(self):
        try:
            if self.idx < len(self.files):
                self.idx += 1
                self.qPixmapVar.load(self.files[self.idx])
                self.Image_label.setPixmap(self.qPixmapVar)
                self.cv_file = cv2.imread(self.files[self.idx])
                self.dir_h, self.dir_w, channel = self.cv_file.shape
                QMainWindow.resize(self, self.dir_w, self.dir_h)

                folderName = self.files[self.idx]
                self.name = folderName[folderName.rfind("/"): - 4]
                self.savename = str(self.path) + str(self.name) + '.txt'
                self.write = open(self.savename, 'w', encoding='utf8')
                # print(self.idx)
                print(self.dir_w, self.dir_h)

        except IndexError:
            self.endOfimage()
            self.idx -= 1
        except AttributeError:
            self.endOfimage()
            self.idx -= 1

    def endOfimage(self):
        msg = QMessageBox()
        msg.setWindowTitle('Error!')
        msg.setIcon(QMessageBox.Warning)
        msg.setText('마지막 이미지 입니다.')
        msg.setStandardButtons(QMessageBox.Ok)
        result = msg.exec_()

    def PreviousClick(self):
        if self.idx > 0:
            self.idx -= 1
            self.qPixmapVar.load(self.files[self.idx])
            self.Image_label.setPixmap(self.qPixmapVar)
            self.cv_file = cv2.imread(self.files[self.idx])
            self.dir_h, self.dir_w, channel = self.cv_file.shape
            QMainWindow.resize(self, self.dir_w, self.dir_h)
            folderName = self.files[self.idx]
            self.name = folderName[folderName.rfind("/"): - 4]
            self.savename = str(self.path) + str(self.name) + '.txt'
            self.write = open(self.savename, 'w', encoding='utf8')
            # print(self.idx)
            print(self.dir_w, self.dir_h)
        else:
            self.firstImage()

    def firstImage(self):
        msg = QMessageBox()
        msg.setWindowTitle('Error!')
        msg.setIcon(QMessageBox.Warning)
        msg.setText('첫 번째 이미지 입니다.')
        msg.setStandardButtons(QMessageBox.Ok)
        result = msg.exec_()

# mouse Draw Event

    def paintEvent(self, event):
        painter = QPainter(self)
        painter.drawPixmap(QPoint(), self.qPixmapVar)

        if not self.start.isNull() and not self.end.isNull():
            rect = QRect(self.start, self.end)
            painter.setPen(QPen(Qt.red, 2, Qt.SolidLine))
            painter.drawRect(rect.normalized())

    def mousePressEvent(self, event):
        if event.buttons() & Qt.LeftButton:
            self.start = event.pos()
            self.end = self.start
            self.update()

    def mouseMoveEvent(self, event):
        if event.buttons() & Qt.LeftButton:
            self.end = event.pos()
            self.update()

    def mouseReleaseEvent(self, event):
        if not event.buttons() & Qt.LeftButton:
            if self.flag == 0:
                rect = QRect(self.start, self.end)
                painter = QPainter(self.qPixmapVar)
                painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
                painter.drawRect(rect.normalized())
                self.fileName()
                print('x0, y0 =', self.start.x(), self.start.y())
                print('x1, y1 =', self.end.x(), self.end.y())
                self.start, self.end = QPoint(), QPoint()
                self.update()
            elif self.flag == 1:
                rect = QRect(self.start, self.end)
                painter = QPainter(self.qPixmapVar)
                painter.setPen(QPen(Qt.yellow, 2, Qt.SolidLine))
                painter.drawRect(rect.normalized())
                self.fileName()
                print('x0, y0 =', self.start.x(), self.start.y())
                print('x1, y1 =', self.end.x(), self.end.y())
                self.start, self.end = QPoint(), QPoint()
                self.update()
            elif self.flag == 2:
                rect = QRect(self.start, self.end)
                painter = QPainter(self.qPixmapVar)
                painter.setPen(QPen(Qt.blue, 2, Qt.SolidLine))
                painter.drawRect(rect.normalized())
                self.fileName()
                print('x0, y0 =', self.start.x(), self.start.y())
                print('x1, y1 =', self.end.x(), self.end.y())
                self.start, self.end = QPoint(), QPoint()
                self.update()

    def fileName(self):
        # folderName = self.files[self.idx]
        # self.name = folderName[folderName.rfind("/"): - 4]
        # self.savename = str(self.path) + str(self.name) + '.txt'
        # if self.flag == 0:
        #     with open(self.savename, 'w', encoding='utf8') as self.write:
        if self.flag == 0:
            self.write.write('{4} 0 0 0 {0} {1} {2} {3} 0 0 0 0 0 0 0\n'
                             .format(self.start.x(), self.start.y(), self.end.x(), self.end.y(), self.Label_G))

        if self.flag == 1:
            self.write.write('{4} 0 0 0 {0} {1} {2} {3} 0 0 0 0 0 0 0\n'
                             .format(self.start.x(), self.start.y(), self.end.x(), self.end.y(), self.Label_Y))

        if self.flag == 2:
            self.write.write('{4} 0 0 0 {0} {1} {2} {3} 0 0 0 0 0 0 0\n'
                             .format(self.start.x(), self.start.y(), self.end.x(), self.end.y(), self.Label_B))

    def Path_error(self):
        msg = QMessageBox()
        msg.setWindowTitle('Error!')
        msg.setIcon(QMessageBox.Warning)
        msg.setText('Label 경로를 설정하세요')
        msg.setStandardButtons(QMessageBox.Ok)
        msg.buttonClicked.connect(self.msg_Button)
        result = msg.exec_()

    def msg_Button(self):
        self.close()

    def openTool(self):
        Label_Dialog(self)

#change_Color

    def greenColor(self):
        self.flag = 0
        self.update()
        print(self.flag)

    def yellowColor(self):
        self.flag = 1
        self.update()
        print(self.flag)

    def blueColor(self):
        self.flag = 2
        self.update()
        print(self.flag)

app = QApplication(sys.argv)
myWindow = ViewerClass(None)
myWindow.show()
app.exec_()