import string

from .numbers_processing import retrieve_ordinal, check_convert_roman
from .numbers_processing import contain_numbers, extract_number_suffix, extract_number
from autocorrect import Speller
from spellchecker import SpellChecker
from . import settings
from .package_logging import log

"""
Module that prepossesses and cleans the date string
"""

def keywords_to_string(keywords):
    """
    Takes the keyword dictionary and transforms the existing keywords into a correctly formatted string

    :param keywords: Dictionary of keywords
    :type keywords: dict
    :return: Correctly formatted string
    :rtype: str
    """
    if keywords["part"]["active"]:
        return " (" + " ".join(keywords["part"]["active"]) + ")"
    elif keywords["split"]["active"] is not None:
        return " (" + str(keywords["ordinal"]["active"]) + " " + str(keywords["split"]["active"]) + ")"
    return ""

def remove_stopwords(text):
    """
    Removes stopwords from the string

    :param text: String to parse and clean
    :type text: list
    :return: Cleaned string
    :rtype: list
    """
    stops = ["and", "or", "of", "the", "to"]
    return [word for word in text if word not in stops]

def remove_ordinal_suffix(text):
    """
    Removes ordinal suffix from a string

    :param text: Date to clean
    :type text: list
    :return: Cleaned string without ordinal suffix
    :rtype: list
    """
    while "st" in text:
        text.remove("st")
    while "rd" in text:
        text.remove("rd")
    while "nd" in text:
        text.remove("nd")
    while "th" in text:
        text.remove("th")

def remove_punct(text, dot=False, replace=""):
    """
    Remove the punctuation from the string
    :param text: Words of the date to clean
    :type text: str
    :param dot: Boolean if dots must be taken into account of not
    :type dot: bool
    :param replace: Character by which punctuation must be replaced, defaults to ""
    :type replace: str
    :return: Word without punctuation
    :type: str
    """
    punct = string.punctuation.replace("-", "")
    if not dot:
        punct = punct.replace(".", "")
    return "".join([char if char not in punct else replace for char in text])

def add_millennium_century(time, text, prepro):
    """
    Add a missing century/millennium keyword if there is a ordinal number without it

    :param time: Time dictionary with date metadata
    :type time: dict
    :param text: Original inputted string
    :type text: list
    :param prepro: Preprocessed string
    :type prepro: list
    """
    if "millennium" in text:
        prepro.append("millennium")
    elif "century" in text:
        prepro.append("century")
    else:
        corrected_text = spell_corrector(time["date_english"].split(), time["date_original_lang"])
        if "millennium" in corrected_text:
            prepro.append("millennium")
        else:
            prepro.append("century")

def check_ordinal_without_century(prepro, time):
    """
    Checks if there is an ordinal word without the "century" keyword.

    :param prepro: Preprocessed string to parse
    :type prepro: list
    :param time: Time dictionary with date metadata
    :type time: dict
    :return: Preprocessed string with added "century" keyword if its missing
    :rtype: str
    """
    #text = time["date_original"].split()
    text = time["date_english"].split()
    if len(prepro) == 1:
        if contain_numbers(prepro[0]) and extract_number_suffix(prepro[0]) in ["st", "nd", "rd", "th"]:
            add_millennium_century(time, text, prepro)
    elif len(prepro) == 2:
        if prepro[0].isnumeric() and prepro[1] in ["st", "nd", "rd", "th"]:
            add_millennium_century(time, text, prepro)
    if len(prepro) > 1:
        if contain_numbers(prepro[-1]) and extract_number_suffix(prepro[-1]) in ["st", "nd", "rd", "th"]:
            add_millennium_century(time, text, prepro)
    if len(prepro) > 2:
        if prepro[-2].isnumeric() and prepro[-1] in ["st", "nd", "rd", "th"]:
            add_millennium_century(time, text, prepro)
    return prepro

def spell_corrector(text, lang):
    """
    Corrects possible spelling mistakes in the string
    thanks to the SpellChecker API

    :param text: Date to clean
    :type text: list
    :param lang: Language of the string
    :type lang: str
    :return: String corrected from spelling mistakes
    :rtype: list
    """
    corrected = text
    keywords = ["century", "centuries", "c.", "millennium"]
    inter = [w for w in text if w in keywords]
    if inter:  # if contains keyword
        return corrected
    elif len(text) == 1 and contain_numbers(text[0]):  # if it's one word and contains a number
        return corrected
    elif len(text) == 2 and contain_numbers(text[1]) and text[0] == "ca.":
        return corrected
    for word in text:
        if word.isnumeric() and int(word) > 30:  # if there is a number present which is bigger than 30
            return corrected
    if lang in ["en", "pl", "ru", "uk", "tr", "es", "cs"]:
        spell = Speller(lang)
        corrected = [spell(word) if not contain_numbers(word) and word != "mid" else word for word in text]
    elif lang in ["fr", "de", "pt"]:
        spell = SpellChecker(language=lang)
        corrected = [spell.correction(word) if not contain_numbers(word) and word != "mid" else word for word in text]
    return corrected

def small_corrections(text):
    """
    Implements small corrections: "century" for "cen.", "centuries", "c."

    :param text: String to parse
    :type text: list
    :return: Cleaned string
    :rtype: list
    """
    new_text = []
    if "centuries" in text or "cen." in text:
        text = ["century" if w in ["centuries", "cen."] else w for w in text]
    for idx, word in enumerate(text):
        if word in ["c.", "c"] and idx != 0 and contain_numbers(text[idx - 1]) and extract_number(text[idx - 1]) < 22:
            new_text.append("century")
        elif contain_numbers(word):
            nbr = extract_number(word)
            suffix = extract_number_suffix(word)
            if nbr and suffix and suffix == retrieve_ordinal(nbr) + "c":
                new_text.append(str(nbr))
                new_text.append("century")
            else:
                new_text.append(word)
        else:
            new_text.append(word)
    return new_text

def check_comma_nbr(preprocessed):
    """
    Checks if there is a number with a comma in it (for ex: "1,000") and correct the word if present

    :param preprocessed: String to parse
    :type preprocessed: list
    :return: Cleaned string
    :rtype: list
    """
    new_text = []
    for word in preprocessed:
        if "," in word and contain_numbers(word):
            splitted = word.split(",")
            if len(splitted) == 2 and splitted[0].isnumeric() and splitted[1].isnumeric() and len(splitted[1]) == 3:
                new_text.append("".join(splitted))
            else:
                new_text.append(word)
        else:
            new_text.append(word)
    return new_text

def preprocess_text(text, time, correct_spelling=True, chronontology=False):
    """
    Preprocess the text in order to have a cleaned string. Steps
    - Split numeric and characters from the string
    - Remove the punctuations except dots
    - Convert roman numbers
    - Convert into lowercases

    :param text: Date to clean
    :type text: str
    :param time: Dictionary with date metadata
    :type time: dict
    :param correct_spelling: Boolean that indicates if the spelling must be corrected or not, defaults to True
    :type correct_spelling: bool
    :return: Cleaned string
    :rtype: list
    """
    if settings.__DEBUG__:
        log("Preprocessing string \"" + text + "\"")
    preprocessed = text
    if time["date_original_lang"] is None or time["date_original_lang"] == "en":
        printable = string.printable
        preprocessed = "".join([w if w in printable else " " for w in text])
    preprocessed = preprocessed.lower().split(" ")
    preprocessed = check_comma_nbr(preprocessed)
    preprocessed = " ".join([remove_punct(w, replace=" ") for w in preprocessed])
    preprocessed = preprocessed.split(" ")
    preprocessed = [w for w in preprocessed if w != ""]
    if not chronontology:
        preprocessed = check_convert_roman(preprocessed)
    if correct_spelling:
        preprocessed = spell_corrector(preprocessed, time["date_original_lang"])
    preprocessed = small_corrections(preprocessed)
    preprocessed = remove_stopwords(preprocessed)
    preprocessed = check_ordinal_without_century(preprocessed, time)
    if preprocessed and preprocessed[-1][-1] == ".":  # Not sure why this is needed
        preprocessed[-1] = preprocessed[-1][:-1]
    if settings.__DEBUG__:
        log(f"Preprocessed string is {preprocessed}")
    return preprocessed


from __future__ import print_function
from __future__ import division

import os
import sys
import time
import os.path as osp
import numpy as np

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn

from args import argument_parser, image_dataset_kwargs
from torchreid.multi_data_manager import ImageDataManager
from torchreid import models
from torchreid.utils.iotools import check_isfile
from torchreid.utils.avgmeter import AverageMeter
from torchreid.utils.loggers import Logger
from torchreid.utils.torchtools import count_num_param
from torchreid.utils.reidtools import visualize_ranked_results
from torchreid.eval_metrics import evaluate

import logging
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'CRITICAL'))

# global variables
parser = argument_parser()
args = parser.parse_args()

os.environ['TORCH_HOME'] = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '.torch'))

def accuracy(output, target, topk=(1,)):
    """Computes the accuracy over the k top predictions for
    the specified values of k.
    Args:
        output (torch.Tensor): prediction matrix with shape (batch_size, num_classes).
        target (torch.LongTensor): ground truth labels with shape (batch_size).
        topk (tuple, optional): accuracy at top-k will be computed. For example,
            topk=(1, 5) means accuracy at top-1 and top-5 will be computed.
    Returns:
        list: accuracy at top-k.
    Examples::
        >>> from torchreid import metrics
        >>> metrics.accuracy(output, target)
    """
    maxk = max(topk)
    batch_size = target.size(0)

    if isinstance(output, (tuple, list)):
        output = output[0]

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        acc = correct_k.mul_(100.0 / batch_size)
        res.append(acc)

    return res

def get_criterion(num_classes: int, use_gpu: bool, args):

    if args.criterion == 'htri':

        from torchreid.losses.hard_mine_triplet_loss import TripletLoss
        criterion = TripletLoss(num_classes, vars(args), use_gpu)

    elif args.criterion == 'xent':

        from torchreid.losses.cross_entropy_loss import CrossEntropyLoss
        criterion = CrossEntropyLoss(num_classes, use_gpu=use_gpu, label_smooth=args.label_smooth)
    else:
        raise RuntimeError('Unknown criterion {}'.format(args.criterion))

    return criterion

def main():
    global args

    if not args.evaluate:
        raise RuntimeError('Test only!')

    torch.manual_seed(args.seed)
    if not args.use_avai_gpus:
        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
    use_gpu = torch.cuda.is_available()
    if args.use_cpu:
        use_gpu = False
    log_name = 'log_test.txt' if args.evaluate else 'log_train.txt'
    log_fn = osp.join(args.save_dir, log_name)
    sys.stderr = sys.stdout = Logger(log_fn)
    print("==========\nArgs:{}\n==========".format(args))

    if use_gpu:
        print("Currently using GPU {}".format(args.gpu_devices))
        cudnn.benchmark = True
        torch.cuda.manual_seed_all(args.seed)
    else:
        print("Currently using CPU, however, GPU is highly recommended")

    print("Initializing image data manager")
    dm = ImageDataManager(use_gpu, **image_dataset_kwargs(args))
    trainloader, testloader_dict = dm.return_dataloaders()

    print("Initializing model: {}".format(args.arch))
    model = models.init_model(name=args.arch, num_classes=dm.num_train_pids, loss={'xent'}, use_gpu=use_gpu, args=vars(args))
    print(model)
    print("Model size: {:.3f} M".format(count_num_param(model)))

    if args.load_weights and check_isfile(args.load_weights):
        # load pretrained weights but ignore layers that don't match in size
        try:
            checkpoint = torch.load(args.load_weights)
        except Exception as e:
            print(e)
            checkpoint = torch.load(args.load_weights, map_location={'cuda:0': 'cpu'})

        pretrain_dict = checkpoint['state_dict']
        model_dict = model.state_dict()
        pretrain_dict = {k: v for k, v in pretrain_dict.items() if k in model_dict and model_dict[k].size() == v.size()}
        model_dict.update(pretrain_dict)
        model.load_state_dict(model_dict)
        print("Loaded pretrained weights from '{}'".format(args.load_weights))

    if args.resume and check_isfile(args.resume):
        checkpoint = torch.load(args.resume)
        state = model.state_dict()
        state.update(checkpoint['state_dict'])
        model.load_state_dict(state)
        # args.start_epoch = checkpoint['epoch'] + 1
        print("Loaded checkpoint from '{}'".format(args.resume))
        print("- start_epoch: {}\n- rank1: {}".format(args.start_epoch, checkpoint['rank1']))

    if use_gpu:
        model = nn.DataParallel(model).cuda()

    if args.evaluate:
        print("Evaluate only")

        for name in args.target_names:
            print("Evaluating {} ...".format(name))
            distmat = test(model, testloader_dict[name], use_gpu, return_distmat=True)

            if args.visualize_ranks:
                visualize_ranked_results(
                    distmat, dm.return_testdataset_by_name(name),
                    save_dir=osp.join(args.save_dir, 'ranked_results', name),
                    topk=20
                )
        return

def test(model, loaders, use_gpu, ranks=[1, 5, 10, 20], return_distmat=True):

    batch_time = AverageMeter()

    model.eval()

    def eval_set(name, loader):

        with torch.no_grad():
            qf, q_pids, q_camids, q_paths = [], [], [], []

            enumerator = enumerate(loader)

            for batch_idx, package in enumerator:
                end = time.time()

                (imgs, pids, camids, paths) = package
                if use_gpu:
                    imgs = imgs.cuda()

                features = model(imgs)[0]

                batch_time.update(time.time() - end)

                features = features.data.cpu()
                qf.append(features)
                q_pids.extend(pids)
                q_camids.extend(camids)
                q_paths.extend(paths)
            qf = torch.cat(qf, 0)
            q_pids = np.asarray(q_pids)
            q_camids = np.asarray(q_camids)

            print("Extracted features for {} set, obtained {}-by-{} matrix".format(name, qf.size(0), qf.size(1)))

            return qf, q_pids, q_camids

    results = {}
    for name, dct in loaders.items():
        results[name] = (eval_set('{} query'.format(name), dct['query']), eval_set('{} gallery'.format(name), dct['gallery']))

    def eval_result(gr, qr):

        gf, g_pids, g_camids = gr
        qf, q_pids, q_camids = qr

        m, n = qf.size(0), gf.size(0)
        distmat = torch.pow(qf, 2).sum(dim=1, keepdim=True).expand(m, n) + \
            torch.pow(gf, 2).sum(dim=1, keepdim=True).expand(n, m).t()
        distmat.addmm_(1, -2, qf, gf.t())
        distmat = distmat.numpy()

        print("Computing CMC and mAP")
        cmc, mAP = evaluate(distmat, q_pids, g_pids, q_camids, g_camids, use_metric_cuhk03=args.use_metric_cuhk03)

        print("Results ----------")
        print("mAP: {:.2%}".format(mAP))
        print("CMC curve")
        for r in ranks:
            print("Rank-{:<3}: {:.2%}".format(r, cmc[r - 1]))
        print("------------------")

        return [mAP, *cmc]

    metrics = []
    for name, (qr, gr) in results.items():
        metrics.append(eval_result(gr, qr))
    metrics = np.average(np.array(metrics), axis=0)
    mAP, *cmc = metrics

    print('====> Average Result')
    print("Results ----------")
    print("mAP: {:.2%}".format(mAP))
    print("CMC curve")
    for r in ranks:
        print("Rank-{:<3}: {:.2%}".format(r, cmc[r - 1]))
    print("------------------")

    return

if __name__ == '__main__':
    main()


import matplotlib
matplotlib.use('Agg')
from ligotools.readligo import loaddata, dq_channel_to_seglist, read_hdf5, FileList
from ligotools.utils import whiten, write_wavfile, reqshift, plot_whitened
import numpy as np
from os.path import exists
from os import remove
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from scipy.interpolate import interp1d
from scipy.signal import butter, filtfilt

def test_whiten():
	fs = 4096
	NFFT = 4*fs
	strain_H1, time_H1, chan_dict_H1 = loaddata("data/H-H1_LOSC_4_V2-1126259446-32.hdf5", 'H1')
	Pxx_H1, freqs = mlab.psd(strain_H1, Fs = fs, NFFT = NFFT)
	psd_H1 = interp1d(freqs, Pxx_H1)
	time = time_H1
	dt = time[1] - time[0]
	strain_H1_whiten = whiten(strain_H1,psd_H1,dt)
	assert strain_H1_whiten is not None

def test_write_wavfile():
	write_wavfile("audio/temp.wav", 4096, np.linspace(0,10,16000))
	assert exists("audio/temp.wav")
	remove("audio/temp.wav")

def test_reqshift():
	fs = 4096
	NFFT = 4*fs
	fband = [43.0, 300.0]
	strain_H1, time_H1, chan_dict_H1 = loaddata("data/H-H1_LOSC_4_V2-1126259446-32.hdf5", 'H1')
	Pxx_H1, freqs = mlab.psd(strain_H1, Fs = fs, NFFT = NFFT)
	psd_H1 = interp1d(freqs, Pxx_H1)
	time = time_H1
	dt = time[1] - time[0]
	strain_H1_whiten = whiten(strain_H1,psd_H1,dt)
	bb, ab = butter(4, [fband[0]*2./fs, fband[1]*2./fs], btype='band')
	normalization = np.sqrt((fband[1]-fband[0])/(fs/2))
	strain_H1_whitenbp = filtfilt(bb, ab, strain_H1_whiten) / normalization
	strain_H1_shifted = reqshift(strain_H1_whitenbp,fshift=400,sample_rate=fs)
	assert strain_H1_shifted is not None

def test_plot_whitened():
	fs = 4096
	NFFT = 4*fs
	fband = [43.0, 300.0]
	strain_H1, time_H1, chan_dict_H1 = loaddata("data/H-H1_LOSC_4_V2-1126259446-32.hdf5", 'H1')
	Pxx_H1, freqs = mlab.psd(strain_H1, Fs = fs, NFFT = NFFT)
	psd_H1 = interp1d(freqs, Pxx_H1)
	time = time_H1
	dt = time[1] - time[0]
	strain_H1_whiten = whiten(strain_H1,psd_H1,dt)
	bb, ab = butter(4, [fband[0]*2./fs, fband[1]*2./fs], btype='band')
	normalization = np.sqrt((fband[1]-fband[0])/(fs/2))
	strain_H1_whitenbp = filtfilt(bb, ab, strain_H1_whiten) / normalization
	timemax = time[6]
	plot_whitened(time, 1126259462.44, strain_H1_whitenbp, 'g', 'H1', timemax, strain_H1_whitenbp, 'GW150914', "png")
	assert exists('figures/'+'GW150914'+'_'+'H1'+'_matchtime.'+'png')
	remove('figures/'+'GW150914'+'_'+'H1'+'_matchtime.'+'png')


#########################################################################
# File Name: pa.py
# Author: Walker
# mail:qngskk@gmail.com
# Created Time: Thu Dec  9 14:54:19 2021
#########################################################################
# !/usr/bin/env python3

import requests
from fake_useragent import UserAgent
from lxml import etree
import csv

class dangdang(object):
    def __init__(self):
        self.url = 'http://bang.dangdang.com/books/bestsellers'
        ua = UserAgent(verify_ssl=False)
        self.headers = {'User-Agent': ua.random}

    def get_html(self, url):
        response = requests.get(url, headers=self.headers)
        html = response.text
        return html

    def parse_html(self, html):
        target = etree.HTML(html)
        pics = target.xpath('//li/div[@class="pic"]/a/img/@title')
        names = target.xpath('//li/div[@class="name"]/a/@title')
        stars = target.xpath(
            '//li/div[@class="star"]/span[@class="tuijian"]/text()')
        publisher_infos_foreign = target.xpath(
            '//li/div[@class="publisher_info"][1]/a[1]/text()')
        publisher_infos_mainland = target.xpath(
            '//li/div[@class="publisher_info"][2]/a/text()')

        price_n_s = target.xpath(
            '//li/div[@class="price"]/p[1]/span[@class="price_n"]/text()')
        price_r_s = target.xpath(
            '//li/div[@class="price"]/p[1]/span[@class="price_r"]/text()')
        price_s_s = target.xpath(
            '//li/div[@class="price"]/p[1]/span[@class="price_s"]/text()')
        price_e_s = target.xpath(
            '//li/div[@class="price"]/p[2][@class="price_e"]')

        with open('./dangdang.csv', 'a', newline='', encoding='gb18030') as f:
            csvwriter = csv.writer(f, delimiter=',')
            for name, star, author, publisher, price_n, price_r, price_s in zip(
                    names, stars, publisher_infos_foreign,
                    publisher_infos_mainland, price_n_s, price_r_s, price_s_s):
                csvwriter.writerow(
                    [name, star, author, publisher, price_n, price_r, price_s])

    def main(self):
        url = self.url
        html = self.get_html(url)
        self.parse_html(html)
        print("完成")

if __name__ == '__main__':
    spider = dangdang()
    spider.main()


import scrapy
from scrapy.exceptions import CloseSpider
from scrapy.crawler import CrawlerProcess

class PDFSpider(scrapy.Spider):
    name = "pdf_spider"
    start_urls = [
        # List of universities / websites to scrape
    ]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.total_universities = len(self.start_urls)
        self.scraped_universities = 0

    custom_settings = {
        'DEPTH_LIMIT': 3,  # Set the depth limit of the spider
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36',
        'FEED_URI': 'output.json',  # Specify the output file
        'FEED_FORMAT': 'json',  # Output format
    }

    def parse(self, response):
        # Find and save all the PDF links on the current page
        for link in response.css('a::attr(href)').getall():
            if link.endswith('.pdf'):
                yield {
                    'pdf_link': response.urljoin(link)
                }

        # Follow all the links on the current page
        for href in response.css('a::attr(href)'):
            yield response.follow(href, self.parse)

        # Increment the scraped universities count
        self.scraped_universities += 1

        # Check if all universities have been scraped
        if self.scraped_universities == self.total_universities:
            raise CloseSpider('All universities scraped')

if __name__ == "__main__":
    process = CrawlerProcess()
    process.crawl(PDFSpider)
    process.start()
    input("Press Enter to exit...")


from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *

from pygtail import Pygtail
import sys
import os
import shutil
import glob
import regex
import time

from radiolog_viewer_ui import Ui_radiolog_viewer

statusColorDict={}
statusColorDict["At IC"]=["22ff22","000000"]
statusColorDict["Available"]=["00ffff","000000"]
statusColorDict["In Transit"]=["2222ff","eeeeee"]
statusColorDict["Waiting for Transport"]=["2222ff","eeeeee"]

stateColorDict={}
stateColorDict["#ff4444"]="#eeeeee"
stateColorDict["#eeeeee"]="#ff4444"

class MyWindow(QDialog,Ui_radiolog_viewer):
    def __init__(self,parent):
        QDialog.__init__(self)
        self.parent=parent
        self.rcFileName="radiolog_viewer.rc"
        self.configFileName="./local/radiolog_viewer.cfg"
        self.readConfigFile()
        if not os.path.isdir(self.watchedDir):
            err=QMessageBox(QMessageBox.Critical,"Error","Specified directory to be watched does not exist:\n \n  "+self.watchedDir+"\n \nAborting.",
                            QMessageBox.Close,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
            err.show()
            err.raise_()
            err.exec_()
            exit(-1)
        self.ui=Ui_radiolog_viewer()
        self.ui.setupUi(self)
        self.setAttribute(Qt.WA_DeleteOnClose)
        self.panels={}
        self.panelWidth=500
        self.panelHeight=120
        self.panelSpacing=2
        # if a new panel would cause the window to expand taller than
        #  maxGridHeight, then add a column instead and start at the top
        #  of the new column
        self.maxGridHeight=600
        self.setStyleSheet("background-color:#333333")
        self.ui.groupBox.setStyleSheet("background-color:lightgray")
        # default window geometry; overridden by previous rc file
        self.x=100
        self.y=100
        self.w=self.panelWidth+20
        self.h=400
        self.fontSize=12
        self.grid=[[0]]
        self.sidebarMode=False
        self.sidebarJustToggled=False
        self.setMinimumSize(200,200)

        self.normalTableFont=QFont()
        self.normalTableFont.setPixelSize(self.fontSize)
        self.latestTableFont=QFont()
        self.latestTableFont.setPixelSize(self.fontSize)
        self.latestTableFont.setBold(True)

        self.latestCallsign=""
        self.normalFrameColor=[150,150,150]
        self.latestFrameColor=[230,230,255]
        self.normalTableColor=[200,200,200]
        self.latestTableColor=[255,255,255]

        sidebarTableFont=QFont()
        sidebarTableFont.setFamily("Segoe UI")
        sidebarTableFont.setPointSize(14)
        sidebarTableFont.setBold(True)
        self.sidebarTableWidget=QTableWidget(0,2,self)
        self.sidebarTableWidget.setFont(sidebarTableFont)
        self.ui.verticalLayout.addWidget(self.sidebarTableWidget)
        self.sidebarTableWidget.hide()
        self.sidebarTableWidget.horizontalHeader().hide()
        self.sidebarTableWidget.verticalHeader().hide()
        self.sidebarTableWidget.setStyleSheet("background:rgb(240,240,240)")
        self.sidebarTableWidget.setSelectionMode(QAbstractItemView.NoSelection)
        self.sidebarTableWidget.setEditTriggers(QAbstractItemView.NoEditTriggers)

        self.loadRcFile()
        self.setGeometry(int(self.x),int(self.y),int(self.w),int(self.h))

        self.resizeTimer=QTimer()
        self.resizeTimer.setSingleShot(True)
        self.resizeTimer.timeout.connect(self.resizeEventPost)

        self.updateClock()

#         self.parent.setStyleSheet("QMessageBox QPushButton{background-color: green}")
        self.ui.notYet=QMessageBox(QMessageBox.Information,"Waiting...","No valid radiolog file was found.\nRe-scanning every few seconds...",
                    QMessageBox.Abort,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
        self.ui.notYet.setStyleSheet("background-color: lightgray")
        self.ui.notYet.setModal(False)
        self.ui.notYet.show()
        self.ui.notYet.buttonClicked.connect(self.notYetButtonClicked)

        self.rescanTimer=QTimer(self)
        self.rescanTimer.timeout.connect(self.rescan)
        self.rescanTimer.start(2000)

        self.refreshTimer=QTimer(self)
        self.refreshTimer.timeout.connect(self.refresh)
        self.refreshTimer.timeout.connect(self.updateClock)
        self.refreshTimer.start(3000)

    def readConfigFile(self):
        # create the file (and its directory) if it doesn't already exist
        dir=os.path.dirname(self.configFileName)
        if not os.path.exists(self.configFileName):
            print("Config file "+self.configFileName+" not found.")
            if not os.path.isdir(dir):
                try:
                    print("Creating config dir "+dir)
                    os.makedirs(dir)
                except:
                    print("ERROR creating directory "+dir+" for config file.  Better luck next time.")
            try:
                defaultConfigFileName=os.path.join(os.path.dirname(os.path.realpath(__file__)),"default.cfg")
                print("Copying default config file "+defaultConfigFileName+" to "+self.configFileName)
                shutil.copyfile(defaultConfigFileName,self.configFileName)
            except:
                print("ERROR copying the config file.  Better luck next time.")

        # specify defaults here
        self.watchedDir="Z:\\"

        configFile=QFile(self.configFileName)
        if not configFile.open(QFile.ReadOnly|QFile.Text):
            warn=QMessageBox(QMessageBox.Warning,"Error","Cannot read configuration file " + self.configFileName + "; using default settings. "+configFile.errorString(),
                            QMessageBox.Ok,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
            warn.show()
            warn.raise_()
            warn.exec_()
            return
        inStr=QTextStream(configFile)
        line=inStr.readLine()
        if line!="[RadioLogViewer]":
            warn=QMessageBox(QMessageBox.Warning,"Error","Specified configuration file " + self.configFileName + " is not a valid configuration file; using default settings.",
                            QMessageBox.Ok,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
            warn.show()
            warn.raise_()
            warn.exec_()
            configFile.close()
            return

        while not inStr.atEnd():
            line=inStr.readLine()
            tokens=line.split("=")
            if tokens[0]=="watchedDir":
                self.watchedDir=tokens[1]
                print("watchedDir specification "+self.watchedDir+" parsed from config file.")
        configFile.close()

        # validation and post-processing of each item
        configErr=""

        # process any ~ characters
        self.watchedDir=os.path.expanduser(self.watchedDir)

        if configErr:
            self.configErrMsgBox=QMessageBox(QMessageBox.Warning,"Non-fatal Configuration Error(s)","Error(s) encountered in config file "+self.configFileName+":\n\n"+configErr,
                             QMessageBox.Ok,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
            self.configErrMsgBox.exec_()

    def notYetButtonClicked(btn):
        exit()

    def rescan(self):
        print("scanning "+self.watchedDir+" for latest valid csv file...")
        self.csvFiles=[]
        self.log=[]
        self.callsigns=[]
        self.sidebarTableWidget.setRowCount(0) # delete all rows
        for callsign in self.panels:
            self.ui.gridLayout.removeWidget(self.panels[callsign])
            self.panels[callsign].setParent(None)
            self.panels[callsign].deleteLater()
        self.panels={}
        self.readDir()
        if self.csvFiles!=[]:
            self.rescanTimer.stop()
            self.ui.notYet.close()
            self.watchedFile=self.csvFiles[0][0]
            self.setWindowTitle("RadioLog Viewer - "+os.path.basename(self.watchedFile))
            # remove the pygtail offset file, if any, so pygtail will
            #  read from the beginning even if this file has already
            #  been read by pygtail
            self.offsetFileName=self.watchedFile+".offset"+str(os.getpid())
            if os.path.isfile(self.offsetFileName):
                os.remove(self.offsetFileName)
            print("  found "+self.watchedFile)
            self.refresh(throb=False)
            if self.latestCallsign!="":
                self.panels[self.latestCallsign].throb(self.latestFrameColor)

    # refresh - this is the main loop
    #  - read any new lines from the log file
    #  - process each new line
    #    - add a new panel (and place it correctly) if it's a new callsign
    #    - add a row to the appropriate panel's table
    def refresh(self,throb=True):
        if self.csvFiles!=[]:
            newEntries=self.readWatchedFile()
            if newEntries:
                for entry in newEntries:
                    if len(entry)==10:
                        time,tf,callsign,msg,radioLoc,status,epoch,d1,d2,d3=entry

                        # add a new panel if needed
                        if not callsign in self.callsigns:
                            st=self.sidebarTableWidget
                            newRowCount=st.rowCount()+1
                            st.setRowCount(newRowCount)
                            t1=QTableWidgetItem(callsign)
                            t2=QTableWidgetItem(time)
                            st.setItem(newRowCount-1,0,t1)
                            st.setItem(newRowCount-1,1,t2)
                            self.newPanel(callsign)

                        # update sidebar table
                        st=self.sidebarTableWidget
                        stRow=0
                        for n in range(st.rowCount()):
                            if st.item(n,0):
                                if st.item(n,0).text()==callsign:
                                    stRow=n
                        st1=QTableWidgetItem(callsign)
                        st2=QTableWidgetItem(time)
#                         st2.setBackground(QColor(statusColorDict.get(status,["eeeeee",""])[0]))
                        st.setItem(stRow,0,st1) # redundant, but, will highlight any table sync issues
                        st.setItem(stRow,1,st2)
#                         st.item(stRow,1).setData(Qt.BackgroundRole,QVariant(QColor(statusColorDict.get(status,["eeeeee",""])[0])))
#                         st.item(stRow,1).setData(Qt.BackgroundRole,QVariant(QColor(255,0,0)))
#                         st.item(stRow,1).setBackground(QColor(200,0,0))
#                         st.item(stRow,1).setBackground(QColor("#00ffff"))
                        print("status:"+status+"  color:"+statusColorDict.get(status,["eeeeee",""])[0])
                        st.item(stRow,1).setBackground(QColor("#"+statusColorDict.get(status,["eeeeee",""])[0]))

                        # add new entry to the callsign's panel's table
                        p=self.panels[callsign]
                        p.statusWidget.setText(status)
                        p.statusWidget.setStyleSheet("color:#"+statusColorDict.get(status,["","000000"])[1]+";background-color:#"+statusColorDict.get(status,["eeeeee",""])[0])
                        t=p.tableWidget
                        newRowCount=t.rowCount()+1
                        t.setRowCount(newRowCount)
                        if tf=="TO":
                            tf=">"
                        elif tf=="FROM":
                            tf="<"
                        else:
                            tf="?"

                        t1=QTableWidgetItem(time)
                        t2=QTableWidgetItem(tf)
                        t3=QTableWidgetItem(msg)

                        r=newRowCount-2
                        t.setItem(r,0,t1)
                        t.setItem(r,1,t2)
                        t.setItem(r,2,t3)

                        self.setRowColor(t,r,"#ff4444")

                        if newRowCount>2:
                            t.item(r-1,0).setFont(self.normalTableFont)
                            t.item(r-1,1).setFont(self.normalTableFont)
                            t.item(r-1,2).setFont(self.normalTableFont)

                        t.item(r,0).setFont(self.latestTableFont)
                        t.item(r,1).setFont(self.latestTableFont)
                        t.item(r,2).setFont(self.latestTableFont)

                        t1.setTextAlignment(Qt.AlignLeft|Qt.AlignTop)
                        t2.setTextAlignment(Qt.AlignLeft|Qt.AlignTop)
                        t3.setTextAlignment(Qt.AlignLeft|Qt.AlignTop)

                        t.resizeRowToContents(r)
                        y=t.verticalHeader().sectionSize(r)
                        y2=y
                        if y==30:
                            y2=15
                            t.verticalHeader().resizeSection(r,y2)

    #                     print(callsign+":"+time+"  row size:"+str(y)+"-->"+str(y2))

                        t.verticalHeader().setStyleSheet("border:0px;padding:-1px;margin:-1px")
                        t.verticalHeader().setMinimumSectionSize(1)
                        t.scrollToBottom()
                        self.setColors(p,throb)
                        self.latestCallsign=callsign

    def setRowColor(self,table,row,color):
        for col in range(table.columnCount()):
            table.item(row,col).setBackground(QColor(color))

    def tableCellClicked(self,row,col):
        table=self.sender()
        i=table.item(row,col)
        if i:
            prevColor=i.background().color().name()
            newColor=stateColorDict.get(prevColor,"#cccccc")
            self.setRowColor(table,row,newColor)

    def setColors(self,panel,throb):
        c=self.normalFrameColor
        t1=self.normalTableColor
        t2=self.latestTableColor
        for callsign in self.panels:
            p1=self.panels[callsign]
            p1.setFrameShape(QFrame.Panel)
            p1.setFrameShadow(QFrame.Sunken)
            p1.palette.setColor(QPalette.Background,QColor(c[0],c[1],c[2]))
            p1.setStyleSheet("background:rgb("+str(c[0])+","+str(c[1])+","+str(c[2])+")")
            p1.tableWidget.setStyleSheet("background:rgb("+str(t1[0])+","+str(t1[1])+","+str(t1[2])+")")
        panel.setFrameShape(QFrame.Panel)
        panel.setFrameShadow(QFrame.Raised)
        panel.tableWidget.setStyleSheet("background:rgb("+str(t2[0])+","+str(t2[1])+","+str(t2[2])+")")
        if throb:
            panel.throb(self.latestFrameColor)

    # get a list of non-clueLog filenames, modification times, and sizes
    #  in the watchedDir, sorted by modification time (so that the most recent
    #  file is the first item in the list)
    def readDir(self):
        f=glob.glob(self.watchedDir+"\\*.csv")
        f=[x for x in f if not regex.match('.*_clueLog.csv$',x)]
        f=[x for x in f if not regex.match('.*_fleetsync.csv$',x)]
        f=[x for x in f if not regex.match('.*_bak[123456789].csv$',x)]
        f=sorted(f,key=os.path.getmtime,reverse=True)
        for file in f:
            l=[file,os.path.getsize(file),os.path.getmtime(file)]
            self.csvFiles.append(l)

    def readWatchedFile(self):
        newEntries=[]
        for line in Pygtail(self.watchedFile,offset_file=self.offsetFileName):
            newEntries.append(line.split(','))
        return newEntries

    def newPanel(self,callsign):
        self.callsigns.append(callsign)
        panel=Panel(self,callsign)
        self.panels[callsign]=panel
        panel.titleWidget.setText(callsign)
        t=panel.tableWidget
        t.cellClicked.connect(self.tableCellClicked)
        t.horizontalHeader().setMinimumSectionSize(15)
        t.horizontalHeader().setStretchLastSection(True)
        t.verticalHeader().setDefaultSectionSize(15)
        t.verticalHeader().setMinimumSectionSize(15)
        t.horizontalHeader().hide()
        t.verticalHeader().hide()
        t.setSelectionMode(QAbstractItemView.NoSelection)
        t.setEditTriggers(QAbstractItemView.NoEditTriggers)
        t.setObjectName(callsign.replace(" ",""))
        t.setMaximumHeight(self.panelHeight-40)
        t.setMinimumHeight(self.panelHeight-40)
        t.setMaximumWidth(self.panelWidth-8)
        t.setMinimumWidth(self.panelWidth-8)
        t.setGeometry(QRect(0,0,300,200))
        t.setRowCount(1)
        self.addPanelToGrid(panel)
        if self.sidebarMode:
            panel.hide()

    def redoGrid(self):
#         self.ui.gridLayoutWidget = QWidget(self)
#         self.ui.gridLayoutWidget.setGeometry(QRect(218, 102, 597, 689))
#         self.ui.gridLayoutWidget.setObjectName("gridLayoutWidget")
#         self.ui.gridLayout = QGridLayout(self.ui.gridLayoutWidget)
        self.ui.gridLayout=QGridLayout()
        self.ui.gridLayout.setContentsMargins(0, 0, 0, 0)
        self.ui.gridLayout.setSpacing(2)
        self.ui.gridLayout.setObjectName("gridLayout")
        self.ui.verticalLayout.addLayout(self.ui.gridLayout)
#         self.ui.gridLayoutWidget.show()
#         self.ui.gridLayoutWidget.raise_()
        for callsign in self.panels:
            self.addPanelToGrid(self.panels[callsign])

    def addPanelToGrid(self,panel):
        [row,col]=self.getFirstOpenGridLocation()
        print("adding panel "+panel.callsign+" at row="+str(row)+" col="+str(col))
        self.ui.gridLayout.addWidget(panel,row,col)
        s=max(self.ui.gridLayout.horizontalSpacing(),self.ui.gridLayout.verticalSpacing())
        gridHeight=(self.panelHeight+s)*self.ui.gridLayout.rowCount()-s
        gridWidth=(self.panelWidth+s)*self.ui.gridLayout.columnCount()-s
#         self.ui.gridLayoutWidget.setGeometry(QRect(0,50,gridWidth,gridHeight))

    def getFirstOpenGridLocation(self):
        rows=self.ui.gridLayout.rowCount()
        cols=self.ui.gridLayout.columnCount()
#         print("the grid currently has "+str(rows)+" rows and "+str(cols)+" columns")
        for c in range(cols):
            for r in range(min(rows,int(self.maxGridHeight/self.panelHeight))+1):
#                 print("checking row:"+str(r)+" col:"+str(c)+" = "+str(self.ui.gridLayout.itemAtPosition(r,c)))
                if not self.ui.gridLayout.itemAtPosition(r,c):
#                     print("empty cell found at row="+str(r)+" col="+str(c))
                    return([r,c])
        return([0,c+1])

    def updateClock(self):
        self.ui.clock.display(time.strftime("%H:%M"))

    def saveRcFile(self):
        print("saving...")
        (self.x,self.y,self.w,self.h)=self.geometry().getRect()
        rcFile=QFile(self.rcFileName)
        if not rcFile.open(QFile.WriteOnly|QFile.Text):
            warn=QMessageBox(QMessageBox.Warning,"Error","Cannot write resource file " + self.rcFileName + "; proceeding, but, current settings will be lost. "+rcFile.errorString(),
                            QMessageBox.Ok,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
            warn.show()
            warn.raise_()
            warn.exec_()
            return
        out=QTextStream(rcFile)
        out << "[RadioLog Viewer]\n"
        out << "font-size=" << self.fontSize << "pt\n"
        out << "x=" << self.x << "\n"
        out << "y=" << self.y << "\n"
        out << "w=" << self.w << "\n"
        out << "h=" << self.h << "\n"
        rcFile.close()

    def loadRcFile(self):
        print("loading...")
        rcFile=QFile(self.rcFileName)
        if not rcFile.open(QFile.ReadOnly|QFile.Text):
            warn=QMessageBox(QMessageBox.Warning,"Error","Cannot read resource file " + self.rcFileName + "; using default settings. "+rcFile.errorString(),
                            QMessageBox.Ok,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
            warn.show()
            warn.raise_()
            warn.exec_()
            return
        inStr=QTextStream(rcFile)
        line=inStr.readLine()
        if line!="[RadioLog Viewer]":
            warn=QMessageBox(QMessageBox.Warning,"Error","Specified resource file " + self.rcFileName + " is not a valid resource file; using default settings.",
                            QMessageBox.Ok,self,Qt.WindowTitleHint|Qt.WindowCloseButtonHint|Qt.Dialog|Qt.MSWindowsFixedSizeDialogHint|Qt.WindowStaysOnTopHint)
            warn.show()
            warn.raise_()
            warn.exec_()
            rcFile.close()
            return
        while not inStr.atEnd():
            line=inStr.readLine()
            tokens=line.split("=")
            if tokens[0]=="x":
                self.x=int(tokens[1])
            elif tokens[0]=="y":
                self.y=int(tokens[1])
            elif tokens[0]=="w":
                self.w=int(tokens[1])
            elif tokens[0]=="h":
                self.h=int(tokens[1])
            elif tokens[0]=="font-size":
                self.fontSize=int(tokens[1].replace('pt',''))
        rcFile.close()

        # make sure specified screen position and size will fit on the available screens
        #  (in case rc file was saved on dual display but now running on single)
        desktop=QApplication.desktop()
        h=desktop.height()
        w=desktop.width()
        print("Available desktop size: "+str(w)+" x "+str(h))
        if self.x>w:
            self.x=self.x-w # assume the same first screen was used when rc was saved
        if self.x>w:
            self.x=200 # if still off the screen, use a hardcoded fallback
        if self.w+self.x>w:
            self.w=w-self.x-100 # cut it off 100px from the right edge of the screen

    def resizeEvent(self,event):
        self.resizeTimer.start(500)
        event.accept()

    def resizeEventPost(self):
        self.maxGridHeight=self.height()-190
        if not self.sidebarJustToggled:
            self.redoGrid()
            self.sidebarJustToggled=False

    def sidebarToggle(self):
        self.sidebarJustToggled=True # to prevent redoGrid during resizeEventPost
        if self.sidebarMode:
            print("Exiting sidebar mode")
            for p in self.panels:
                self.panels[p].show()
            self.sidebarTableWidget.hide()
            self.setGeometry(self.x,self.y,self.w,self.h)
            self.ui.rescanButton.show()
            self.redoGrid()
            self.sidebarMode=False
        else:
            print("Entering sidebar mode")
            for p in self.panels:
                self.panels[p].hide()
            self.sidebarTableWidget.show()
            (self.x,self.y,self.w,self.h)=self.geometry().getRect()
            self.ui.rescanButton.hide()
            self.setGeometry(0,0,200,QApplication.desktop().screenGeometry().height())
            self.sidebarMode=True

    def closeEvent(self,event):
        self.saveRcFile()
        event.accept()
        self.parent.quit()

class Panel(QFrame):
    def __init__(self,parent,callsign):
        super().__init__()
        self.parent=parent
        self.callsign=callsign
        font=QFont()
        font.setFamily("Segoe UI")
        font.setPointSize(14)
        font.setBold(True)
        smallFont=QFont()
        smallFont.setFamily("Segoe UI")
        smallFont.setPointSize(10)
        self.palette=QPalette()
        self.layout=QVBoxLayout(self)
        self.titleBarLayout=QHBoxLayout()
        self.titleWidget=QLabel("label")
        self.titleWidget.setFont(font)
        self.statusWidget=QLabel()
        self.statusWidget.setFont(smallFont)
        self.statusWidget.setAlignment(Qt.AlignHCenter|Qt.AlignVCenter)
        self.closeButton=QPushButton(self)
        self.closeButton.setMinimumHeight(20)
        self.closeButton.setMaximumHeight(20)
        self.closeButton.setMinimumWidth(20)
        self.closeButton.setMaximumWidth(20)
        self.closeButton.setStyleSheet("border:none")
        self.closeButton.clicked.connect(self.close)
        self.closeButton.setStyleSheet("""
            QPushButton:hover
                {border-image:url(:/radiolog_viewer_ui/close_dark.png)}
            QPushButton
                {border-image:url(:/radiolog_viewer_ui/close.png)}
            """)
        self.titleBarLayout.addWidget(self.titleWidget)
        self.titleBarLayout.addWidget(self.statusWidget)
        self.titleBarLayout.insertSpacing(2,20)
        self.titleBarLayout.addWidget(self.closeButton)
        self.tableWidget=QTableWidget(0,3,self)
        # default vertical header ResizeToContents behavior apparently gets
        #  rid of vertical padding on rows that have been expanded
        #  for multi-line text, but padding remains on rows that
        #  have not been expanded (only have one line of text);
        #  this is wasted space; would like to have no vertical padding
        #  regardless of number of rows; if sectionResizeMode is
        #  left at the default, there is no vertical padding (but rows
        #  do not get resized, i.e. long entries get chopped off);
        #  the following line does not change the behavior
        # minimumSectionSize gets overridden when ResizeToContents is used
#         self.tableWidget.setStyleSheet("QTableWidget:item {border:0px;padding:0px;margin:0px}")
        self.tableWidget.setWordWrap(True)
        self.tableWidget.setTextElideMode(Qt.ElideNone)
        vh=self.tableWidget.verticalHeader()
        vh.setMinimumSectionSize(20) # minimum row height
        hh=self.tableWidget.horizontalHeader()
        hh.setMinimumSectionSize(20)
        hh.resizeSection(0,40)
        hh.resizeSection(1,20)
        hh.setStretchLastSection(True)

        self.layout.insertLayout(0,self.titleBarLayout)
        self.layout.addWidget(self.tableWidget)
        self.setFrameShape(QFrame.Panel)
        self.setFrameShadow(QFrame.Raised)
        self.setLineWidth(2)
        self.statusWidget.setMinimumHeight(20)
        self.statusWidget.setMaximumHeight(20)
        self.setStyleSheet("background-color:#666666")
        self.titleBarLayout.setSpacing(2)
        self.layout.setSpacing(2)
        self.layout.setContentsMargins(2,2,2,2)
        self.setMinimumHeight(self.parent.panelHeight)
        self.setMaximumHeight(self.parent.panelHeight)
        self.setMinimumWidth(self.parent.panelWidth)
        self.setMaximumWidth(self.parent.panelWidth)
        self.throbTimer=None

    def closeEvent(self,event):
        print("closed panel "+self.callsign)
        event.accept()
        del self.parent.panels[self.callsign]
        self.parent.callsigns.remove(self.callsign)
        self.parent.redoGrid()
        if self.throbTimer:
            self.throbTimer.stop()

    def throb(self,final=[255,255,255],n=0):
        # this function calls itself recursivly 25 times to throb the background blue->white
        self.palette.setColor(QPalette.Background,QColor(n*10,n*10,255))
        self.setStyleSheet("background:rgb("+str(n*10)+","+str(n*10)+",255)")
        self.setPalette(self.palette)
        if n<25:
            self.throbTimer=QTimer()
            self.throbTimer.timeout.connect(lambda:self.throb(final,n+1))
            self.throbTimer.setSingleShot(True)
            self.throbTimer.start(25)
        else:
            self.throbTimer=None
            self.palette.setColor(QPalette.Background,QColor(final[0],final[1],final[2]))
            self.setStyleSheet("background:rgb("+str(final[0])+","+str(final[1])+","+str(final[2])+")")
            self.setPalette(self.palette)

def main():
    app = QApplication(sys.argv)
    w = MyWindow(app)
    w.show()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()


#python 3.0 이상부터는 urllib 보다 requests 모듈이 더 효율적

import sys
import io
import requests

sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding = 'utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding = 'utf-8')

s = requests.Session()
#r = s.get('https://www.naver.com')#PUT(FETCH), DELETE, GET, POST
#print('1',r.text)

#r = s.get('http://httpbin.org/cookies',cookies={'from' : 'myName'}) # 들어가보면 요청한 것을 반환해주는 연습 사이트
#print(r.text) #cookies 를 response해줌

url = 'http://httpbin.org/get'
headers = {'user-agent':'myPythonApp_1.0.0'}

#r = s.get(url, headers=headers)
#print(r.text)

s.close()

with requests.Session() as s:
    r = s.get('https://www.naver.com')
    print(r.text)


"""
Created on 11/20/2018

@author: RH
"""
# This simulator is created by using the ideas of Extended Data Figure 3.
import matplotlib
matplotlib.use('Agg')
import numpy as np
import cv2

Pr = 6  # Antibiotics radius of species P.
RDr = 3  # Removing antibiotics radius of species RD
r = 4  # Reproducing radius of all 3 species
sz = 200  # Size of simulation graph
Srepro = 1  # Number of offsprings produced by species S each round
RDrepro = 1  # Number of offsprings produced by species RD each round
Prepro = 1  # Number of offsprings produced by species P each round

# Initialize the graph with 'num' of species randomly placed
def init_placement(sz, num, x, Pr, RDr):
    mp = np.full((sz+1, sz+1, 3), 1)
    loc = []
    for i in range(num):
        rand = np.random.randint(sz, size=2)
        if x == 2:
            mp[rand[0], rand[1], x] = 255
            mp[rand[0], rand[1], 0] = 0
            mp[rand[0], rand[1], 1] = 0
        elif x == 1:
            mp[np.maximum((rand[0]-RDr), 0):np.minimum((rand[0] + RDr), sz),
            np.maximum((rand[1]-RDr), 0):np.minimum((rand[1] + RDr), sz), x] = 255

            mp[np.maximum((rand[0] - RDr), 0):np.minimum((rand[0] + RDr), sz),
            np.maximum((rand[1] - RDr), 0):np.minimum((rand[1] + RDr), sz), 0] = 0

            mp[np.maximum((rand[0] - RDr), 0):np.minimum((rand[0] + RDr), sz),
            np.maximum((rand[1] - RDr), 0):np.minimum((rand[1] + RDr), sz), 2] = 0
        elif x == 0:
            mp[np.maximum((rand[0] - Pr), 0):np.minimum((rand[0] + Pr), sz),
            np.maximum((rand[1] - Pr), 0):np.minimum((rand[1] + Pr), sz), x] = 255

            mp[np.maximum((rand[0] - Pr), 0):np.minimum((rand[0] + Pr), sz),
            np.maximum((rand[1] - Pr), 0):np.minimum((rand[1] + Pr), sz), 1] = 1

            mp[np.maximum((rand[0] - Pr), 0):np.minimum((rand[0] + Pr), sz),
            np.maximum((rand[1] - Pr), 0):np.minimum((rand[1] + Pr), sz), 2] = 0
        loc.append(rand)
    return mp, loc

# Update the graph (simulation happens: killing, protection, reproducing)
def update(sloc, rdloc, ploc, oldmp, sz, repros, reprord, reprop, r):
    mp = np.full((sz+1, sz+1, 3), 1)
    nsloc = []
    nploc = []
    nrdloc = []
    for a in sloc:
        if oldmp[a[0], a[1], 2] > 10 or oldmp[a[0], a[1], 1] > 10:
            mp[a[0], a[1], 2] = 255
            mp[a[0], a[1], 1] = 0
            mp[a[0], a[1], 0] = 0
            nsloc.append(a)
            for i in range(repros):
                rand = np.random.randint(low=-r, high=r, size=2)
                x = a[0]+rand[0]
                y = a[1]+rand[1]
                offspring = np.asarray([x,y])
                offspring = np.clip(offspring, 0, 200)
                nsloc.append(offspring)
    for c in ploc:
        nploc.append(c)
        mp[c[0], c[1], 2] = 0
        mp[c[0], c[1], 1] = 1
        mp[c[0], c[1], 0] = 255
        for j in range(reprop):
            rand = np.random.randint(low=-r, high=r, size=2)
            x = c[0] + rand[0]
            y = c[1] + rand[1]
            offspring = np.asarray([x, y])
            offspring = np.clip(offspring, 0, 200)
            if mp[offspring[0], offspring[1], :].tolist() == [1, 1, 1]:
                nploc.append(offspring)
    for b in rdloc:
        nrdloc.append(b)
        mp[b[0], b[1], 2] = 0
        mp[b[0], b[1], 1] = 255
        mp[b[0], b[1], 0] = 0
        for j in range(reprord):
            rand = np.random.randint(low=-r, high=r, size=2)
            x = b[0] + rand[0]
            y = b[1] + rand[1]
            offspring = np.asarray([x, y])
            offspring = np.clip(offspring, 0, 200)
            if mp[offspring[0], offspring[1], :].tolist() == [1, 1, 1]:
                nrdloc.append(offspring)
    return mp, nsloc, nploc, nrdloc

# Generate new graphs for next round based on the result of last round
def new_graph(loc, sz, Pr, RDr, x):
    mp = np.full((sz+1, sz+1, 3), 1)
    for i in loc:
        if x == 2:
            mp[i[0], i[1], x] = 255
            mp[i[0], i[1], 0] = 0
            mp[i[0], i[1], 1] = 0
        elif x == 1:
            mp[np.maximum((i[0]-RDr), 0):np.minimum((i[0] + RDr), sz),
            np.maximum((i[1]-RDr), 0):np.minimum((i[1] + RDr), sz), x] = 255

            mp[np.maximum((i[0] - RDr), 0):np.minimum((i[0] + RDr), sz),
            np.maximum((i[1] - RDr), 0):np.minimum((i[1] + RDr), sz), 0] = 0

            mp[np.maximum((i[0] - RDr), 0):np.minimum((i[0] + RDr), sz),
            np.maximum((i[1] - RDr), 0):np.minimum((i[1] + RDr), sz), 2] = 0
        elif x == 0:
            mp[np.maximum((i[0] - Pr), 0):np.minimum((i[0] + Pr), sz),
            np.maximum((i[1] - Pr), 0):np.minimum((i[1] + Pr), sz), x] = 255

            mp[np.maximum((i[0] - Pr), 0):np.minimum((i[0] + Pr), sz),
            np.maximum((i[1] - Pr), 0):np.minimum((i[1] + Pr), sz), 1] = 1

            mp[np.maximum((i[0] - Pr), 0):np.minimum((i[0] + Pr), sz),
            np.maximum((i[1] - Pr), 0):np.minimum((i[1] + Pr), sz), 2] = 0
    return mp

# Main method. Save process and result graph of each round and eventually generate movies of simulation.
def main(round, init_num, sz, Pr, RDr, Srepro, RDrepro, Prepro, r):
    S, Sloc = init_placement(sz, init_num, 2, Pr, RDr)
    RD, RDloc = init_placement(sz, init_num, 1, Pr, RDr)
    P, Ploc = init_placement(sz, init_num, 0, Pr, RDr)
    result = np.multiply((S+RD-1), P)
    result_view = result.repeat(5, axis=0).repeat(5, axis=1)
    cv2.imwrite('round-1.png', result_view)
    updatedmap, nsloc, nploc, nrdloc = update(Sloc, RDloc, Ploc, result, sz, Srepro, RDrepro, Prepro, r)
    updatedmap_view = updatedmap.repeat(5, axis=0).repeat(5, axis=1)
    cv2.imwrite('result-1.png', updatedmap_view)

    height, width, layers = updatedmap_view.shape
    result_video = cv2.VideoWriter('result_video.mov', -1, 1, (width, height))
    rimg = cv2.imread('result-1.png')
    result_video.write(rimg)

    height, width, layers = result_view.shape
    process_video = cv2.VideoWriter('process_video.mov', -1, 1, (width, height))
    pimg = cv2.imread('round-1.png')
    process_video.write(pimg)

    for i in range(round-1):
        S = new_graph(nsloc, sz, Pr, RDr, 2)
        RD = new_graph(nrdloc, sz, Pr, RDr, 1)
        P = new_graph(nploc, sz, Pr, RDr, 0)
        new = np.multiply((S+RD-1), P)
        new_view = new.repeat(5, axis=0).repeat(5, axis=1)
        cv2.imwrite('round-{}.png'.format(str(i+2)), new_view)
        pimg = cv2.imread('round-{}.png'.format(str(i+2)))
        process_video.write(pimg)

        updatedmap, nsloc, nploc, nrdloc = update(nsloc, nrdloc, nploc, new, sz, Srepro, RDrepro, Prepro, r)
        updatedmap_view = updatedmap.repeat(5, axis=0).repeat(5, axis=1)
        cv2.imwrite('result-{}.png'.format(str(i+2)), updatedmap_view)
        rimg = cv2.imread('result-{}.png'.format(str(i+2)))
        result_video.write(rimg)
    cv2.destroyAllWindows()
    process_video.release()
    result_video.release()

if __name__ == "__main__":
    # Run
    main(20, 15, sz, Pr, RDr, Srepro, RDrepro, Prepro, r)


from rest_framework import status
from rest_framework.response import Response
from rest_framework.decorators import api_view, permission_classes
from apps.trayectos.models import Trayecto
from apps.trayectos.api.serializers import TrayectoListSerializer, TrayectoSerializer
from .permissions import IsAdminUserCustom

@api_view(['GET', 'POST'])
@permission_classes((IsAdminUserCustom, ))
def trayecto_api_view(request):
    if request.method == 'GET':
        # queryset
        trayectos = Trayecto.objects.all()
        trayectos_serializer = TrayectoListSerializer(trayectos, many=True)
        return Response(trayectos_serializer.data, status=status.HTTP_200_OK)

    elif request.method == 'POST':
        trayecto_serializer = TrayectoSerializer(data=request.data)
        if trayecto_serializer.is_valid():
            trayecto_serializer.save()
            return Response({'message': 'Trayecto creado correctamente!'}, status=status.HTTP_201_CREATED)
        return Response(trayecto_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET', 'PUT', 'DELETE'])
@permission_classes((IsAdminUserCustom, ))
def trayecto_detail_api_view(request, id=None):
    # queryset
    trayecto = Trayecto.objects.filter(id=id).first()

    # validation
    if trayecto:

        # retrieve
        if request.method == 'GET':
            trayecto_serializer = TrayectoListSerializer(trayecto)
            return Response(trayecto_serializer.data, status=status.HTTP_200_OK)

        # update
        elif request.method == 'PUT':
            trayecto_serializer = TrayectoSerializer(
                trayecto, data=request.data)
            if trayecto_serializer.is_valid():
                trayecto_serializer.save()
                return Response(trayecto_serializer.data, status=status.HTTP_200_OK)
            return Response(trayecto_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        # delete
        elif request.method == 'DELETE':
            trayecto.delete()
            return Response({'message': 'Trayecto eliminado correctamente!'}, status=status.HTTP_200_OK)

    return Response({'message': 'No se ha encontrado un trayecto con estos datos'}, status=status.HTTP_400_BAD_REQUEST)


import cv2

from picamera import PiCamera
from time import sleep
import numpy as np
import runModel

def picture(x):
    path = '/home/pi/Desktop/'
    # start picamera
    camera = PiCamera()
    #camera.start_preview()

    #start logitech
    cap = cv2.VideoCapture(0)
    cap.set(3, 1280)
    cap.set(4, 720)

    # wait to focus cams
    sleep(3)
    # take pic from picam
    pipath = f'{path}pi{x}.jpg'
    camera.capture(pipath)
    #camera.stop_preview()

    # take pic from logitech
    _, img = cap.read()
    logpath = f'{path}log{x}.jpg'
    cv2.imwrite(logpath, img)
    cap.release()

    return pipath, logpath

def main():
    pipath, path = picture(1)
    pimg = cv2.imread(pipath, cv2.IMREAD_GRAYSCALE)
    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
    piPred = runModel.runModel(pimg)
    pred = runModel.runModel(img)
    print(piPred, pred)

if __name__ == "__main__":
    main()


from django.core.management.base import BaseCommand
from places.models import Place, Image
from django.core.files.base import ContentFile
from io import BytesIO
import requests
import json
import os

def update_or_create_place(place_dataset):

    place, created = Place.objects.update_or_create(
        title=place_dataset['title'],
        defaults={
            'description_short': place_dataset.get('description_short', ''),
            'description_long': place_dataset.get('description_long', ''),
            'lon': place_dataset['coordinates']['lng'],
            'lat': place_dataset['coordinates']['lat']
        }

    )

    if not created:
        place.photos.all().delete()

    related_images = []
    for img_number, img_url in enumerate(place_dataset['imgs'], 1):
        image = requests.get(img_url)
        image_binary_content = BytesIO(image.content).read()
        image_content_file = ContentFile(image_binary_content, name=f'{img_number} {place.title}.jpg')
        current_image = Image(
            location=place,
            photo=image_content_file
        )

        related_images.append(current_image)

    Image.objects.bulk_create(related_images)

class Command(BaseCommand):
    help = 'Creating or update a place model from JSON-file'

    def add_arguments(self, parser):
        parser.add_argument('place_dir', nargs='+', type=str)

    def handle(self, *args, **options):
        file_path = options['place_dir'][0]

        if file_path.startswith(('https://', 'http://',)):
            place_datasets = [requests.get(file_path).json() for file_path in file_path.split()]
        else:
            place_datasets = []
            for root, dirs, files in os.walk(file_path):
                for file in files:
                    if not file.endswith('.json'):
                        continue
                    with open(os.path.join(root, file), encoding='utf8') as place_file:
                        place_dataset = json.load(place_file)
                    place_datasets.append(place_dataset)

        for place_dataset in place_datasets:
            update_or_create_place(place_dataset)


import PySimpleGUI as sg
from shutil import move
from os import listdir
from os.path import isfile, join, isdir, splitext, abspath
from tkinter import messagebox

def updateFileList():
  fnames = [
        f
        for f in files
        if isfile(join(to_move_folder_path, f))
    ]
  window['scanned_files_list'].update(fnames)

def moveApplicableFiles():
  for filename in files:
    # check if the patient has a folder in the destination folder and if a file of the same name doesn't already exist in the patient's folder. if true, move the file ()

    # splitext removes the .<ext> and split separates the file name on spaces; both return an array
    patient_folder_path = join(organize_in_folder_path, splitext(filename)[0].split()[0])
    potential_file_in_dest_path = join(patient_folder_path, filename)
    file_to_be_moved_path = join(to_move_folder_path, filename)

    if isdir(patient_folder_path) and not isfile(potential_file_in_dest_path):
      move(file_to_be_moved_path, patient_folder_path)
      print('moved', file_to_be_moved_path, 'to', patient_folder_path)

# create GUI layout
layout = [[sg.Text('Scanned Files Folder'),
        sg.In('Source', size=(23, 1), enable_events=True, key='scanned_files_directory'),sg.FolderBrowse()],
        [sg.Listbox( values=[], enable_events=True, size=(40, 15), key='scanned_files_list')],
        [sg.Text('Folder of Patient Folders'),
        sg.In('Destination', size=(20, 1), enable_events=True, key='patient_folders_directory'),sg.FolderBrowse()],
        [sg.Button('Organize away!', button_color=(sg.theme_background_color()), font=('Courier', 13))]]

# create the window
window = sg.Window('File Organizer', layout)

# create an event loop
while True:
  event, values = window.read()
  # End program if user closes window
  if event == sg.WIN_CLOSED:
    break
  if event == 'scanned_files_directory':
    # get folder path
    to_move_folder_path = values['scanned_files_directory']
    print('Scanned Files Folder Path: ', to_move_folder_path)
    try:
        # Get list of files in folder
        files = listdir(to_move_folder_path)
        print('File Names:', files)
    except:
        files = []
    updateFileList() # update the file list box
  elif event == 'patient_folders_directory':
    # get folder path
    organize_in_folder_path = values['patient_folders_directory']
    print('Folder of Patient Folder Path: ', organize_in_folder_path)
  elif event == 'Organize away!':
    # move files to correct destination
    try:
      moveApplicableFiles() # moves files while avoiding duplicates
      updateFileList() # update the file list box
      # alert if files weren't moved
      if (len(listdir(to_move_folder_path)) > 0):
        messagebox.showinfo('Head\'s up!', str(len([f for f in listdir(to_move_folder_path) if isfile(join(to_move_folder_path, f))])) + ' files could not be moved. You can view unmoved files in the file list box. They may not have been moved because duplicates exist in their destination folder, the patient\'s folder hasn\'t been created yet, or the file was incorrectly named.')
      else:
        messagebox.showinfo('Success!', 'All files were able to be moved! You may close the program!')
    except:
      messagebox.showwarning('Oops!', 'Something may have went wrong. Contact the developer for help.')

window.close()


# encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http:# mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import, division, unicode_literals

from collections import deque

from jx_base.domains import SetDomain, SimpleSetDomain
from jx_base.expressions import Variable as Variable_, Variable
from jx_base.expressions.false_op import FALSE
from jx_base.expressions.query_op import DEFAULT_LIMIT
from jx_base.expressions.true_op import TRUE
from jx_base.language import is_op
from jx_elasticsearch.es52.agg_format import agg_formatters
from jx_elasticsearch.es52.agg_op_field import agg_field
from jx_elasticsearch.es52.agg_op_formula import agg_formula
from jx_elasticsearch.es52.decoders import AggsDecoder
from jx_elasticsearch.es52.es_query import Aggs, FilterAggs, NestedAggs, simplify
from jx_elasticsearch.es52.expressions import split_expression_by_path
from jx_elasticsearch.es52.expressions.utils import pre_process, query_to_outer_joins, ES52
from jx_elasticsearch.es52.painless import Painless
from jx_python import jx
from mo_dots import Data, Null, coalesce, listwrap, literal_field, unwrap, unwraplist, to_data
from mo_future import first, next, text
from mo_imports import export
from mo_logs import Log
from mo_times.timer import Timer

DEBUG = False

def is_aggsop(es, query):
    if query.edges or query.groupby or any(a != None and a != "none" for a in listwrap(query.select).aggregate):
        return True
    return False

def get_decoders_by_path(query, schema):
    """
    RETURN MAP FROM QUERY PATH TO LIST OF DECODER ARRAYS

    :param query:
    :return:
    """
    output = {}

    if query.edges:
        if query.sort and query.format != "cube":
            # REORDER EDGES/GROUPBY TO MATCH THE SORT
            query.edges = sort_edges(query, "edges")
    elif query.groupby:
        if query.sort and query.format != "cube":
            query.groupby = sort_edges(query, "groupby")

    for edge in to_data(coalesce(query.edges, query.groupby, [])):
        limit = coalesce(edge.domain.limit, query.limit, DEFAULT_LIMIT)
        vars_ = coalesce(edge.value.vars(), set())

        if edge.range:
            vars_ |= edge.range.min.vars() | edge.range.max.vars()
            for v in vars_:
                if not schema[v.var]:
                    Log.error("{{var}} does not exist in schema", var=v)
        elif edge.domain.dimension:
            vars_ |= set(Variable(v) for v in edge.domain.dimension.fields)
            edge.domain.dimension = edge.domain.dimension.copy()
            edge.domain.dimension.fields = [c.es_column for v in vars_ for c in schema[v.var]]
        elif edge.domain.partitions and all(p.where for p in edge.domain.partitions):
            for p in edge.domain.partitions:
                vars_ |= p.where.vars()
        else:
            # SIMPLE edge.value
            decoder = AggsDecoder(edge, query, limit)
            depths = list(reversed(sorted(set(n for v in vars_ for n, _ in schema.split_values(v.var).items()))))
            output.setdefault(first(depths), []).append(decoder)
            continue

        depths = set(c.nested_path[0] for v in vars_ for c in schema.leaves(v.var))
        if not depths:
            Log.error(
                "Do not know of column {{column}}",
                column=unwraplist([v for v in vars_ if schema[v.var] == None])
            )
        if len(depths) > 1:
            Log.error("expression {{expr|quote}} spans tables, can not handle", expr=edge.value)

        decoder = AggsDecoder(edge, query, limit)
        output.setdefault(first(depths), []).append(decoder)
    return output

def sort_edges(query, prop):
    ordered_edges = []
    remaining_edges = getattr(query, prop)
    for s in jx.reverse(query.sort):
        for e in remaining_edges:
            if e.value == s.value:
                if isinstance(e.domain, (SimpleSetDomain, SetDomain)):
                    pass  # ALREADY SORTED?
                else:
                    e.domain.sort = s.sort
                ordered_edges.append(e)
                remaining_edges.remove(e)
                break
        else:
            Log.error("Can not sort by {{expr}}, can only sort by an existing edge expression", expr=s.value)

    ordered_edges.extend(remaining_edges)
    for i, o in enumerate(ordered_edges):
        o.dim = i  # REORDER THE EDGES
    return ordered_edges

def extract_aggs(select, query_path, schema):
    """
    RETURN ES AGGREGATIONS
    """

    new_select = Data()  # MAP FROM canonical_name (USED FOR NAMES IN QUERY) TO SELECT MAPPING
    formula = []
    for s in select:
        if is_op(s.value, Variable_):
            s.query_path = query_path
            if s.aggregate == "count":
                new_select["count_"+literal_field(s.value.var)] += [s]
            else:
                new_select[literal_field(s.value.var)] += [s]
        elif s.aggregate:
            op, split_select = split_expression_by_path(s.value, schema, lang=Painless)
            for si_key, si_value in split_select.items():
                if si_value:
                    if s.query_path:
                        Log.error("can not handle more than one depth per select")
                    s.query_path = si_key
            formula.append(s)

    acc = Aggs()
    agg_field(acc, new_select, query_path, schema)
    agg_formula(acc, formula, query_path, schema)
    return acc

def aggop_to_es_queries(select, query_path, schema, query):
    base_agg = extract_aggs(select, query_path, schema)
    base_agg = NestedAggs(query_path).add(base_agg)

    all_paths, split_decoders, var_to_columns = pre_process(query)

    # WE LET EACH DIMENSION ADD ITS OWN CODE FOR HANDLING INNER JOINS
    concat_outer = query_to_outer_joins(query, all_paths, {}, var_to_columns)

    start = 0
    decoders = [None] * (len(query.edges) + len(query.groupby))
    output = NestedAggs(".")
    for i, outer in enumerate(concat_outer.terms):
        acc = base_agg
        for p, path in enumerate(all_paths):
            decoder = split_decoders.get(path, Null)

            for d in decoder:
                decoders[d.edge.dim] = d
                acc = d.append_query(path, acc)
                start += d.num_columns

            where = first(nest.where for nest in outer.nests if nest.path == path).partial_eval(ES52)
            if where is FALSE:
                continue
            elif not where or where is TRUE:
                pass
            else:
                acc = FilterAggs("_filter" + text(i) + text(p), where, None).add(acc)
            acc = NestedAggs(path).add(acc)
        output.add(acc)
    simpler_output = simplify(output)
    es_query = to_data(simpler_output.to_es(schema))
    es_query.size = 0
    return simpler_output, decoders, es_query

def es_aggsop(es, frum, query):
    query = query.copy()  # WE WILL MARK UP THIS QUERY
    schema = frum.schema
    query_path = schema.query_path[0]
    selects = listwrap(query.select)

    acc, decoders, es_query = aggop_to_es_queries(selects, query_path, schema, query)

    with Timer("ES query time", verbose=DEBUG) as es_duration:
        result = es.search(es_query)

    try:
        format_time = Timer("formatting", verbose=DEBUG)
        with format_time:
            if result.aggregations.doc_count == None:
                # IT APPEARS THE OLD doc_count IS GONE
                result.aggregations.doc_count = result.hits.total
            aggs = unwrap(result.aggregations)

            edges_formatter, groupby_formatter, value_fomratter, mime_type = agg_formatters[query.format]
            if query.edges:
                output = edges_formatter(aggs, acc, query, decoders, selects)
            elif query.groupby:
                output = groupby_formatter(aggs, acc, query, decoders, selects)
            else:
                output = value_fomratter(aggs, acc, query, decoders, selects)

        output.meta.timing.formatting = format_time.duration
        output.meta.timing.es_search = es_duration.duration
        output.meta.content_type = mime_type
        output.meta.es_query = es_query
        return output
    except Exception as e:
        if query.format not in agg_formatters:
            Log.error("Format {{format|quote}} not supported yet", format=query.format, cause=e)
        Log.error("Some problem", cause=e)

EMPTY = {}
EMPTY_LIST = []

def _children(agg, children):
    for child in children:
        name = child.name
        if name is None:
            yield None, agg, child, None
            continue

        v = agg[name]
        if name == "_match":
            for i, b in enumerate(v.get("buckets", EMPTY_LIST)):
                yield i, b, child, b
        elif name.startswith("_match"):
            i = int(name[6:])
            yield i, v, child, v
        elif name.startswith("_missing"):
            if len(name) == 8:
                i = None
            else:
                i = int(name[8:])
            yield None, v, child, v
        else:
            yield None, v, child, None

def aggs_iterator(aggs, es_query, decoders, give_me_zeros=False):
    """
    DIG INTO ES'S RECURSIVE aggs DATA-STRUCTURE:
    RETURN AN ITERATOR OVER THE EFFECTIVE ROWS OF THE RESULTS

    :param aggs: ES AGGREGATE OBJECT
    :param es_query: THE ABSTRACT ES QUERY WE WILL TRACK ALONGSIDE aggs
    :param decoders: TO CONVERT PARTS INTO COORDINATES
    """
    coord = [0] * len(decoders)
    parts = deque()
    stack = []

    gen = _children(aggs, es_query.children)
    while True:
        try:
            index, c_agg, c_query, part = next(gen)
        except StopIteration:
            try:
                gen = stack.pop()
            except IndexError:
                return
            parts.popleft()
            continue

        if c_agg.get('doc_count') == 0 and not give_me_zeros:
            continue
        parts.appendleft(part)
        for d in c_query.decoders:
            coord[d.edge.dim] = d.get_index(tuple(p for p in parts if p is not None), c_query, index)

        children = c_query.children
        selects = c_query.selects
        if selects or not children:
            parts.popleft()  # c_agg WAS ON TOP
            yield (
                tuple(p for p in parts if p is not None),
                tuple(coord),
                c_agg,
                selects
            )
            continue

        stack.append(gen)
        gen = _children(c_agg, children)

def count_dim(aggs, es_query, decoders):
    if not any(hasattr(d, "done_count") for d in decoders):
        return [d.edge for d in decoders]

    def _count_dim(parts, aggs, es_query):
        children = es_query.children
        if not children:
            return

        for child in children:
            name = child.name
            if not name:
                if aggs.get('doc_count') != 0:
                    _count_dim(parts, aggs, child)
                continue

            agg = aggs[name]
            if agg.get('doc_count') == 0:
                continue
            elif name == "_match":
                for i, b in enumerate(agg.get("buckets", EMPTY_LIST)):
                    if not b.get('doc_count'):
                        continue
                    b["_index"] = i
                    new_parts = (b,) + parts
                    for d in child.decoders:
                        d.count(new_parts)
                    _count_dim(new_parts, b, child)
            elif name.startswith("_missing"):
                new_parts = (agg,) + parts
                for d in child.decoders:
                    d.count(new_parts)
                _count_dim(new_parts, agg, child)
            else:
                _count_dim(parts, agg, child)

    _count_dim(tuple(), aggs, es_query)
    for d in decoders:
        done_count = getattr(d, "done_count", Null)
        done_count()
    return [d.edge for d in decoders]

export("jx_elasticsearch.es52.agg_format", aggs_iterator)
export("jx_elasticsearch.es52.agg_format", count_dim)


from django.urls import path
from rest_framework.routers import DefaultRouter

#My imports
from apps.cards.views import CardAPI,CardSearchAPIView,CommentAPIViewSet

router = DefaultRouter()
router.register(prefix="cards", viewset= CardAPI, basename="cards")

urlpatterns = [
    path('cards/search/', CardSearchAPIView.as_view(), name='search'),
    path('cards/comment/', CommentAPIViewSet.as_view(), name='comment'),

]
urlpatterns += router.urls


from django.urls import path
from accounts import views

urlpatterns = [

    path('login/', views.login, name='login'),
    path('signup/', views.signup, name='signup'),
    path('logout/', views.logout, name='logout'),
    path('mobile-login/', views.mobile_login, name='mobile_login'),
    path('verify-codelogin/', views.verify_codelogin, name='verify_codelogin'),
    path('user-dashboard/', views.user_dashboard, name='user_dashboard'),
    path('my-orders/', views.my_orders, name='my_orders'),
    path('edit-profile/', views.edit_profile, name='edit_profile'),
    path('change-password/', views.change_password, name='change_password'),
    path('order-detail/<int:order_id>/', views.order_detail, name='order_detail'),
    path('myAddress/', views.myAddress, name='myAddress'),
    path('addAddress/', views.addAddress, name='addAddress'),
    path('deleteAddress/<int:id>/', views.deleteAddress, name='deleteAddress'),
    path('editAddress/<int:id>/', views.editAddress, name='editAddress'),
    path('deleteCheckoutAddress/<int:id>/', views.deleteCheckoutAddress, name='deleteCheckoutAddress'),
    path('AddCheckoutAddress/', views.AddCheckoutAddress, name='AddCheckoutAddress'),

]


import os
import requests
from bs4 import BeautifulSoup

class BaseScraper:
  def __init__(self, market_name, urls):
    self.market_name = market_name
    self.urls = urls
    self.raw_pages_content = []
    self.items_data = []
    self.content = ""

  def start_requests(self):
    raw_pages_content = []

    for url in self.urls:
      try:
        response = requests.get(url)
        website_text = response.text
        raw_pages_content.append(website_text)
      except requests.exceptions:
        print(requests.exceptions)
        continue

    self.raw_pages_content = raw_pages_content

  def generate_text_list(self):
    list_header = f"\n-------------------------- {self.market_name} 🏪 --------------------------"

    items_str = "\n".join([ f"🔵 {item['name']} - €{item['price']}" for item in self.items_data ])

    content_str = f"{list_header}\n{items_str}"

    self.content = content_str

  def parse(self, parse_data):
    items_data = []

    for html_content in self.raw_pages_content:
      soup = BeautifulSoup(html_content, 'html.parser')

      parsed_data = parse_data(soup=soup)
      items_data.append(parsed_data)

    self.items_data = items_data

  def store_data(self):
    try:
      file_path = "email.txt"
      new_content = f"{self.content}\n"

      # Check if file exists
      if os.path.isfile(file_path):
        with open(file_path, "a") as file:
          file.write(new_content)
      else:
        with open(file_path, "w") as file:
          file.write(new_content)
    except:
      pass


import sys
import pygame

def title_page(screen, settings):
	#music and music loop
	menu_song = pygame.mixer.music.load('sounds/Epoq-Lepidoptera.ogg')
	pygame.mixer.music.play(-1)
	#front images and position
	frontpage = pygame.image.load("assets/main.jpg").convert()
	frontpage = pygame.transform.scale(frontpage,
        (settings.screen_width,settings.screen_height), screen)

	screen.blit(frontpage,(0,0))
	pygame.display.update()

	while True:
		#poll() retrieves only a single event
		eventpoll = pygame.event.poll()
		if eventpoll.type == pygame.KEYDOWN:
			if eventpoll.key == pygame.K_RETURN:
				break
			elif eventpoll.key == pygame.K_ESCAPE:
				sys.exit()
		else:
			text_button(screen, "Press [ENTER] To Begin", 32, settings.screen_width/2,
						settings.screen_height/2-25)
			text_button(screen, "Press [ESC] To Quit", 32, settings.screen_width/2,
						settings.screen_height/2)
			pygame.display.update()

    #make music stop
	readypage = pygame.image.load("assets/readybar.jpg").convert()
	readypage = pygame.transform.scale(readypage,
        (settings.screen_width,settings.screen_height), screen)
	screen.blit(readypage,(0,0))
	pygame.display.update()
	pygame.mixer.music.stop()

def text_button(surf, text, size, x, y):
    ## selecting a cross platform font to display the score
    font = pygame.font.Font(pygame.font.match_font('font/PTN55F.ttf'), size)
    text_surface = font.render(text, True, (189,179,137))   ## True denotes the font to be anti-aliased
    text_rect = text_surface.get_rect()
    text_rect.midtop = (x, y)
    surf.blit(text_surface, text_rect)


import os
import argparse
import sys
from easydev.console import red, purple, darkgreen
from easydev import DevTools
from dreamtools import Challenge
import dreamtools
import textwrap

def get_challenge(challenge_name):
    """Test validity of the challenge and returns an instance

    :param challenge_name: a valid name e.g. D5C3, D9dot5C1
    :return: instance of the challenge
    """
    try:
        c = Challenge(challenge_name)
    except:
        txt = "The challenge %s could not be found. " % challenge_name
        txt += "See --help for valid names"
        print_color(txt, red)
        sys.exit()
    return c

# Define the simple scoring functions here below
def generic_scoring(challenge_name, filename, subname=None, goldstandard=None):
    c = get_challenge(challenge_name)
    class_inst = c.import_scoring_class()
    try:
        score = class_inst.score(filename, subname=subname,
                                 goldstandard=goldstandard)
    except:
        try:
            score = class_inst.score(filename, subname=subname)
        except:
            try:
                score = class_inst.score(filename)
            except Exception as err:
                msg = "Error was caught in dreamtools.\n"
                msg += "Most probably an incorrect file format.\n"
                msg += "Here is the full error message to report it if needed\n"
                print_color(msg, red)
                raise err
    return {'Results': score}

# Figure out if the challenge exists and what are the sub challenges
def get_subchallenges(challenge_name):
    c = get_challenge(challenge_name)
    class_inst = c.import_scoring_class()
    return class_inst.sub_challenges

# ------------------------------------------------ The User Interface
def print_color(txt, func_color, underline=False):
    import easydev
    try:
        if underline:
            print(easydev.underline(func_color(txt)))
        else:
            print(func_color(txt))
    except:
        print(txt)

def scoring(args=None):
    """This function is used by the standalone application called dreamscoring

    ::

        dreamscoring --help

    """
    d = DevTools()

    if args is None:
        args = sys.argv[:]
    user_options = Options(prog="dreamtools")

    if len(args) == 1:
        user_options.parse_args(["prog", "--help"])
    else:
        options = user_options.parse_args(args[1:])

    if options.version is True:
        print("%s" % dreamtools.version)
        sys.exit()

    # Check on the challenge name
    if options.challenge is None:
        print_color('--challenge must be provided', red)
        sys.exit()
    else:
        options.challenge = options.challenge.upper()
        options.challenge = options.challenge.replace('DOT', 'dot')

        from dreamtools.admin.download_data import get_challenge_list
        if options.challenge not in get_challenge_list():
            print_color("This challenge %s is not registered in dreamtools." %
                    options.challenge, red)
            print("Here is the list of registered challenges: " +
                ", ".join(get_challenge_list()))
            sys.exit()

    # Check that the challenge can be loaded
    class_inst = get_challenge(options.challenge)
    try:
        this = class_inst.import_scoring_class()
    except NotImplementedError as err:
        print("\n"+str(err))
        sys.exit()
    else:
        # User may just request some information about the challenge.
        if options.info is True:
            print(this)
            sys.exit()
        elif options.onweb is True:
            this.onweb()
            sys.exit()

    # Checks name of the sub-challenges
    subchallenges = get_subchallenges(options.challenge)

    if len(subchallenges) and options.sub_challenge is None:
        txt = "This challenge requires a sub challenge name. "
        txt += "Please use --sub-challenge followed by one value in %s " % subchallenges
        print_color(txt, red)
        sys.exit(0)

    if options.sub_challenge is not None and len(subchallenges) != 0:
        try:
            d.check_param_in_list(options.sub_challenge, subchallenges)
        except ValueError as err:
            txt = "DREAMTools error: unknown sub challenge or not implemented"
            txt += "--->" + str(err)
            print_color(txt, red)
            sys.exit()

    # maybe users just need a template
    if options.download_template is True:
        c = Challenge(options.challenge)
        class_inst = c.import_scoring_class()
        if options.sub_challenge is None:
            print(class_inst.download_template())
        else:
            print(class_inst.download_template(options.sub_challenge))
        return

    # similary for the GS
    if options.download_goldstandard is True:
        c = Challenge(options.challenge)
        class_inst = c.import_scoring_class()
        if options.sub_challenge is None:
            print(class_inst.download_goldstandard())
        else:
            print(class_inst.download_goldstandard(options.sub_challenge))
        return

    # finally, we need a submission
    if options.filename is None:
        txt = "---> filename not provided. You must provide a filename with correct format\n"
        txt += "You may get a template using --download-template \n"
        txt += "Alternatively, you can user either --info or --onweb option to get information about the challenge.\n"
        txt += "https://github.com/dreamtools/dreamtools, or http://dreamchallenges.org\n"
        print_color(txt, red)
        sys.exit()

    # filename
    # filename in general is a single string but could be a list of filenames
    # Because on the parser, we must convert the string into a single string
    # if the list haa a length of 1
    for filename in options.filename:
        if os.path.exists(filename) is False:
            raise IOError("file %s does not seem to exists" % filename)
    if len(options.filename) == 1:
        options.filename = options.filename[0]

    print_color("DREAMTools scoring", purple, underline=True)
    print('Challenge %s (sub challenge %s)\n\n' % (options.challenge,
        options.sub_challenge))

    res = generic_scoring(options.challenge,
            options.filename,
            subname=options.sub_challenge,
            goldstandard=options.goldstandard)

    txt = "Solution for %s in challenge %s" % (options.filename,
            options.challenge)

    if options.sub_challenge is not None:
        txt += " (sub-challenge %s)" % options.sub_challenge
    txt += " is :\n"

    for k in sorted(res.keys()):
        txt += darkgreen("     %s:\n %s\n" %(k, res[k]))
    print(txt)

class Options(argparse.ArgumentParser):
    description = "tests"
    def __init__(self, prog=None):

        usage = """usage: python %s --challenge D8C1 --sub-challenge SC1A --submission <filename>\n""" % prog
        usage += """      python %s --challenge D5C2 --submission <filename>""" % prog
        epilog="""Author(s): Thomas Cokelaer (DREAMTools framework) and authors from
the DREAM consortium. Please see the scoring files headers for details
and the GitHub repository.

Source code on: https://github.com/dreamtools/dreamtools
Issues or bug report ? Please fill an issue on http://github.com/dreamtools/dreamtools/issues """
        description = """General Description:
    You must provide the challenge alias (e.g., D8C1 for DREAM8, Challenge 1)
    and if there were several sub-challenges, you also must provide the
    sub-challenge alias (e.g., sc1). Finally, the submission has to be
    provided. The format must be in agreement with the description of the
    challenge itself.

    Help and documentation about the templates may be found either within
    the online documentation http://pythonhosted.org/dreamtools/ or within
    the source code hosted on github http://github.org/dreamtools/dreamtools.

    Registered challenge so far (and sub-challenges) are:
"""
        # FIXME : not robust but will work for now
        registered = sorted([x for x in dir(dreamtools) if x.startswith('D')
                 and 'C' in x])

        for this in textwrap.wrap(", ".join(registered), 80):
            description += this + "\n"
        super(Options, self).__init__(usage=usage, prog=prog,
                epilog=epilog, description=description,
                formatter_class=argparse.RawDescriptionHelpFormatter)
        self.add_input_options()

    def add_input_options(self):
        """The input oiptions.

        Default is None. Keep it that way because otherwise, the contents of
        the ini file is overwritten in :class:`apps.Apps`.
        """
        group = self.add_argument_group("General",
                "General options (compulsary or not)")

        group.add_argument("--version", dest='version',
                         action="store_true",
                         help="verbose option.")

        group.add_argument("--challenge", dest='challenge',
                         default=None, type=str,
                         help="alias of the challenge (e.g., D8C1 stands for"
                         "dream8 challenge 1).")
        group.add_argument("--sub-challenge", dest='sub_challenge',
                         default=None, type=str,
                         help="Name of the data files")
        group.add_argument("--verbose", dest='verbose',
                         action="store_true",
                         help="verbose option.")
        group.add_argument("--submission", dest='filename', nargs='*',
                         help="submission/filename to score.")
        group.add_argument("--filename", dest='filename', nargs='*',
                         help="submission/filename to score.")
        group.add_argument("--gold-standard", dest='goldstandard',
                         help="""a gold standard filename. This may be
                         required in some challenges e.g. D2C3""")
        group.add_argument("--onweb", dest='onweb',action='store_true',
                help="Open synapse project page in a browser")
        group.add_argument("--info", dest='info', action="store_true",
                help="Prints general information about the challenge")
        group.add_argument("--download-template",
                dest='download_template',
                help="""Download template. Templates for challenge may be
                downloaded using this option. It returns the path to
                template.""",
                action='store_true')
        group.add_argument("--download-gold-standard",
                dest='download_goldstandard',
                help="""Download a gold standard, which can be used as a
                submissions as well. It returns the location of the file.""",
                action='store_true')

if __name__ == "__main__":
    scoring(sys.argv)