from django.db import models
from django.urls import reverse
from django.utils import timezone
from django.contrib.auth.models import User
from PIL import Image
from pathlib import Path
import os.path
from .myutilits import Photoes
from parler.models import TranslatableModel, TranslatedFields
from django.contrib.sites.models import Site

class Pictures(TranslatableModel):
    translations = TranslatedFields(
    alt=models.CharField(max_length=200, verbose_name='Название раздела', help_text='название основного раздела'),
    )

    image = models.ImageField(blank=True)
    width = models.IntegerField(blank=True, verbose_name='ширина', help_text='название раздела', default=600)
    height = models.IntegerField(blank=True, verbose_name='высота', help_text='название раздела', default=600)

    class Meta:
        verbose_name = "Картинки страницы"
        verbose_name_plural = "Картинки страницы"

    def save(self, force_insert=False, force_update=False):
        img = Image.open(self.image)
        if (self.width,self.height)==img.size:
            img.close
            super(Pictures, self).save(force_insert, force_update)

        else:
            if (self.width,self.height)==(0,0) or (self.width,self.height)==(None, None):
                (self.width, self.height) = img.size
            upload_path='material-kit-master/products/'
            if str(self.image).find(upload_path)==0:
                os.remove(self.image.path)
                self.image = str(self.image)[str(self.image).rfind('/') + 1:]
            baseurl = str(os.path.join(Path(__file__).resolve().parent.parent, ''))
            stroka0 = str(self.image)[:str(self.image).rfind('/') + 1] + upload_path + str(self.width) + 'x' + str(self.height) + '/'
            stroka1 = str(self.image)[str(self.image).rfind('/') + 1:]
            stroka2 = stroka1[:stroka1.find('.')]
            folder = baseurl + str(self.image)[:str(self.image).rfind('/') + 1] + upload_path + str(self.width) + 'x' + str(self.height) + '/'
            output_image_path = baseurl + stroka0 + stroka2 + '.png'
            Photoes(self.width, self.height, img, output_image_path,folder)

            self.image=upload_path + str(self.width) + 'x' + str(self.height) + '/'+ stroka2 + '.png'
            # print(dir(self.image))
            super(Pictures, self).save(force_insert, force_update)

    def __str__(self):
        return self.alt

class Post(models.Model):
    STATUS_CHOICES = (
        ('draft', 'Draft'),
        ('published', 'Published'),
    )
    title = models.CharField(max_length=250)
    slug = models.SlugField(max_length=250, unique_for_date='publish')
    author = models.ForeignKey(User, related_name='blog_posts', on_delete=models.CASCADE)
    body = models.TextField()
    publish = models.DateTimeField(default=timezone.now)
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='draft')

    def get_absolute_url(self):
        return reverse('standart:post_detail',
                       args=[self.publish.year,
                             self.publish.strftime('%m'),
                             self.publish.strftime('%d'),
                             self.slug])

    class Meta:
        ordering = ('-publish',)

    def __str__(self):
        return self.title

class Category(TranslatableModel):
    translations = TranslatedFields(
        name=models.CharField(max_length=200, db_index=True),
        slug=models.SlugField(max_length=200, db_index=True, unique=True)
    )

    class Meta:
        verbose_name = 'category'
        verbose_name_plural = 'categories'

class Templatecategory(TranslatableModel):
    STATUS_CHOICES = (
        ('odinary', 'Odinary'),
        ('form', 'Form'),
        ('call', 'Call'),
        ('service', 'Service'),
    )
    translations = TranslatedFields(
        name=models.CharField(max_length=200, db_index=True),
        slug=models.SlugField(max_length=200, db_index=True, unique=True)
    )
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='odinary')
    class Meta:
        verbose_name = 'Название категории'
        verbose_name_plural = 'Категории меню'
    def __str__(self):
        return self.slug

class Navconstruct(TranslatableModel):
    MENU_CHOICES = (
        ('first', 'First'),
        ('second', 'Second'),

    )
    STATUS_CHOICES = (
        ('odinary', 'Odinary'),
        ('form', 'Form'),
        ('call', 'Call'),
        ('service', 'Service'),
        ('example', 'Example'),
        ('login', 'Login'),
    )
    translations = TranslatedFields(
    name = models.CharField(max_length=200, verbose_name='Название раздела', help_text='название основного раздела'),
    slug = models.SlugField(max_length=200,null=True, unique=True, blank=True, verbose_name='URL адрес', help_text='название url адреса'),
    hreflogo = models.TextField(max_length=100, blank=True, verbose_name='Текст блока', help_text='текст блока'),
    alt = models.TextField(max_length=300, blank=True, verbose_name='Текст блока', help_text='текст блока'),
    title = models.CharField(max_length=300, blank=True, verbose_name='Заголовок', help_text='описание заголовка в строке браузера'),
    descrtionmeta = models.TextField(max_length=300, blank=True, verbose_name='Описание страницы', help_text='описание страницы в поискаовика'),
    keywordsmeta = models.TextField(max_length=300, blank=True, verbose_name='Ключевые слова', help_text='ключевые слова для поисковика'),

    )
    order= models.IntegerField(null=True, blank=True, verbose_name='Порядок в меню', help_text='последователность отображения в меню')
    template_name = models.ForeignKey('Templates', on_delete=models.CASCADE, related_name="navtemp", null=True, blank=True,
                                verbose_name='Templates', help_text='привязка к Templates')
    example = models.BooleanField(verbose_name='Наличие примера', help_text='наличие примеров в разделе',default=False)
    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE, null=True, blank=True, related_name="navpict", )
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='odinary')
    bar = models.CharField(max_length=10, choices=MENU_CHOICES, default='first')
    newslug = models.ForeignKey(Templatecategory,on_delete=models.CASCADE, null=True, blank=True, related_name="newslug", verbose_name='Первое меню',
                            )
    newsslug = models.ForeignKey(Templatecategory,on_delete=models.CASCADE, null=True, blank=True, related_name="newsslug", verbose_name='Второе меню',
                            )
    site = models.ManyToManyField(Site, blank=True, related_name="site", )
    class Meta:
        verbose_name = "Раздел"
        verbose_name_plural = "Основное меню"
        ordering = [
            "order",

        ]

    def get_absolute_url(self):
        if self.bar=='first':
            return reverse('standart:navigator',
                               args=[self.newslug])
        elif self.bar=='second':
            return reverse('standart:submain',
                           args=[self.newslug, self.newsslug])
    def __str__(self):
        return self.name

class Subnavigator(TranslatableModel):
    STATUS_CHOICES = (
        ('odinary', 'Odinary'),
        ('form', 'Form'),
        ('call', 'Call'),
        ('service', 'Service'),
        ('example', 'Example'),
        ('login', 'Login'),
    )
    translations = TranslatedFields(
    name = models.CharField(max_length=200, verbose_name='Название раздела', help_text='название основного раздела'),
    slug = models.SlugField(max_length=200,null=True, unique=True, blank=True, verbose_name='URL адрес', help_text='название url адреса'),
    hreflogo = models.CharField(max_length=100, blank=True, verbose_name='URL картинка', help_text='URL картинка'),
    alt = models.CharField(max_length=150, blank=True, verbose_name='Alt картинка', help_text='описание картинки'),
    title = models.CharField(max_length=300, blank=True, verbose_name='Заголовок', help_text='описание заголовка в строке браузера'),
    descrtionmeta = models.TextField(max_length=300, blank=True, verbose_name='Описание страницы', help_text='описание страницы в поискаовика'),
    keywordsmeta = models.TextField(max_length=300, blank=True, verbose_name='Ключевые слова', help_text='ключевые слова для поисковика'),

    )
    subname = models.ForeignKey('Navconstruct', on_delete=models.CASCADE, related_name="sub", null=True, blank=True, verbose_name='Основное меню', help_text='привязка к основному меню')
    template_name = models.ForeignKey('Templates', on_delete=models.CASCADE, related_name="subtemp", null=True, blank=True,
                                verbose_name='Templates', help_text='привязка к Templates')
    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE, null=True, blank=True, related_name="subpict", )
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='odinary')
    class Meta:
        verbose_name = "Подраздел"
        verbose_name_plural = "Подменю"
    def get_absolute_url(self):
        return reverse('standart:submain',
            args=[self.subname.slug, self.slug])
    def __str__(self):
        return self.name

class Templates(models.Model):

    class Meta:
        verbose_name = "Template"
        verbose_name_plural = "Template"

    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название темплейта')
    description = models.TextField(blank=True, null=True, verbose_name='Текст', help_text='описание')
    def __str__(self):
        return self.title

class Metka(TranslatableModel):
    class Meta:
        verbose_name = "Категория проекта"
        verbose_name_plural = "Категории проектов"
        ordering = [
            "translations__title",

        ]

    translations = TranslatedFields(
    title = models.CharField(max_length=100, blank=True, verbose_name='Категория проекта', help_text='категория проекта'),
    )
    def __str__(self):
        return self.title

class Examples(TranslatableModel):
    class Meta:
        verbose_name = "Пример"
        verbose_name_plural = "Примеры"

    name = models.ForeignKey('Navconstruct', on_delete=models.CASCADE, related_name="nexamples", null=True, blank=True,
                                verbose_name='Основное меню', help_text='привязка к основному меню')
    marks = models.ManyToManyField('Metka', blank=True,related_name="marks",)

    translations = TranslatedFields(
    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название раздела'),
    slug=models.SlugField(max_length=200, null=True, unique=True, blank=True, verbose_name='URL адрес', help_text='название url адреса'),
    )
    def __str__(self):
        return self.title
    def get_absolute_url(self):
        return reverse('standart:submain',
            args=[self.name.slug, self.slug])

class Container(models.Model):
    class Meta:
        verbose_name = "Тип блока"
        verbose_name_plural = "Тип блока"
        ordering = [
            "order",

        ]

    name = models.ForeignKey('Navconstruct', on_delete=models.CASCADE, related_name="ncont", null=True, blank=True,
                                verbose_name='Основное меню', help_text='привязка к основному меню')
    examples = models.ForeignKey('Examples', on_delete=models.CASCADE, related_name="examples", null=True, blank=True,
                                verbose_name='Пример', help_text='привязка к примеру')

    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название раздела')
    template_name = models.ForeignKey('Templates', on_delete=models.CASCADE, related_name="template", null=True, blank=True,
                                verbose_name='Templates', help_text='привязка к Templates')
    order = models.IntegerField(null=True, blank=True, verbose_name='Порядок на странице',
                                help_text='последователность отображения на странице')
    def __str__(self):
        return self.title

class Text(TranslatableModel):
    class Meta:
        verbose_name = "Информация страницы"
        verbose_name_plural = "Информация страницы"
        ordering = [
            "order",
        ]

    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE,null=True,blank=True,related_name="picture", )
    container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name="container", null=True,
                                      blank=True,
                                      verbose_name='Templates', help_text='привязка к Templates')
    order = models.IntegerField(null=True, blank=True, verbose_name='Порядок на странице',
                                help_text='последователность отображения на странице')
    translations = TranslatedFields(
    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название раздела'),
    text = models.TextField(blank=True, verbose_name='Текст', help_text='описание'))
    examples = models.ForeignKey('Examples', on_delete=models.CASCADE, related_name="example", null=True, blank=True,
                                 verbose_name='Пример', help_text='привязка к примеру')
    def __str__(self):
        return self.title
    def get_absolute_url(self):
        return reverse('standart:example',
                       args=[self.title])

class Table(TranslatableModel):
    class Meta:
        verbose_name = "Информация таблицы"
        verbose_name_plural = "Информация таблицы"
        ordering = [
            "order",
        ]

    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE,null=True,blank=True,related_name="table", )
    picturesvs = models.ForeignKey(Pictures, on_delete=models.CASCADE, null=True, blank=True, related_name="tablevs", )
    order = models.IntegerField(null=True, blank=True, verbose_name='Порядок на странице',
                                help_text='последователность отображения на странице')
    container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name="contable", null=True,
                                      blank=True,
                                      verbose_name='Templates', help_text='привязка к Templates')
    translations = TranslatedFields(
    description = models.TextField(blank=True, verbose_name='Текст', help_text='описание'),
    one = models.TextField(blank=True, verbose_name='Текст', help_text='описание'),
    two=  models.TextField(blank=True, verbose_name='Текст', help_text='описание'))
    def __str__(self):
        return self.description


import os
from pathlib import Path
import cv2
import chainer
import numpy as np
from tools.detector_hyperface import HyperFaceModel, apply_hyperface
import tools.detector_hyperface as detector_hyperface

def create_lists(img_path, train_file, test_file, train_output='lists/train_list.txt', test_output='lists/test_list.txt'):
    f1 = open(train_file,'r')
    f2 = open(test_file,'r')

    train_list = f1.readlines()
    test_list = f2.readlines()

    f3 = open(train_output, 'w')
    f4 = open(test_output, 'w')

    clip_length = 16

    for line in train_list:
        name = line.split(' ')[0]
        image_path = img_path+name
        label = line.split(' ')[-1]
        images = os.listdir(image_path)
        nb = len(images) // clip_length
        for i in range(nb):
            f3.write(name+' '+ str(i*clip_length+1)+' '+label)

    for line in test_list:
        name = line.split(' ')[0]
        image_path = img_path+name
        label = line.split(' ')[-1]
        images = os.listdir(image_path)
        nb = len(images) // clip_length
        for i in range(nb):
            f4.write(name+' '+ str(i*clip_length+1)+' '+label)

    f1.close()
    f2.close()
    f3.close()
    f4.close()

def create_ucf101_list():

    img_path = '/home/tianz/datsets/ucfimgs/'
    f1 = 'ucfTrainTestlist/train_file.txt'
    f2 = 'ucfTrainTestlist/test_file.txt'
    create_lists(img_path, f1, f2)

def create_lr2_vid_list():

    img_path = ''
    f1 = 'lrs2TrainTestlist/train_file.txt'
    f2 = 'lrs2TrainTestlist/test_file.txt'
    create_lists(img_path, f1, f2)

def create_afew():

    img_path = ''
    f1 = 'afew-va/train_file.txt'
    f2 = 'afew-va/test_file.txt'
    create_lists(img_path, f1, f2)

def make_label(dirName, label = '#', train_output = 'lists/train_list.txt', test_output = 'lists/test_list.txt', maximage = 9999):

    # dirName = '/home/varun/Downloads'

    print("****************")

    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    for (dirpath, dirnames, filenames) in os.walk(dirName):
        if len(filenames) == 0:# or '_45D' in dirpath or '_00D' in dirpath:
            continue
        # listOfFiles += [dirpath]
        # listOfFiles += [os.path.join(dirpath, file) for file in filenames]

        # filenames.sort(key=str.lower)

        nb = len(filenames) // clip_length
        nb = min(nb,maximage)
        for i in range(nb):
            if count % 4:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + label]
            else:
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + label]

        count += 1

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    with open(test_output, 'w') as f:
        for item in testFiles:
            f.write("%s\n" % item)

def make_label_ibugs(dirName, label = '#', train_output = 'lists/train_list.txt', test_output = 'lists/test_list.txt', maximage=9999, max_seq=9999):

    # dirName = '/home/varun/Downloads'

    print("****************")

    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    n_seq = 0
    for (dirpath, dirnames, filenames) in os.walk(dirName):
        if len(filenames) == 0: # or '_45D' in dirpath or '_00D' in dirpath:
            continue
        if not '-' in label:
            if '_00D' in dirpath:
                l = label
            elif '_45D' in dirpath:
                l = str(int(label) + 1)
            elif '_90D' in dirpath:
                l = str(int(label) + 2)
            elif '_270D' in dirpath:
                l = str(int(label) + 3)
            elif '_315D' in dirpath:
                l = str(int(label) + 4)
        else:
            l = str(abs(int(label)))
        # listOfFiles += [dirpath]
        # listOfFiles += [os.path.join(dirpath, file) for file in filenames]

        # filenames.sort(key=str.lower)
        filenames = [k for k in filenames if '.wav' not in k and 'flow' not in k]
        n_seq += 1

        nb = len(filenames) // clip_length
        nb = min(nb,maximage)
        for i in range(nb):
            if count % 4:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]
            else:
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]

            count += 1
            # if count > max_seq:
            #     break

        # if count > max_seq:
        #     break

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    with open(test_output, 'w') as f:
        for item in testFiles:
            f.write("%s\n" % item)

    return  n_seq, count

def make_label_amicorpus(dirName, label = '#', train_output = 'lists/train_list.txt', test_output = 'lists/test_list.txt', maximage=9999, max_seq=9999):
    print("****************")

    # Load hyperface
    hyperface_module = HyperFaceModel()
    hyperface_module.train = False
    hyperface_module.report = False
    hyperface_module.backward = False
    chainer.serializers.load_npz("/local/users/jfmadrig/ownCloud/LinTo/src/tools/weights/model_epoch_190", hyperface_module)
    # Setup GPU
    config_gpu = 0
    if config_gpu >= 0:
        chainer.cuda.check_cuda_available()
        chainer.cuda.get_device(config_gpu).use()
        hyperface_module.to_gpu()
        xp = chainer.cuda.cupy
    else:
        xp = np

    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    n_seq = 0
    IMG_SIZE = (227, 227)

    for (dirpath, dirnames, filenames) in os.walk(dirName):
        if len(filenames) == 0: # or '_45D' in dirpath or '_00D' in dirpath:
            continue
        if not '-' in label:
            if '_00D' in dirpath:
                l = label
            elif '_45D' in dirpath:
                l = str(int(label) + 1)
            elif '_90D' in dirpath:
                l = str(int(label) + 2)
            elif '_270D' in dirpath:
                l = str(int(label) + 3)
            elif '_315D' in dirpath:
                l = str(int(label) + 4)
        else:
            l = str(abs(int(label)))
        # listOfFiles += [dirpath]
        # listOfFiles += [os.path.join(dirpath, file) for file in filenames]

        # filenames.sort(key=str.lower)
        filenames = [k for k in filenames if '.wav' not in k and 'flow' not in k]
        n_seq += 1

        filenames = filenames.sort()

        nb = len(filenames) // clip_length
        nb = min(nb,maximage)
        for i in range(nb):

            imgs = []
            for j in range(clip_length):
                frame = cv2.imread(dirpath + '/' + filenames[clip_length*i + j])
                img = frame.astype(np.float32) / 255.0  # [0:1]
                img = cv2.resize(img, IMG_SIZE)
                img = cv2.normalize(img, None, -0.5, 0.5, cv2.NORM_MINMAX)
                img = np.transpose(img, (2, 0, 1))
                imgs.append(img)

            for j in range(clip_length):
                frame = cv2.imread(dirpath + '/' + filenames[clip_length*i + j])
                poses = apply_hyperface(frame, None, None, hyperface_module, bh, xp)

            if 'CV5' in dirpath:
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]
            else:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]
            # if count > max_seq:
            #     break

        # if count > max_seq:
        #     break

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    with open(test_output, 'w') as f:
        for item in testFiles:
            f.write("%s\n" % item)

    return n_seq, count

def make_label_ict3d(dir_path, gt_label_path='#', train_output='lists/train_list.txt', test_output='lists/test_list.txt',
                     maximage=9999, max_seq=9999):
    from numpy import genfromtxt
    from tools.utils import rotation_matrix, project_plane_yz, get_xpose
    import numpy as np
    import cv2

    def _draw_line(img, pt1, pt2, color, thickness=2):
        pt1 = (int(pt1[0]), int(pt1[1]))
        pt2 = (int(pt2[0]), int(pt2[1]))
        cv2.line(img, pt1, pt2, color, int(thickness))

    print("****************")
    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    n_seq = 0

    for (dirpath, dirnames, filenames) in os.walk(dir_path):
        if len(filenames) == 0:  # or '_45D' in dirpath or '_00D' in dirpath:
            continue
        seq = dirpath.split('/')[-1]
        gt_path = gt_label_path + seq + '/' + "polhemusNorm.csv"
        gt_data = genfromtxt(gt_path, delimiter=',')

        init_pose = np.loadtxt(gt_label_path + seq + '/' + "initpose.txt", delimiter=',')

        n_seq += 1

        nb = len(filenames) // clip_length
        nb = min(nb, maximage)
        for i in range(nb):

            img_path = dirpath + '/{:06d}'.format(i * clip_length) + '.jpg'
            if not os.path.isfile(img_path):
                continue
            # img = cv2.imread(img_path)

            pose = gt_data[i*clip_length][3:]
            rotpose = rotation_matrix(-pose[2], pose[0], -pose[1])
            # rotinit = rotation_matrix(-init_pose[2], init_pose[0], -init_pose[1])
            # rotmat = np.transpose(rotpose).dot(rotinit)

            rotmat = rotpose
            pred = get_xpose(rotmat)
            l = np.argmax(pred)
            # print(pred)

            zvec = np.array([0, 0, 1], np.float32)
            yvec = np.array([0, 1, 0], np.float32)
            xvec = np.array([1, 0, 0], np.float32)
            zvec = project_plane_yz(rotmat.dot(zvec))
            yvec = project_plane_yz(rotmat.dot(yvec))
            xvec = project_plane_yz(rotmat.dot(xvec))

            # # Lower left
            # size = 30
            # idx = 0
            # org_pt = ((size + 5) * (2 * idx + 1), img.shape[0] - size - 5)
            # _draw_line(img, org_pt, org_pt + zvec * size, (255, 0, 0), 3)
            # _draw_line(img, org_pt, org_pt + yvec * size, (0, 255, 0), 3)
            # _draw_line(img, org_pt, org_pt + xvec * size, (0, 0, 255), 3)
            # cv2.imshow('result', img)
            # cv2.waitKey(1)
            if '02' in dirpath and test_output is not '':
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + str(l)]
            else:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + str(l)]
            if count > max_seq:
                break

        # if count > max_seq:
        #     break

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    if test_output is not '':
        with open(test_output, 'w') as f:
            for item in testFiles:
                f.write("%s\n" % item)

    return n_seq, count

if __name__ == '__main__':
    # create_ucf101_list()
    # create_lr2_vid_list()
    # create_afew()
    # make_label('/data/jfmadrig/mvlrs_v1/pretrain_heads', '0',
    #            train_output = 'lrs2TrainTestlist/train_heads_list.txt',
    #            test_output = 'lrs2TrainTestlist/test_heads_list.txt')
    # make_label('/data/jfmadrig/VidTIMIT_heads', '1',
    #            train_output='lists/vid/train_heads_list.txt',
    #            test_output='lists/vid/test_heads_list.txt')
    # make_label('/data/jfmadrig/VidTIMIT_heads_slow', '1',
    #            train_output='lists/vid/train_heads_slow_short_list.txt',
    #            test_output='lists/vid/test_heads_slow_short_list.txt',
    #            maximage=100)
    # make_label('/data/jfmadrig/VidTIMIT_lips', '1',
    #            train_output='lists/vid/train_lips_list.txt',
    #            test_output='lists/vid/test_lips_list.txt')
    # make_label('/data/jfmadrig/mvlrs_v1/pretrain_lips', '0',
    #            train_output = 'lists/lrs2TrainTestlist/train_lips_list.txt',
    #            test_output = 'lists/lrs2TrainTestlist/test_lips_list.txt')
    # make_label('/data/jfmadrig/VidTIMIT_lips_slow', '1',
    #            train_output='lists/vid/train_lips_list_slow.txt',
    #            test_output='lists/vid/test_lips_list_slow.txt',
    #            maximage=200)
    #
    # make_label('/data/jfmadrig/hpedatasets/ict3DHP_lips', '1',
    #            train_output='lists/ict3DHP/train_lips_list.txt',
    #            test_output='lists/ict3DHP/test_lips_list.txt')

    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Silence_Digits_lips', '0',
    #            train_output='lists/ibug-avs/Silence/train_lips_list_angle.txt',
    #            test_output='lists/ibug-avs/Silence/test_lips_list_angle.txt')
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Voice_Digits_lips', '0',
    #            train_output='lists/ibug-avs/Voice/train_lips_list_angle.txt',
    #            test_output='lists/ibug-avs/Voice/test_lips_list_angle.txt')
    # silence_train_output = 'lists/ibug-avs/Silence/train_lips_list_angle_c6.txt'
    # silence_test_output = 'lists/ibug-avs/Silence/test_lips_list_angle_c6.txt'
    # speaker_train_output='lists/ibug-avs/Voice/train_lips_list_angle_c6.txt'
    # speaker_test_output='lists/ibug-avs/Voice/test_lips_list_angle_c6.txt'

    # silence_train_output = 'lists/ibug-avs/Silence/train_heads_list_angle_c5.txt'
    # silence_test_output = 'lists/ibug-avs/Silence/test_heads_list_angle_c5.txt'
    # speaker_train_output='lists/ibug-avs/Voice/train_heads_list_angle_c5.txt'
    # speaker_test_output='lists/ibug-avs/Voice/test_heads_list_angle_c5.txt'
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Silence_Digits_heads', '5',
    #            train_output=silence_train_output,
    #            test_output=silence_test_output)
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Voice_Digits_heads', '0',
    #            train_output=speaker_train_output,
    #            test_output=speaker_test_output)

    dataset = 'ibugs'
    dataset = 'amicorpus'
    # dataset = 'ict3d'

    if dataset == 'ibugs':
        silence_train_output = 'lists/ibug-avs/Silence_v2/train_heads_list_angle_c5.txt'
        silence_test_output = 'lists/ibug-avs/Silence_v2/test_heads_list_angle_c5.txt'
        silence_dir_path = '/data/jfmadrig/ibug-avs/Silence_Digits_v2_head/'
        # silence_init_label = '-1' # Binary
        silence_init_label = '5'
        speaker_train_output='lists/ibug-avs/Voice_v2/train_heads_list_angle_c5.txt'
        speaker_test_output='lists/ibug-avs/Voice_v2/test_heads_list_angle_c5.txt'
        speaker_dir_path = '/data/jfmadrig/ibug-avs/Voice_Digits_v2_head/'
        # speaker_init_label = '-0' # binary
        speaker_init_label = '0'

        train_outpu_file = 'lists/ibug-avs/train_heads_list_angle_c10_v2.txt'
        test_outpu_file = 'lists/ibug-avs/test_heads_list_angle_c10_v2.txt'

    elif dataset == 'amicorpus':
        # silence_train_output = 'lists/amicorpus/Silence/train_heads_list_angle_binary.txt'
        # silence_test_output  = 'lists/amicorpus/Silence/test_heads_list_angle_binary.txt'
        # silence_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_Silence/CV1/IS1002b/video/Closeup1/'
        # speaker_train_output = 'lists/amicorpus/Speaking/train_heads_list_angle_binary.txt'
        # speaker_test_output  = 'lists/amicorpus/Speaking/test_heads_list_angle_binary.txt'
        # speaker_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_Speaking/CV1/IS1002b/video/Closeup1/'
        # train_outpu_file = 'lists/amicorpus/train_heads_list_binary.txt'
        # test_outpu_file  = 'lists/amicorpus/test_heads_list_binary.txt'
        silence_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_wholeshort_v2_Silence/'
        speaker_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_wholeshort_v2_Speaking/'

        silence_train_output = 'lists/amicorpus/Silence/train_orientation_head_list_cv5.txt'
        silence_test_output  = 'lists/amicorpus/Silence/test_orientation_head_list_cv5.txt'
        speaker_train_output = 'lists/amicorpus/Speaking/train_orientation_head_list_cv5.txt'
        speaker_test_output  = 'lists/amicorpus/Speaking/test_orientation_head_list_cv5.txt'
        train_outpu_file = 'lists/amicorpus/train_orientation_head_list_cv5.txt'
        test_outpu_file  = 'lists/amicorpus/test_orientation_head_list_cv5.txt'

        # silence_init_label = '5'
        # speaker_init_label = '0'
        silence_init_label = '-1' # Binary
        speaker_init_label = '-0' # binary
    elif dataset == 'ict3d':
        imgs_path   = '/data/jfmadrig/hpedatasets/ict3DHP_rgb_of_heads/'
        gt_path = '/data/jfmadrig/hpedatasets/ict3DHP_data/'
        train_output = 'lists/ict3DHP/orientation/train_heads_list.txt'
        test_output = ''
        make_label_ict3d(imgs_path, gt_path, train_output, test_output)
        exit()

    max_seq = 5000
    maximage= 100
    n_seq, max_seq = \
    make_label_amicorpus(speaker_dir_path, speaker_init_label,
                     train_output=speaker_train_output,
                     test_output=speaker_test_output,
                     maximage=maximage,
                     max_seq=max_seq)

    make_label_amicorpus(silence_dir_path, silence_init_label,
                    train_output=silence_train_output,
                    test_output=silence_test_output,
                     maximage=maximage,
                    max_seq=max_seq )
    ## Binary
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Silence_Digits_v2_head/', '-1',
    #            train_output=silence_train_output,
    #            test_output=silence_test_output)
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Voice_Digits_v2_head/', '-0',
    #            train_output=speaker_train_output,
    #            test_output=speaker_test_output)

    filenames = [silence_train_output, speaker_train_output]
    with open(train_outpu_file, 'w') as outfile:
        for fname in filenames:
            with open(fname) as infile:
                for line in infile:
                    outfile.write(line)
    filenames = [silence_test_output, speaker_test_output]
    with open(test_outpu_file, 'w') as outfile:
        for fname in filenames:
            with open(fname) as infile:
                for line in infile:
                    outfile.write(line)

    # make_label_ibugs('/data/jfmadrig/ibug-avs/Static_Digits_heads', '0',
    #            train_output='lists/ibug-avs/Static/train_heads_list_angle.txt',
    #            test_output='lists/ibug-avs/Static/test_heads_list_angle.txt')


#!/usr/bin/env python

import datetime, time, os
from optparse import OptionParser
from multiprocessing.pool import ThreadPool

import zkclient
from zkclient import ZKClient, CountingWatcher, zookeeper

SESSIONS_NUM = 12
total_writes = 0

usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("", "--servers", dest="servers",
                  default="localhost:2181", help="comma separated list of host:port (default %default), test each in turn")
parser.add_option("", "--cluster", dest="cluster",
                  default=None, help="comma separated list of host:port, test as a cluster, alternative to --servers")
parser.add_option("", "--config",
                  dest="configfile", default=None,
                  help="zookeeper configuration file to lookup cluster from")
parser.add_option("", "--timeout", dest="timeout", type="int",
                  default=5000, help="session timeout in milliseconds (default %default)")
parser.add_option("", "--root_znode", dest="root_znode",
                  default="/zk-latencies", help="root for the test, will be created as part of test (default /zk-latencies)")
parser.add_option("", "--znode_size", dest="znode_size", type="int",
                  default=25, help="data size when creating/setting znodes (default %default)")
parser.add_option("", "--znode_count", dest="znode_count", default=10000, type="int",
                  help="the number of znodes to operate on in each performance section (default %default)")
parser.add_option("", "--watch_multiple", dest="watch_multiple", default=1, type="int",
                  help="number of watches to put on each znode (default %default)")

parser.add_option("", "--force",
                  action="store_true", dest="force", default=False,
                  help="force the test to run, even if root_znode exists - WARNING! don't run this on a real znode or you'll lose it!!!")

parser.add_option("", "--synchronous",
                  action="store_true", dest="synchronous", default=False,
                  help="by default asynchronous ZK api is used, this forces synchronous calls")

parser.add_option("-v", "--verbose",
                  action="store_true", dest="verbose", default=False,
                  help="verbose output, include more detail")
parser.add_option("-q", "--quiet",
                  action="store_true", dest="quiet", default=False,
                  help="quiet output, basically just success/failure")
parser.add_option("", "--type", dest="type", default=False,
                  help="quiet output, basically just success/failure")

(options, args) = parser.parse_args()

zkclient.options = options

zookeeper.set_log_stream(open("cli_log_%d.txt" % (os.getpid()),"w"))

class SmokeError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class ZkData:

    def __init__(self, session, data, startTime, id):
        self.session = session
        self.data = data
        self.startTime = startTime
        self.id = id

def child_path(i, id):
    return "%s/session_%d_%d" % (options.root_znode, i, id)

def create_asynchronous_latency_test(zd):
    # create znode_count znodes (perm)

    global SESSIONS_NUM
    callbacks = []
    for j in xrange(options.znode_count/SESSIONS_NUM):
        cb = zkclient.CreateCallback()
        cb.cv.acquire()
        zd.session.acreate(child_path(j, zd.id), cb, zd.data)
        callbacks.append(cb)

    count = 0
    for j, cb in enumerate(callbacks):
        cb.waitForSuccess()
        #print(child_path(j, zd.id))
        if cb.path != child_path(j, zd.id):
            raise SmokeError("invalid path %s for operation %d on handle %d" %
                            (cb.path, j, cb.handle))
        count += 1
        if time.time() - zd.startTime >= 10:
            break
    return count

def get_asynchronous_latency_test(zd):
    # create znode_count znodes (perm)
    global SESSIONS_NUM
    callbacks = []
    for j in xrange(options.znode_count/SESSIONS_NUM):
        cb = zkclient.GetCallback()
        cb.cv.acquire()
        zd.session.aget(child_path(j, zd.id), cb)
        callbacks.append(cb)

    count = 0
    for cb in callbacks:
        cb.waitForSuccess()
        if cb.value != data:
            raise SmokeError("invalid data %s for operation %d on handle %d" %
                             (cb.value, j, cb.handle))
        count += 1
        if time.time() - zd.startTime >= 10:
            break
    return count

def set_asynchronous_latency_test(zd):
    # create znode_count znodes (perm)
    global SESSIONS_NUM
    callbacks = []
    for j in xrange(options.znode_count/SESSIONS_NUM):
        cb = zkclient.SetCallback()
        cb.cv.acquire()
        zd.session.aset(child_path(j, zd.id), cb, zd.data)
        callbacks.append(cb)

    count = 0
    for cb in callbacks:
        cb.waitForSuccess()
        count += 1
        if time.time() - zd.startTime >= 10:
            break
    return count

def log_result(result):
    global total_writes
    #print("done")
    total_writes += result

def apply_async_with_callback(sessions, data):
    global total_writes
    pool = ThreadPool(processes=SESSIONS_NUM)
    startTime = time.time()
    print("Start time: ", str(startTime))
    for i, s in enumerate(sessions):
        zd = ZkData(s, data, startTime, i)
        if options.type == "create":
            pool.apply_async(create_asynchronous_latency_test, args=(zd,), callback=log_result)
        elif options.type == "set":
            pool.apply_async(set_asynchronous_latency_test, args=(zd,), callback=log_result)
        elif options.type == "get":
            pool.apply_async(get_asynchronous_latency_test, args=(zd,), callback=log_result)
    pool.close()
    pool.join()
    print("Duration: ", str(time.time()-startTime))
    print("Total writes: ", total_writes)

if __name__ == '__main__':
    data = options.znode_size * "x"
    servers = options.servers.split(",")
    # create all the sessions first to ensure that all servers are
    # at least available & quorum has been formed. otw this will
    # fail right away (before we start creating nodes)
    sessions = []
    # create sessions to each of the servers in the ensemble
    for server in servers:
        for i in range(SESSIONS_NUM):
            sessions.append(ZKClient(server, options.timeout))

    # ensure root_znode doesn't exist
    if sessions[0].exists(options.root_znode) and options.type == "create":
        # unless forced by user
        if not options.force:
            raise SmokeError("Node %s already exists!" % (options.root_znode))

        children = sessions[0].get_children(options.root_znode)
        for child in children:
            sessions[0].delete("%s/%s" % (options.root_znode, child))
    elif options.type == "create":
        sessions[0].create(options.root_znode,
                           "smoketest root, delete after test done, created %s" %
                           (datetime.datetime.now().ctime()))

    apply_async_with_callback(sessions, data)

    sessions[0].delete(options.root_znode)

    # close sessions
    for s in sessions:
        s.close()

    print("Latency test complete")


# -*- coding:utf-8 -*-
#导入包
from wechatsogou import *
from wechatsogou import *
import logging
import logging.config

# wechats = WechatSogouAPI()
#
# name = 'Python'
# wechat = wechats.search_gzh(name)
# print(wechat)
# wechatid = 'python_shequ'
#
# data = wechats.get_gzh_info(wechatid)
# print(data)
#日志
logging.config.fileConfig('logging.conf')
logger = logging.getLogger()
#实例
wechats = WechatSogouAPI()
exit()
#搜素一个微信公众号
name = '中国民航大学'
wechat_infos = wechats.search_gzh(name)

#对于数据库中的某一个微信号进行搜索
wechat_id = 'nanhangqinggong'
wechat_info = wechats.get_gzh_info(wechat_id)

#可以对用户开放搜索文章的功能
keywords = '傅立叶变换'
wechat_articles = wechats.search_article(keywords)
#上面我们得到过微信号的最近文章页的地址，现在，就去获取该页的数据
data = wechats.get_article_content(wechat_id)
data_all = wechats.get_gzh_info(wechat_id)

#对于上面获取的文章链接，需要出来，注意此方法获得的'yuan'字段市文章固定地址，应该存储这个
messages = wechats.get_gzh_info(wechat_id)
for m in messages:
    if m['type'] == '49':
        article_info = wechats.deal_article(m['content_url'])

#直接获取热门文章
article_single = wechats.get_gzh_article_by_hot(wechat_id)
articles_all = wechats.get_gzh_article_by_history(wechat_id)

sugg_keyword = wechats.get_sugg('中国梦')


import pandas as pd
import matplotlib.pyplot as plt
from sklearn import linear_model, preprocessing, tree, model_selection
from sklearn.ensemble import RandomForestRegressor
import numpy as np
from sklearn.model_selection import train_test_split

data = pd.read_csv("Titanic_dataset.csv")
fig = plt.figure(figsize=(18,6))

#1)--------------visual analytics---------------
plt.subplot2grid((2,3), (0,0))
data.Survived.value_counts(normalize=True).plot(kind="barh")
plt.title("Survived")
plt.ylabel("Survived")

plt.subplot2grid((2,3), (0,1))
plt.scatter(data.Survived, data.Age, alpha=0.1)
plt.title("Age_Survived")
plt.ylabel("Age")

plt.subplot2grid((2,3), (0,2))
data.Pclass.value_counts(normalize=True).plot(kind="barh")
plt.title("Class_Structure")
plt.ylabel("Class")

plt.subplot2grid((2,3), (1,0), colspan=2)
for x in [1,2,3]:
    data.Age[data.Pclass == x].plot(kind="kde")
plt.title("Class_Age")
plt.legend(("1st", "2nd", "3rd"))

plt.subplot2grid((2,3), (1,2))
data.Embarked.value_counts(normalize=True).plot(kind="barh")
plt.title("Embarked")
plt.ylabel("City")

plt.show()

#Broader_visualization
fig = plt.figure(figsize=(18, 6))
female_color = "#FA0000"

plt.subplot2grid((3,4), (0,0))
data.Survived.value_counts(normalize=True).plot(kind="bar", alpha=0.7)
plt.title("Survived")

plt.subplot2grid((3,4), (0,1))
data.Survived[data.Sex == "male"].value_counts(normalize=True).plot(kind="bar", alpha=0.7)
plt.title("Men_Survived")

plt.subplot2grid((3,4), (0,2))
data.Survived[data.Sex == "female"].value_counts(normalize=True).plot(kind="bar", alpha=0.7, color = female_color)
plt.title("Female_Survived")

plt.subplot2grid((3,4), (0,3))
data.Sex[data.Survived == 1].value_counts(normalize=True).plot(kind="bar", alpha=0.7, color = [female_color, 'b'])
plt.title("Sex_Survived")

plt.subplot2grid((3,4), (1,0), colspan=4)
for x in [1,2,3]:
    data.Survived[data.Pclass == x].plot(kind="kde")
plt.title("Class_Survived")
plt.legend(("1st", "2nd", "3rd"))

plt.subplot2grid((3,4), (2,0))
data.Survived[(data.Sex == "male") & (data.Pclass == 1)].value_counts(normalize=True).plot(kind="bar", alpha=0.7)
plt.title("1ste Class male_Survival")

plt.subplot2grid((3,4), (2,1))
data.Survived[(data.Sex == "male") & (data.Pclass == 3)].value_counts(normalize=True).plot(kind="bar", alpha=0.7)
plt.title("3rd Class male_Survival")

plt.subplot2grid((3,4), (2,2))
data.Survived[(data.Sex == "female") & (data.Pclass == 1)].value_counts(normalize=True).plot(kind="bar", alpha=0.7, color = female_color)
plt.title("1ste Class female_Survival")

plt.subplot2grid((3,4), (2,3))
data.Survived[(data.Sex == "female") & (data.Pclass == 3)].value_counts(normalize=True).plot(kind="bar", alpha=0.7, color = female_color)
plt.title("3rd Class female_Survival")

plt.show()

#Prediction Algoritme
def clean_data(data):
    data["Fare"] = data["Fare"].fillna(data["Fare"].dropna().median())
    data["Age"] = data["Age"].fillna(data["Age"].dropna().median())

    data.loc[data["Sex"] == "male", "Sex"] = 0
    data.loc[data["Sex"] == "female", "Sex"] = 1

    data["Embarked"] = data["Embarked"].fillna("S")
    data.loc[data["Embarked"] == "S", "Embarked"] = 0
    data.loc[data["Embarked"] == "C", "Embarked"] = 1
    data.loc[data["Embarked"] == "Q", "Embarked"] = 2

clean_data(data)

model_1 = linear_model.LinearRegression()
model_2 = RandomForestRegressor()

target = data["Survived"].values
features = data[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]].values

X_train , X_test , Y_train, Y_test = train_test_split(features , target, test_size=0.2)

print(X_train)

classifier_1 = model_1.fit(X_train, Y_train)
print(classifier_1.score(X_train, Y_train)*100, "% Accuracy")

classifier_2 = model_2.fit(X_train, Y_train)
print(classifier_2.score(X_train, Y_train)*100, "% Accuracy")

predict_1 = model_1.predict(X_test)
predict_2 = model_2.predict(X_test)

#Model_acurracy_Visualization
fig = plt.figure(figsize=(18, 6))

plt.subplot2grid((2,2), (0,0), colspan=2)
plt.scatter(predict_1, Y_test, alpha=0.35)
plt.ylabel("true result")
plt.xlabel("prediction")
plt.title("Prediction_Linear")

plt.subplot2grid((2,2), (1,0), colspan=2)
plt.scatter(predict_2, Y_test, alpha=0.35)
plt.ylabel("true result")
plt.xlabel("prediction")
plt.title("Prediction_RandomForestRegressor")

plt.show()


from django.http import HttpResponse, HttpResponseRedirect, JsonResponse
from django.shortcuts import render
from django.views import View
from django.views.generic import ListView, UpdateView, DeleteView
from general.models import Message
from django.urls import reverse_lazy

class IndexView(ListView):
    template_name = 'messages.html'
    model = Message

class CreateMessageView(View):
    def post(self, request):
        Message.objects.create(
            text=request.POST.get('text')
        )
        return JsonResponse({'status': 'done'})

class DeleteMessageView(DeleteView):
    model = Message
    pk_url_kwarg = 'pk'
    success_url = reverse_lazy('index')

class EditMessageView(UpdateView):
    template_name = 'edit.html'
    model = Message


from Crypto.Cipher import AES

def get_all(filename):
    with open(filename, "rb") as f:
        return f.read()

if __name__ == "__main__":
    bsecret = u"h3y_gUyZ"
    print
    "keystring:", bsecret
    secret = ""
    for e in bsecret:
        secret += "%x" % ord(e)
    print
    "encoded key:", secret
    print
    "raw:", get_all("player1.plr")
    print("")
    for x in range(1, 6):
        cipher = AES.new(secret, x, secret)
        decoded = cipher.decrypt(get_all("player1.plr"))
        print
        "Decrypted string " + str(x) + ":", decoded
        print("")
        with open(str(x) + "decode.txt", "wb") as f:
            f.write(decoded)


#! /usr/bin/env python

'''
Script for interactively changing the input parameters of
ForegroundRenderer and BackgroundRenderer.
'''

import os
import sys
import argparse
import numpy
from nose.tools import (assert_true, assert_in)
import matplotlib
from matplotlib import pylab
from matplotlib.widgets import Slider
from simplelearn.data.memmap_dataset import MemmapDataset
from simplelearn.data import data_path
from simplelearn.utils import safe_izip
from poselearn.data.compositing_dataset import (ForegroundRenderer,
                                                BackgroundRenderer)
from poselearn.scripts.browse_foreground_renderer import get_value_arrays

def parse_args():
    '''
    Parses command-line args.
    '''

    def existing_file_path(arg):
        assert_true(os.path.isfile(arg), "Couldn't find {}".format(arg))
        return arg

    default_partition = 'train'

    default_fg_path = os.path.join(
        data_path,
        'ssd',
        'foreground_datasets',
        '6_object_dataset',
        'npy',
        '6_object_dataset_split_0.9_{}.npy'.format(default_partition))

    default_fg_sizes_path = os.path.join(os.path.split(default_fg_path)[0],
                                         'object_sizes.txt')

    default_bg_path = os.path.join(
        data_path,
        'ssd',
        'background_datasets',
        '2015_08_08_desktop_photos-grid_2_3-scale_0.5-shuffle_yes_split_0.9_'
        '{}.npy'.format(default_partition))

    parser = argparse.ArgumentParser(
        description=("Interactive viewer for poselearn.data."
                     "compositing_dataset.ForegroundRenderer."))

    parser.add_argument('--fg',
                        type=existing_file_path,
                        default=default_fg_path)

    parser.add_argument('--fg-sizes',
                        type=existing_file_path,
                        help="File with list of object sizes.")

    parser.add_argument('--no-fg-sizes',
                        action='store_true',
                        help="Don't use forground scaling file")

    parser.add_argument('--bg',
                        default=default_bg_path,
                        type=existing_file_path,
                        help="The background MemmapDataset's .npy file")

    args = parser.parse_args()

    if args.fg == default_fg_path and not args.no_fg_sizes:
        args.fg_sizes = default_fg_sizes_path

    return args

def main():
    '''
    Entry point of script.
    '''
    args = parse_args()

    def get_scales(fg_sizes_path):
        sizes = numpy.loadtxt(fg_sizes_path)
        result = sizes / sizes.max()

        # add a 1.0 for the blank category
        result.resize(result.size + 1)
        result[-1] = 1.0
        return result

    fg_dataset = MemmapDataset(args.fg)
    if args.fg_sizes:
        fg_baseline_scales = get_scales(args.fg_sizes)
    else:
        fg_baseline_scales = None

    fg_renderer = ForegroundRenderer(fg_dataset)

    bg_dataset = MemmapDataset(args.bg)
    bg_renderer = BackgroundRenderer(bg_dataset,
                                     fg_dataset.tensors[0].shape[1:3])

    figure = pylab.figure(figsize=(12, 7))
    figure.canvas.set_window_title(os.path.split(args.fg)[1] +
                                   " and " +
                                   os.path.split(args.bg)[1])

    num_sliders = 9

    text_axes = pylab.subplot(3, 2, 1)
    text_axes.text(-0.5,  # x
                   0.5,  # y
                   "fg image: ",
                   verticalalignment='center',
                   transform=text_axes.transAxes)

    # Hide tick marks
    text_axes.get_xaxis().set_visible(False)
    text_axes.get_yaxis().set_visible(False)

    sliders_axes = [pylab.subplot(num_sliders + 6, 2, 11 + i * 2)
                    for i in range(num_sliders)]

    image_axes = pylab.subplot(1, 2, 2)

    fg_rc_shift = numpy.asarray([0.0, 0.0])
    fg_scale = numpy.asarray(1.0)
    fg_roll = numpy.asarray(0.0)

    bg_frame_number = numpy.asarray(0)
    bg_rc_shift = numpy.asarray([0.0, 0.0])
    bg_scale = numpy.asarray(1.0)
    bg_roll = numpy.asarray(0.0)

    value_arrays = get_value_arrays(fg_dataset.tensors[1])

    # The current indices into value_arrays
    indices = numpy.zeros(4, dtype=int)

    # Which index to edit
    dimension_to_edit = [0]

    max_indices = numpy.asarray([len(value_array) - 1
                                 for value_array
                                 in value_arrays])

    fg_rshift_slider = Slider(sliders_axes[0], 'fg row shift', -50, 50, 0)
    fg_cshift_slider = Slider(sliders_axes[1], 'fg col shift', -50, 50, 0)
    fg_scale_slider = Slider(sliders_axes[2], 'fg scale', 0.5, 1.5, 1.0)
    fg_roll_slider = Slider(sliders_axes[3], 'fg roll', -numpy.pi, numpy.pi, 0)

    bg_frame_slider = Slider(sliders_axes[4],
                             'bg frame',
                             0,
                             bg_dataset.num_examples() - 1,
                             0)
    bg_rshift_slider = Slider(sliders_axes[5], 'bg row shift', -50, 50, 0)
    bg_cshift_slider = Slider(sliders_axes[6], 'bg col shift', -50, 50, 0)
    bg_scale_slider = Slider(sliders_axes[7], 'bg scale', 0.5, 1.5, 1.0)
    bg_roll_slider = Slider(sliders_axes[8], 'bg roll', -numpy.pi, numpy.pi, 0)

    def get_current_values():
        return [value_array[index]
                for value_array, index
                in safe_izip(value_arrays, indices)]

    def draw_indices():
        '''
        Draws the current indices and their corresponding values.
        '''

        values = get_current_values()

        def rad_to_deg(radians):
            return radians / numpy.pi * 180.0

        values[1] = int(rad_to_deg(values[1]))
        values[2] = int(rad_to_deg(values[2]))

        index_names = ['id', 'elev', 'azim', 'light']

        lines = ['{}: {}'.format(index_name, value)
                 for index_name, value
                 in safe_izip(index_names, values)]

        lines[dimension_to_edit[0]] = '==> ' + lines[dimension_to_edit[0]]

        if fg_baseline_scales is not None:
            lines.append('fg baseline scale: {:0.2f}'.format(
                fg_baseline_scales[values[0]]))

        text_axes.clear()

        # "transAxes": 0, 0 = bottom-left, 1, 1 at upper-right.
        text_axes.text(0.1,  # x
                       0.5,  # y
                       '\n'.join(['fg image:', ''] +lines),
                       verticalalignment='center',
                       transform=text_axes.transAxes)

    def draw_image():
        '''
        Draws the image rendered according to <indices>.
        '''
        values = get_current_values()
        values = [numpy.asarray([v]) for v in values]

        fg_baseline_scale = (1.0 if fg_baseline_scales is None
                             else fg_baseline_scales[values[0]])

        fg_images = fg_renderer(fg_ids=values[0],
                                elevations=values[1],
                                azimuths=values[2],
                                lightings=values[3],
                                rc_shifts=fg_rc_shift[numpy.newaxis, :],
                                scales=(fg_scale[numpy.newaxis, ...] *
                                        fg_baseline_scale),
                                rolls=fg_roll[numpy.newaxis, ...])

        bg_images = bg_renderer(bg_frame_number[numpy.newaxis],
                                bg_rc_shift[numpy.newaxis, :],
                                bg_scale[numpy.newaxis],
                                bg_roll[numpy.newaxis])

        composite_images = (fg_images[..., :3] * fg_images[..., 3:] +
                            bg_images[...] * (1.0 - fg_images[..., 3:]))

        image_axes.clear()
        image_axes.imshow(composite_images[0, ...],
                          norm=matplotlib.colors.NoNorm(),
                          aspect='equal')  # match aspect ratio of image

    # Define and connect keyboard callback for choosing fg image, and quitting.

    def on_key_press(event):
        '''
        Key press callback.
        '''

        def increment_dimension_to_edit(increment):
            '''
            Increments or decrements index_dimension[0] by <increment>.
            '''

            assert_in(increment, (-1, 1))
            dimension_to_edit[0] = \
                (dimension_to_edit[0] + increment) % len(indices)

        def increment_index(increment):
            '''
            Increments or decrements inidices[dimension_to_edit[0]]
            '''

            assert_in(increment, (-1, 1))
            max_index = max_indices[dimension_to_edit[0]]
            indices[dimension_to_edit[0]] = \
                (indices[dimension_to_edit[0]] + increment) % (max_index + 1)

        if event.key == 'q':
            sys.exit(0)
        elif event.key == 'up':
            increment_dimension_to_edit(-1)
            draw_indices()
        elif event.key == 'down':
            increment_dimension_to_edit(1)
            draw_indices()
        elif event.key in ('left', 'right'):
            increment_index(-1 if event.key == 'left' else 1)
            draw_indices()
            draw_image()
        else:
            return  # nothing changed, so don't bother drawing

        figure.canvas.draw()

    figure.canvas.mpl_connect('key_press_event', on_key_press)

    # Define and connect fg sliders' callbacks

    def on_fg_r_shift_slider_change(arg):
        fg_rc_shift[0] = arg
        draw_image()

    fg_rshift_slider.on_changed(on_fg_r_shift_slider_change)

    def on_fg_c_shift_slider_change(arg):
        fg_rc_shift[1] = arg
        draw_image()

    fg_cshift_slider.on_changed(on_fg_c_shift_slider_change)

    def on_fg_scale_slider_change(arg):
        fg_scale[...] = arg
        draw_image()

    fg_scale_slider.on_changed(on_fg_scale_slider_change)

    def on_fg_roll_slider_change(arg):
        fg_roll[...] = arg
        draw_image()

    fg_roll_slider.on_changed(on_fg_roll_slider_change)

    # Define and connnect bg sliders' callbacks
    def on_bg_frame_slider_change(arg):
        bg_frame_number[...] = arg
        draw_image()

    bg_frame_slider.on_changed(on_bg_frame_slider_change)

    def on_bg_r_shift_slider_change(arg):
        bg_rc_shift[0] = arg
        draw_image()

    bg_rshift_slider.on_changed(on_bg_r_shift_slider_change)

    def on_bg_c_shift_slider_change(arg):
        bg_rc_shift[1] = arg
        draw_image()

    bg_cshift_slider.on_changed(on_bg_c_shift_slider_change)

    def on_bg_scale_slider_change(arg):
        bg_scale[...] = arg
        draw_image()

    bg_scale_slider.on_changed(on_bg_scale_slider_change)

    def on_bg_roll_slider_change(arg):
        bg_roll[...] = arg
        draw_image()

    bg_roll_slider.on_changed(on_bg_roll_slider_change)

    # Draw initial screen state, start render loop

    draw_indices()
    draw_image()
    pylab.show()

if __name__ == '__main__':
    main()


import re

from typing import Optional, Dict, List

creature_loot_pattern = re.compile(r"\|{{Loot Item\|(?:([\d?+-]+)\|)?([^}|]+)")
min_max_pattern = re.compile(r"(\d+)-(\d+)")
loot_stats_pattern = re.compile(r"\|([\s\w]+),\s*times:(\d+)(?:,\s*amount:([\d-]+))?")
kills_pattern = re.compile(r"kills=(\d+)")
int_pattern = re.compile(r"[+-]?\d+")
float_pattern = re.compile(r'[+-]?(\d*[.])?\d+')
named_links_pattern = re.compile(r'\[\[[^]|]+\|([^]]+)\]\]')
link_pattern = re.compile(r'\[\[([^|\]]+)')
links_pattern = re.compile(r'\[\[([^]]+)\]\]')
external_links_pattern = re.compile(r'\[[^]]+\]')

teaches_template = re.compile(r"{{Teaches\s*(?:\|name=([^|]+))?([^}]+)}}")
spells_pattern = re.compile(r"\|([^|]+)")

price_to_template = re.compile(r"{{Price to (?:Buy|Sell)\s*([^}]+)}}")
npc_offers = re.compile(r"\|([^|:\[]+)(?::\s?(\d+))?(?:\s?\[\[([^\]]+))?")

trades_sell_template = re.compile(r"{{Trades/Sells\s*(?:\|note=([^|]+))?([^}]+)}}")
npc_trades = re.compile(r"\|([^|,\[]+)(?:,\s?([+-]?\d+))?(?:\s?\[\[([^\]]+))?")

transport_template = re.compile(r"{{Transport\s*(?:\|discount=([^|]+))?([^}]+)}}")
npc_destinations = re.compile(r"\|([^,]+),\s?(\d+)(?:;\s?([^|]+))?")

def parse_item_offers(value: str) -> List:
    match = price_to_template.search(value)
    if match:
        return npc_offers.findall(match.group(1))
    else:
        return []

def parse_item_trades(value: str) -> List:
    result = []
    for note, trades in trades_sell_template.findall(value):
        result.extend(npc_trades.findall(trades))
    return result

def parse_destinations(value: str) -> List:
    result = []
    for __, destinations in transport_template.findall(value):
        result.extend(npc_destinations.findall(destinations))
    return result

def parse_loot(value: str):
    return creature_loot_pattern.findall(value)

def parse_loot_statistics(value):
    match = kills_pattern.search(value)
    if match:
        return int(match.group(1)), loot_stats_pattern.findall(value)
    else:
        return 0, None

def parse_min_max(value):
    match = min_max_pattern.search(value)
    if match:
        return int(match.group(1)), int(match.group(2))
    else:
        return 0, parse_integer(value, 1)

def parse_integer(value: str, default=0):
    if value is None:
        return default
    match = int_pattern.search(value)
    if match:
        return int(match.group(0))
    else:
        return default

def parse_float(value: str, default=0):
    if value is None:
        return default
    match = float_pattern.search(value)
    if match:
        return float(match.group(0))
    else:
        return default

def parse_maximum_integer(value: str) -> Optional[int]:
    if value is None:
        return None
    matches = int_pattern.findall(value)
    try:
        return max(list(map(int, matches)))
    except ValueError:
        return None

def parse_boolean(value: str):
    return value is None or value.strip().lower() == "yes"

def clean_links(content):
    if content is None:
        return None
    # Named links
    content = named_links_pattern.sub('\g<1>', content)
    # Links
    content = links_pattern.sub('\g<1>', content)
    # External links
    content = external_links_pattern.sub('', content)
    # Double spaces
    content = content.replace('  ', ' ')
    return content

def parse_attributes(content) -> Dict[str, str]:
    attributes = dict()
    depth = 0
    parse_value = False
    attribute = ""
    value = ""
    for i in range(len(content)):
        if content[i] == '{' or content[i] == '[':
            depth += 1
            if depth >= 3:
                if parse_value:
                    value = value + content[i]
                else:
                    attribute = attribute + content[i]
        elif content[i] == '}' or content[i] == ']':
            if depth >= 3:
                if parse_value:
                    value = value + content[i]
                else:
                    attribute = attribute + content[i]
            if depth == 2:
                attributes[attribute.strip()] = value.strip()
                parse_value = False
                attribute = ""
                value = ""
            depth -= 1
        elif content[i] == '=' and depth == 2:
            parse_value = True
        elif content[i] == '|' and depth == 2:
            attributes[attribute.strip()] = value.strip()
            parse_value = False
            attribute = ""
            value = ""
        elif parse_value:
            value = value + content[i]
        else:
            attribute = attribute + content[i]
    return dict((k, v.strip()) for k, v in attributes.items() if v.strip())

def parse_spells(value):
    result = []
    for name, spell_list in teaches_template.findall(value):
        spells = spells_pattern.findall(spell_list)
        spells = [s.strip() for s in spells]
        result.append((name, spells))
    return result

def convert_tibiawiki_position(pos) -> int:
    """Converts from TibiaWiki position system to regular numeric coordinates

    TibiaWiki takes the coordinates and splits in two bytes, represented in decimal, separated by a period."""
    position_splits = pos.strip().split(".")
    try:
        coordinate = int(position_splits[0]) << 8
        if len(position_splits) > 1 and position_splits[1].strip():
            coordinate += int(position_splits[1])
        return coordinate
    except (ValueError, IndexError):
        return 0

def parse_links(value):
    return list(link_pattern.findall(value))


# edit by sakana - 2019/10/31
# 這個 python 是實驗刪除 bucket 內所有 versioning 的檔案, 然後嘗試刪除 bucket
# 還待修正內容
import boto3

# 定義 BUCKET, 要執行的 bucket
BUCKET = 'sakanatest'

# 定義使用 s3 resource
s3 = boto3.resource('s3')

# 使用 for 迴圈列出所有 bucket 的名稱
for bucket in s3.buckets.all():
  print(bucket.name)

# 定義 bucket, 爲剛剛定義的 bucket
bucket = s3.Bucket(BUCKET)

# 檢查相關資訊
print("bucket is :", bucket)

# 將該 bucket 內的所有 versioning 刪除
bucket.object_versions.all().delete()

# 刪除 bucket
bucket.delete()


from django.conf.urls import url
from . import views
from django.contrib.auth.decorators import login_required

app_name ='kilogram'

urlpatterns = [
  url(r'^$', login_required(views.IndexView.as_view()), name = 'index'),
  url(r'^upload$', views.upload, name = 'upload'),
  url(r'^profile/(?P<pk>[0-9]+)/$', login_required(views.ProfileView.as_view()), name='profile'),
  url(r'^profile/update/$', login_required(views.ProfileUpdateView.as_view()),
  name='profile_update'),
]


import sys
import os
from os import path
from focal_loss import FocalLoss
print(path.dirname(__file__))
sys.path.append(path.join(path.dirname(__file__), '..'))

import torch.nn as nn
from torch.autograd import Variable
import torch.optim as optim
from models.deeppix.deeppix_six_channel import DeepPixBiS
from trainers.deeppix_trainer import Trainer
import torch
from dataloaders.deeppixdataloader import get_dataloaders

# Specify other training parameters
batch_size = 32
num_workers = 4
epochs = 50
learning_rate = 0.0001
weight_decay = 0.00001
save_interval = 1
seed = 3
use_gpu = True
output_dir = '/home/ec2-user/SageMaker/gaze-research/ckpts/deeppix/six_channel/proto3/'
save_path = f'{output_dir}/proto3-best.pt'

if not os.path.exists(output_dir):
    os.makedirs(output_dir)

network = DeepPixBiS(pretrained=True)
gpus = [i for i in range(4)]
if torch.cuda.device_count() > 1:
    network = nn.DataParallel(network, device_ids=gpus).cuda(gpus[0])
# set trainable parameters
for name, param in network.named_parameters():
    param.requires_grad = True

# loss definitions
criterion_pixel = nn.BCELoss()

criterion_bce = nn.BCELoss()

# optimizer initialization
optimizer = optim.Adam(filter(lambda p: p.requires_grad, network.parameters(
)), lr=learning_rate, weight_decay=weight_decay)

def compute_loss(network, img, labels, device):
    """
    Compute the losses, given the network, data and labels and
    device in which the computation will be performed.
    """

#     imagesv = Variable(img['image'].to(device))
    labelsv_pixel = Variable(labels['pixel_mask'].to(device))
    labelsv_binary = Variable(labels['binary_target'].to(device))
    out = network(img)

    beta = 0.5
#     loss_pixel = {}
    loss_pixel = criterion_pixel(out[0].squeeze(1), labelsv_pixel.float())
    loss_bce = criterion_bce(out[1], labelsv_binary.unsqueeze(1).float())
    loss = beta*loss_bce + (1.0-beta)*loss_pixel

    return loss

trainer = Trainer(network, optimizer, compute_loss, learning_rate=learning_rate,
                  batch_size=batch_size, device=f'cuda:{gpus[0]}' if torch.cuda.is_available() else 'cpu', do_crossvalidation=True, save_interval=save_interval, save_path=save_path)

print("Data loading started!")
# train_dataloader, val_dataloader = get_dataloaders()
dataloader = get_dataloaders(data_type="oulu", color_mode=['hsv', 'ycr_cb'], hard_protocol='Protocol_3')#rgb
print("Data Loaded!")
trainer.train(dataloader, n_epochs=epochs, output_dir=output_dir)


from utils.functions import send_class
from discord.ext import commands, tasks
import discord
import os
import re

# If testing locally, use this instead of the OS environment variables
# TOKEN = ''

# with open('config.txt', 'r') as f:
#    TOKEN = f.readline().strip()

TOKEN = os.environ['CLASSBOT_TOKEN'].strip()
# print(TOKEN.strip())

# init member caching (for member count across guilds)
intents = discord.Intents.default()
intents.members = True

bot = commands.Bot(command_prefix=('c$', 'C$'), case_insensitive=True, help_command=None, intents=intents)
messages = []

# What to do when bot is online
@bot.event
async def on_ready():
    # Log events to console.
    print('Bot online.')
    print("Name: {}".format(bot.user.name))
    print("ID: {}".format(bot.user.id))

    print('In {} guilds'.format(len(bot.guilds)))
    members = [] # sum([guild.member_count for guild in bot.guilds])
    total_members = 0
    for guild in bot.guilds:
        for member in guild.members:
            members.append(member.id)
        total_members += guild.member_count
        print('In {}, with owner {}\t\tUsers: {}'.format(guild.name, guild.owner, guild.member_count))
        # print('Guild permissions: {}'.format(guild.me.guild_permissions))

    members = set(members)
    print('Serving a total of {} members'.format(total_members))
    print('Total unique members: {}'.format(len(members)))
    # Set status
    await bot.change_presence(activity=discord.Activity(type=discord.ActivityType.listening, name='c$info'))

# Parse every message
@bot.event
async def on_message(message):
    if message.author.bot:
        return

    # TODO Move string parsing to helper function
    potential_message = '[' and ']' in message.content
    classes = []
    if potential_message:
        # Search for quotes in the message
        msg = message.content.split('\n')
        # All quotes start with '> ' and end with new line
        msg = [x for x in msg if not x.startswith('> ')]
        if len(msg) > 0:
            # Remake the message but without the quote part.
            msg = ''.join(x for x in msg)
            msg = msg.upper()
            # Parse the message. Converting to a set removes duplicates.
            classes = list(set(re.findall('\\\\?\[([A-Za-z]{2,4})\s?(\d{3})\\\\?\]', msg)))

    # Find all classes in an input string.
    if len(classes) > 6:
        await message.channel.send("To reduce spam, please limit your message to 6 or less classes.")
        # Iterate through the courses
    elif len(classes) > 0:
        for course in classes:
            await send_class(message.channel, course)

    await bot.process_commands(message)

# Add a help command to the bot.
@bot.command(name='info', aliases=['help'])
async def await_info(ctx):
    desc = 'To get a class, do [`department` `number`]. For example: `[cs 225]`. This is case insensitive, ' \
           'and the space between the department and the class number is optional. '
    embed = discord.Embed(title='Help', description=desc, url='https://github.com/timot3/uiuc-classes-bot/')
    embed.add_field(name='AP - c$AP', value='Links the UIUC AP credit page')
    embed.add_field(name='Geneds - c$Gened', value='Links the Geneds by GPA page.')

    embed.add_field(name='API Latency', value=str(round(bot.latency * 1000, 1))+'ms.')
    embed.set_footer(text='Having issues with the bot? Send a DM to @10x engineer#9075')
    await ctx.send(embed=embed)

@bot.command(name='AP')
async def await_ap(ctx):
    await ctx.send('https://admissions.illinois.edu/Apply/Freshman/college-credit-AP')

@bot.command(name='geneds', aliases=['gened'])
async def await_ap(ctx):
    await ctx.send('http://waf.cs.illinois.edu/discovery/every_gen_ed_at_uiuc_by_gpa/')

@bot.command(name='users', aliases=['usercount'])
async def await_usercount(ctx):
    members = []
    for guild in bot.guilds:
        for member in guild.members:
            members.append(member.id)
    total_members = len(members)
    unique_members = len(set(members))
    guilds = len(bot.guilds)
    await ctx.send('Online with {} servers and {} total members (Unique members: {}).'
                   .format(guilds, total_members, unique_members))

# Run the bot.
bot.run(TOKEN)


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 17 13:44:25 2017

@author: nownow
"""
import numpy as np
import matplotlib.pyplot as plt
plt.ion()
from scipy.stats import norm

def show_manifold(generator,digit):
    n = 15  # figure with 15x15 digits
    digit_size = 28
    figure = np.zeros((digit_size * n, digit_size * n))
    # linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
    # to produce values of the latent variables z, since the prior of the latent space is Gaussian

    class_vec = np.zeros(shape = [1,10])

    class_vec[0,digit] = 1

    grid_x = norm.ppf(np.linspace(0.05, 0.95, n), loc = 0 , scale = 1)
    grid_y = norm.ppf(np.linspace(0.05, 0.95, n), loc = 0, scale = 1)

    for i, yi in enumerate(grid_x):
        for j, xi in enumerate(grid_y):
            z_sample = np.array([[xi, yi]])
            x_decoded = generator.predict([z_sample,class_vec])
            digit = x_decoded[0].reshape(digit_size, digit_size)
            figure[i * digit_size: (i + 1) * digit_size,
                   j * digit_size: (j + 1) * digit_size] = digit
    plt.figure(figsize=(10, 10))
    plt.imshow(figure, cmap=plt.cm.gray)
    plt.show()

def show_digits(generator,a):
    latent_dim = generator.input_shape[0][1]
    vector = np.random.normal(size = [len(a),latent_dim])
    class_input = np.zeros(shape=[len(a),10])
    for i in range(len(a)):
        class_input[i][int(a[i])] = 1

    pred1 = generator.predict([vector,class_input])
    pred2 = np.reshape(pred1,(len(a)*28,28))
    plt.imshow(pred2, cmap = plt.cm.gray)
    plt.show()


# ###################
# Data pull from MongoDB
from pymongo import MongoClient
import pprint
mongo = MongoClient()

db = mongo.thoroughbred
pipeline = [{"$unwind": "$name"},
            {"$lookup": {"from": "earnings", "localField": "name", "foreignField": "name", "as": "earnings"}},
            {"$lookup": {"from": "current", "localField": "name", "foreignField": "name", "as": "current"}},
            {"$replaceRoot": {"newRoot": {"$mergeObjects": [
                {"$arrayElemAt": ["$current", 0]}, "$$ROOT"]}}},
            {"$project": {"current": 0}},
            {"$replaceRoot": {"newRoot": {"$mergeObjects": [
                {"$arrayElemAt": ["$earnings", 0]}, "$$ROOT"]}}},
            {"$project": {"earnings": 0}},
            {"$match": {"workouts": {"$ne": []}}}]
m_query = db.workouts.aggregate(pipeline)
# pprint.pprint(list(m_query))

# ###################
# Preprocess data
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

df = pd.DataFrame(list(m_query))
print(list(df))
df1 = df[['age', 'date', 'distance', 'notes',
          'race', 'sex', 'speed', 'time', 'track', 'type', 'winnings', 'speed_rating', 'starts', 'win', 'thirds', 'seconds']]

# remove space in notes:
df1['notes'] = df1['notes'].replace(' ', '', regex=True)

df2 = df1[['age', 'distance', 'notes', 'race', 'sex', 'speed',
           'time', 'track', 'type', 'winnings', 'speed_rating', 'starts', 'win', 'thirds', 'seconds']]

df2['race'].replace('', 'Raced', inplace=True)
df2['speed_rating'].replace('N/A', '', inplace=True)
df2.replace('', np.nan, inplace=True)
df2 = df2.dropna()

# assign numbers to values
from sklearn import preprocessing
from sklearn.preprocessing import LabelBinarizer

enc = preprocessing.OneHotEncoder()
le = preprocessing.LabelEncoder()

# df3 = df2[['notes', 'time']]

df2 = df2[['sex', 'type', 'time', 'distance', 'speed',
           'race', 'age', 'notes', 'winnings', 'speed_rating', 'starts', 'win', 'thirds', 'seconds']]
# view how many of each catagory
print(df2.groupby('type').nunique())
# n ominal datasets:
df2['sex'] = df2['sex'].astype('category')
df2['notes'] = df2['notes'].astype('category')
df2['race'] = df2['race'].astype('category')
df2['type'] = df2['type'].astype('category')
df2['speed'] = df2['speed'].astype('category')
df2 = pd.get_dummies(df2, columns=['sex', 'type', 'race', 'notes', 'speed'])

# ordinal datasets:
# scale_mapper = {'Fast': 1, 'Firm': 2, 'Wet Fast': 3,
#                'Good': 4, 'Sloppy': 5, 'Muddy': 6, 'Yielding': 7}
#df2['speed'] = df2['speed'].replace(scale_mapper)

# avoid dummy trap
df2.drop('notes_b', axis=1, inplace=True)
df2.drop('sex_F', axis=1, inplace=True)
df2.drop('speed_Fast', axis=1, inplace=True)
df2.drop('type_Turf', axis=1, inplace=True)
df2.drop('race_Raced', axis=1, inplace=True)
print(df2)
# place into X and Y
X = df2.drop('time', axis=1)
Y = df2.time

# ###################
# Split Data sets
from sklearn import cross_validation

X_train, X_test, Y_train, Y_test = cross_validation.train_test_split(
    X, Y, test_size=0.2, random_state=5)

from sklearn import linear_model

lm = linear_model.LinearRegression()
model = lm.fit(X_train, Y_train)
pred_train = lm.predict(X_train)
pred_test = lm.predict(X_test)

print(model.coef_)
# R2 score - closer to 1 the better
print('R2 Score: ')
print(model.score(X_test, Y_test))
# MSE - average amount off by
print('MSE: ')
print(np.mean((model.predict(X_test) - Y_test) ** 2))
# sum of all MSE's
print('MSE Sum: ')
print(((Y_test - model.predict(X_test)) ** 2).sum())


from tweepy import StreamListener, Stream, OAuthHandler
import json

CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_KEY = ''
ACCESS_SECRET = ''

class StdOutListener(StreamListener):
    def on_data(self, data):
        tweet_info = json.loads(data)
        tweet_text = tweet_info["text"]
        tweet_link = "https://twitter.com/" + tweet_info["user"]["screen_name"] + "/status/" + str(tweet_info["id"])
        if(( "source" in tweet_text or "sources" in tweet_text) and ("ESPN" in tweet_text)):
            print(tweet_link)

    def on_error(self, status):
        print(status)

auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)

twitterStream = Stream(auth,StdOutListener())
twitterStream.filter(follow=["1147076348642385920"])