# -*- coding: utf-8 -*-
"""

Script Name: Attributes.py
Author: Do Trinh/Jimmy - 3D artist.

Description:

"""
# -------------------------------------------------------------------------------------------------------------
""" import """

import weakref

from utils                      import attr_type
from bin.data.damg              import DAMG

class Attribute(DAMG):

    attribute_type = 'generic'
    REQUIRED = ['name', 'attr_type', 'value', '_edges']

    def __init__(self, name, value, dagnode=None, user=True, **kwargs):

        # private attributes
        self._dag               = weakref.ref(dagnode) if dagnode else None
        self._type              = kwargs.get('attr_type', None)
        self._edges             = []

        self.name               = name
        self.label              = kwargs.get('label', "")
        self.default_value      = kwargs.get('default_value', "")
        self.value              = value

        self.doctstring         = kwargs.get('doctstring', '')
        self.desc               = kwargs.get('desc', '')

        # globals
        self.user               = user
        self.private            = kwargs.get('private', False)  # hidden
        self.hidden             = kwargs.get('hidden', False)
        self.connectable        = kwargs.get('connectable', False)
        self.locked             = kwargs.get('locked', False)
        self.required           = kwargs.get('required', False)

        # connection
        self.connection_type    = kwargs.get('connection_type', 'input')
        self.data_type          = kwargs.get('data_type', None)
        self.max_connections    = kwargs.get('max_connections', 1)  # 0 = infinite

        if self.connectable:
            pass

    def update(self, **kwargs):
        for name, value in kwargs.iteritems():
            if value not in [None, 'null']:
                if name not in ['_edges']:
                    if hasattr(self, name) and value != getattr(self, name):
                        print('# DEBUG: Attribute "%s" updating value: "%s": "%s" - "%s"' % (
                        self.name, name, value, getattr(self, name)))
                    setattr(self, name, value)

    @property
    def data(self):

        for attr in ['label', 'value', 'desc', '_edges', 'attr_type', 'private',
                     'hidden', 'connectable', 'connection_type', 'locked', 'required', 'user']:
            if hasattr(self, attr):
                value = getattr(self, attr)
                if value or attr in self.REQUIRED:
                    self._data[attr] = value
        return self._data

    @property
    def dagnode(self):
        return self._dag()

    @property
    def attr_type(self):
        if self._type is not None:
            return self._type
        return attr_type(self.value)

    @attr_type.setter
    def attr_type(self, val):
        self._type = val

    @property
    def is_input(self):
        if not self.connectable:
            return False
        return self.connection_type == 'input'

    @property
    def is_output(self):
        if not self.connectable:
            return False
        return self.connection_type == 'output'

    def rename(self, name):
        old_name = self.name
        self.name = name

# -------------------------------------------------------------------------------------------------------------
# Created by panda on 21/07/2018 - 11:09 PM
# © 2017 - 2018 DAMGteam. All rights reserved


"""hrkmusic URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from rest_framework import routers
from hrkapp.views import ArtistaViewSet, AlbumViewSet, CancionViewSet, TipomusicaViewSet, CatalogoViewSet, UsuarioViewSet, PlaylistViewSet

router = routers.DefaultRouter()
# router.register(r'users', views.UserViewSet)
# router.register(r'groups', views.GroupViewSet)

router.register(r'artista', ArtistaViewSet, basename = 'artista')
router.register(r'album', AlbumViewSet, basename = 'album')
router.register(r'cancion', CancionViewSet, basename = 'cancion')
router.register(r'tipomusica', TipomusicaViewSet, basename = 'tipomusica')
router.register(r'catalogo', CatalogoViewSet, basename = 'catalogo')
router.register(r'usuario', UsuarioViewSet, basename = 'usuario')
router.register(r'playlist', PlaylistViewSet, basename = 'playlist')

urlpatterns = [
   path('', include(router.urls)),
   path('admin/', admin.site.urls),
   # path('api-auth/', include('rest_framework.urls', namespace='rest_framework'))
]


import cv2
import numpy as np

if __name__ == '__main__':
    def nothing(*arg):
        pass

cv2.namedWindow( "result" ) # создаем главное окно
cv2.namedWindow( "settings" ) # создаем окно настроек

# создаем 6 бегунков для настройки начального и конечного цвета фильтра
cv2.createTrackbar('h1', 'settings', 0, 255, nothing)
cv2.createTrackbar('l1', 'settings', 0, 255, nothing)
cv2.createTrackbar('s1', 'settings', 0, 255, nothing)
cv2.createTrackbar('h2', 'settings', 255, 255, nothing)
cv2.createTrackbar('l2', 'settings', 255, 255, nothing)
cv2.createTrackbar('s2', 'settings', 255, 255, nothing)
crange = [0,0,0, 0,0,0]

while True:
    img = cv2.imread("way.jpg")
    hls = cv2.cvtColor(img, cv2.COLOR_BGR2HLS )

    # считываем значения бегунков
    h1 = cv2.getTrackbarPos('h1', 'settings')
    l1 = cv2.getTrackbarPos('l1', 'settings')
    s1 = cv2.getTrackbarPos('s1', 'settings')
    h2 = cv2.getTrackbarPos('h2', 'settings')
    l2 = cv2.getTrackbarPos('l2', 'settings')
    s2 = cv2.getTrackbarPos('s2', 'settings')

    # формируем начальный и конечный цвет фильтра
    h_min = np.array((h1, l1, s1), np.uint8)
    h_max = np.array((h2, l2, s2), np.uint8)

    # накладываем фильтр на кадр в модели HSV
    thresh = cv2.inRange(hls, h_min, h_max)
    res = cv2.bitwise_and(img,img, mask= thresh)
    """
    cv2.imshow("res.jpg",res)
    low_threshold = 50
    high_threshold = 150
    edges = cv2.Canny(res, low_threshold, high_threshold)
    rho = 1  # distance resolution in pixels of the Hough grid
    theta = np.pi / 180  # angular resolution in radians of the Hough grid
    threshold = 15  # minimum number of votes (intersections in Hough grid cell)
    min_line_length = 100  # minimum number of pixels making up a line
    max_line_gap = 5
    line_image = np.copy(img) * 0
    lines = cv2.HoughLinesP(edges, rho, theta, threshold, np.array([]),
                    min_line_length, max_line_gap)
    for line in lines:
        for x1,y1,x2,y2 in line:
            cv2.line(line_image,(x1,y1),(x2,y2),(0,255,0),5)

    lines_edges = cv2.addWeighted(img, 0.8, line_image, 1, 0)

    #cv2.imwrite("ou.jpg",lines_edges)
	"""
    cv2.imshow('result', res)

    ch = cv2.waitKey(5)
    if ch == 27:
        break


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import messages
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.shortcuts import render, get_object_or_404, get_list_or_404, redirect
from django.http import HttpResponse, HttpResponseRedirect

from .forms import PostForm
from .models import Post

# Create your views here.
def post_create(request):
	form = PostForm(request.POST or None)
	if form.is_valid():
		instance = form.save(commit=False)
		#print form.cleaned_data.get('titulo')
		instance.save()
		messages.success(request, 'Tu post se ha creado con éxito')
		return HttpResponseRedirect(instance.get_absolute_url())
	#if request.method == 'POST':
	#	print request.POST
	context = {
		'form':form,
	}
	return render(request, 'post_form.html', context)
	#return HttpResponse("
CREATE
")

def post_detail(request, id=None):
	instance = get_object_or_404(Post, id=id)
	context = {
		'titulo': 'detail',
		'instance': instance
	}
	return render(request, 'post_detail.html', context)

def post_list(request):
	#queryset = get_list_or_404(Post).order_by("-timestamp")
	queryset = get_list_or_404(Post)

	paginator = Paginator(queryset, 5)

	page = request.GET.get('page')
	try:
		posts = paginator.page(page)
	except PageNotAnInteger:
		posts = paginator.page(1)
	except EmptyPage:
		posts = paginator.page(paginator.num_pages)

	context = {
		'titulo': 'list',
		'data': posts,
	}

	return render(request, 'post_list.html', context)

def post_update(request, id=None):
	instance = get_object_or_404(Post, id=id)
	form = PostForm(request.POST or None, instance=instance)
	if form.is_valid():
		instance = form.save(commit=False)
		#print form.cleaned_data.get('titulo')
		instance.save()
		messages.success(request, 'Tu post se ha actualizado con éxito', extra_tags="otra-clase")
		return HttpResponseRedirect(instance.get_absolute_url())
	context = {
		'titulo': instance.titulo,
		'instance': instance,
		'form':form,
	}
	return render(request, 'post_form.html', context)
	#return HttpResponse("
UPDATE
")

def post_delete(request, id):
	instance = get_object_or_404(Post, id=id)
	instance.delete()
	messages.success(request, 'Tu post se ha eliminado con éxito', extra_tags="otra-clase")
	return redirect('posts:list')


import multiprocessing as mp

from szqj.authors_networking.account_networking import *
from utils.mysql import data_fetch, Database, row_count

def update_author_networking_from_meta():
    start = 0
    db = Database()
    total = int(data_fetch("COUNT(*)", "essays", start=start, limit=None)[0][0])
    scan_count = start
    insert_count = 0
    limit = 10000

    while scan_count < total:

        if limit > total - scan_count:
            limit = total - scan_count

        data = data_fetch("*", "essays", start=start, limit=limit)

        for raw_item in data:
            obj = Essay(raw_item)
            response = obj.author_insert(db.conn)
            scan_count += 1
            if response:
                insert_count += response

            if scan_count % 1000 == 0:
                print(
                    "Scanned {} out of {} essays \n Inserted {} relation \n \n".format(scan_count, total, insert_count))
        start += limit

def worker(w_id, start, end):
    print("===============Process {} has started================".format(w_id))

    db = Database()
    total = end
    scan_count = 0
    limit = 10000
    chunk_size = end - start

    media_inserted = 0
    author_inserted = 0

    while scan_count < (end - start):

        if limit > total - scan_count - start:
            limit = total - scan_count - start

        data = data_fetch("*", "essays", start=start, limit=limit, tail_condition="ORDER BY `insert_time` DESC")

        for raw_item in data:
            obj = Essay(raw_item)
            media_count, author_count = obj.extractor_info_insert(db.conn)
            author_count += obj.meta_author_insert(db.conn)
            scan_count += 1
            if media_count + author_count:
                media_inserted += media_count
                author_inserted += author_count

            if scan_count % 1000 == 0:
                print(
                    "Process {} has scanned {} out of {} essays \n Inserted {} author relations \n Inserted {} media relations\n".format(
                        w_id, scan_count, chunk_size, author_inserted, media_inserted))
        start += limit
    print("===============Process {} has ended================".format(w_id))

if __name__ == "__main__":
    _ = 1

    num = row_count("essays", host_IP="192.168.164.15", database="raw")

    manager = mp.Manager()
    items = manager.list()

    process_num = 2
    inputs = []
    start_ind = 0
    chunk = int((num - start_ind) / process_num)

    for i in range(process_num):
        inputs.append((start_ind, start_ind + chunk))
        start_ind += chunk + 1
    print(inputs)
    counter = 0
    processes = []
    for input1, input2 in inputs:
        processes.append(mp.Process(target=worker, args=(counter, input1, input2,)))
        counter += 1

    # 运行所有进程
    for p in processes:
        p.start()

    # 确定所有进程结束
    for p in processes:
        p.join()


import logging
import hashlib
import json
import requests
from .http_util import get_cookie_from_cookiejar

lg = logging.getLogger('xiami.client')

class HTTPClient:
    base_uri = None

    def __init__(self, session: requests.Session, base_uri=None, headers=None):
        if base_uri:
            self.base_uri = base_uri
        self.headers = headers or {}
        self.session = session

    def request(self, method, uri, *args, **kwargs):
        url = self.base_uri + uri
        if 'headers' in kwargs:
            headers = dict(self.headers)
            headers.update(kwargs['headers'])
            kwargs['headers'] = headers
        else:
            if self.headers:
                kwargs['headers'] = self.headers

        if 'json_data' in kwargs:
            kwargs['data'] = json.dumps(kwargs.pop('json_data'))
            if 'headers' in kwargs:
                kwargs['headers'].update({
                    'Content-Type': 'application/json',
                })
        lg.debug(
            'HTTPClient request, %s, %s, %s, %s',
            method, url, args, kwargs)
        resp = getattr(self.session, method)(url, *args, **kwargs)
        lg.debug('Response: %s, %s', resp.status_code, resp.content[:100])
        return resp

    def get(self, uri, *args, **kwargs):
        return self.request('get', uri, *args, **kwargs)

    def post(self, uri, *args, **kwargs):
        return self.request('post', uri, *args, **kwargs)

class XiamiClient(HTTPClient):
    base_uri = 'https://www.xiami.com'

    def __init__(self, session, headers=None):
        super().__init__(session, headers=headers)

    # API methods

    def set_user_id(self, user_id):
        self.user_id = user_id

    def make_page_q(self, page, page_size):
        q = {
            "userId": self.user_id,
            "type": 1,
            "pagingVO": {
                "page": page,
                "pageSize": page_size,
            },
        }
        return q

    def get_fav_songs(self, page, page_size=30):
        lg.info(f'get_fav_songs: page={page}')
        uri = '/api/favorite/getFavorites'
        q = self.make_page_q(page, page_size)
        params = {
            '_q': param_json_dump(q),
            '_s': create_token(self.session, uri, q),
        }
        r = self.get(uri, params=params)
        # print(r.status_code, r.content.decode('utf-8'))
        data = r.json()

        # when out of max page, songs is "null"
        return data['result']['data']['songs']

    def get_play_info(self, song_ids):
        lg.info(f'get_play_info: song_ids={song_ids}')
        uri = '/api/song/getPlayInfo'
        q = {
            'songIds': song_ids,
        }
        r = self.get(uri, params={
            '_q': param_json_dump(q),
            '_s': create_token(self.session, uri, q),
        })
        data = r.json()

        return data['result']['data']['songPlayInfos']

def param_json_dump(o):
    return json.dumps(o, separators=(',', ':'))

def create_token(session, path, q):
    tk = get_cookie_from_cookiejar(session.cookies, 'xm_sg_tk')
    if not tk:
        raise ValueError('could not get xm_sg_tk from cookie')
    if q:
        q_json = param_json_dump(q)
    else:
        q_json = ''
    token_value = tk.value.split('_')[0] + '_xmMain_' + path + '_' + q_json
    token = get_md5_hex(token_value.encode())
    return token

def get_md5_hex(b: bytes):
    return hashlib.md5(b).hexdigest()


from flask import Flask, render_template, request, redirect, url_for, flash
from flask_mysqldb import MySQL

app = Flask(__name__)

#Mysql Connection
app.config['MYSQL_HOST'] = 'localhost'
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PASSWORD'] = 'John_20'
app.config['MYSQL_DB'] = 'flaskformulario'
mysql = MySQL(app)

# settings
app.secret_key ='mysecretkey'

# cambios para la tabla
@app.route('/')
def Index():
      cur = mysql.connection.cursor()
      cur.execute('SELECT * FROM eval')
      data = cur.fetchall()
      return render_template('index.html', eval = data)

@app.route('/add_contact', methods=['POST'])
def add_contact():
    if request.method == 'POST':
        fullname = request.form['fullname']
        phone = request.form['phone']
        email = request.form['email']
        num1 = request.form['num1']
        num2 = request.form['num2']
        num3 = request.form['num3']
        num4 = request.form['num4']
        num5 = request.form['num5']
        num6 = request.form['num6']
        cur = mysql.connection.cursor()
        cur.execute("INSERT INTO eval (fullname, phone, email, num1, num2, num3, num4, num5, num6) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", (fullname, phone, email, num1, num2, num3, num4, num5, num6))
        mysql.connection.commit()
        flash('Contact Added successfully')
        return redirect(url_for('Index'))

@app.route('/edit/<id>')
def get_contact(id):
    cur = mysql.connection.cursor()
    cur.execute('SELECT * FROM eval WHERE id = %s', (id))
    data = cur.fetchall()
    return render_template('edit-for.html', contact = data[0])

@app.route('/update/<id>', methods = ['POST'])
def update_contact(id):
    if request.method == 'POST':
        fullname = request.form['fullname']
        phone = request.form['phone']
        email = request.form['email']
        num1 = request.form['num1']
        num2 = request.form['num2']
        num3 = request.form['num3']
        num4 = request.form['num4']
        num5 = request.form['num5']
        num6 = request.form['num6']
        cur = mysql.connection.cursor()
        cur.execute("""
            UPDATE eval
            SET fullname = %s,
                email = %s,
                phone = %s,
                num1 = %s,
                num2 = %s,
                num3 = %s,
                num4 = %s,
                num5 = %s,
                num6 = %s
            WHERE id = %s
        """, (fullname, email, phone, num1, num2, num3, num4, num5, num6, id))
        flash('Edision Exitosa')
        mysql.connection.commit()
        return redirect(url_for('Index'))

@app.route('/delete/<string:id>')
def delete_contact(id):
  cur = mysql.connection.cursor()
  cur.execute('DELETE from eval WHERE id = {0}'. format(id))
  mysql.connection.commit()
  flash('Contacto Eliminado')
  return redirect(url_for('Index'))

if __name__ == '__main__':
 app.run(port = 3000, debug = True)


"""
Redis RDB backup script.
Written in Python 2.7
"""
# -*- coding: utf-8 -*-

from time import sleep
from datetime import datetime, timedelta

import argparse
import redis
import sys
import os
import shutil
import hashlib
import logging

__author__ = 'Luke.The.Coder'

def file_md5(filename, blocksize=2**20):
    f = open(filename)
    md5 = hashlib.md5()
    while True:
        data = f.read(blocksize)
        if not data:
            break
        md5.update(data)
    f.close()
    return md5.digest()

def checksum_compare(src, dst):
    """
    """
    assert(os.path.isfile(src) and os.path.isfile(dst))
    return file_md5(src) == file_md5(dst)

def bgsave_and_wait(r, timeout=timedelta(seconds=60)):
    assert (isinstance(r, redis.Redis))

    bgsave_begin = datetime.now()

    t0 = r.lastsave()
    if r.bgsave():
        while True:
            if r.lastsave() != t0:
                break
            if datetime.now() - bgsave_begin > timeout:
                return 'timeout'
            sleep(1)
        return 'ok'
    else:
        return 'failed'

def rdb_path(r):
    """
    Get&return redis config `dbfilename`
    """
    assert (isinstance(r, redis.Redis))
    d = r.config_get('dir')
    dbfilename = r.config_get('dbfilename')
    return '%s/%s' % (d['dir'], dbfilename['dbfilename'])

def aof_path(r, aof_filename):
    """
    Get&return redis config `appendfilename`
    """
    assert (isinstance(r, redis.Redis))
    d = r.config_get('dir')
    return os.path.join(d['dir'], aof_filename)

def copy_data_file(data_file, backup_dir, backup_filename, port, file_type):
    """
    Copies and renames the redis data file to backup dir, compare checksums
    when finished.

    The final backup name is:
    data_file_mtime.strftime(backup_filename)

    Returns backup file path when the copy was success and passed the checksum
    check. otherwise, return None
    """
    logger = logging.getLogger("main")

    df_mtime = os.path.getmtime(data_file)
    df_mtime = datetime.fromtimestamp(df_mtime).strftime(backup_filename)
    backup_filename = '%s.%s' % (df_mtime, file_type)
    backup_path = os.path.join(backup_dir, backup_filename)

    if not os.path.exists(backup_dir):
        os.makedirs(backup_dir)
    elif not os.path.isdir(backup_dir):
        logger.fatal('backupdir: %s is not a directory.\n' % backup_dir)
        return None
    elif os.path.exists(backup_path):
        logger.fatal('backupfile: %s already exists.\n' % backup_path)
        return None

    shutil.copy2(data_file, backup_path)

    if not checksum_compare(data_file, backup_path):
        logger.fatal('failed to copy dbfile %s, checksum compare failed.'
                         % data_file)
        return None
    logger.info('backup %s created. %s bytes, checksum ok!' \
                % (backup_path, os.path.getsize(backup_path)))
    return backup_path

def copy_rdb(rdb, backup_dir, backup_filename, port):
    """
    Copies and renames the rdb file to backup dir, compare checksums when
    finished.

    Returns backup file path when the copy was success and passed the checksum
    check. otherwise, return None
    """
    return copy_data_file(rdb, backup_dir, backup_filename, port, 'rdb')

def copy_aof(aof, backup_dir, backup_filename, port):
    """
    Copies and renames the aof file to backup dir, compare checksums when
    finished.

    Returns backup file path when the copy was success and passed the checksum
    check. otherwise, return None
    """
    return copy_data_file(aof, backup_dir, backup_filename, port, 'aof')

def clean_backup_dir(backup_dir, max_backups, port, file_type):
    """
    Removes oldest backups if the total number of backups exceeds max_backups
    """
    logger = logging.getLogger("main")

    file_suffix = '(port_%d).%s' % (port, file_type)
    files = [f for f in os.listdir(backup_dir) if f.endswith(file_suffix)]
    n_files = len(files)
    if n_files <= max_backups:
        return

    logger.info('number of backups(%d) exceeds limit(%d), deleting old backups.'\
        % (n_files, max_backups))

    files_time = []
    for filename in files:
        fp = '%s/%s' % (backup_dir, filename)
        # some time error
        files_time.append((fp, os.path.getmtime(fp)))
    files_time.sort(key=lambda x: x[1])

    for fp in files_time[:n_files - max_backups]:
        logger.info('delete %s' % fp[0])
        os.remove(fp[0])

    files = [f for f in os.listdir(backup_dir) if f.endswith(file_suffix)]
    assert(len(files) == max_backups)

def clean_rdb_backup(backup_dir, max_backups, port):
    """
    Removes oldest rdb backups if the total number of backups exceeds
    max_backups
    """
    return clean_backup_dir(backup_dir, max_backups, port, 'rdb')

def clean_aof_backup(backup_dir, max_backups, port):
    """
    Removes oldest aof backups if the total number of backups exceeds
    max_backups
    """
    return clean_backup_dir(backup_dir, max_backups, port, 'aof')

def main():
    # Ensures that there is only one instance of this script is running.
    # code from
    # http://stackoverflow.com/questions/380870/python-single-instance-of-program
    # from tendo import singleton
    # me = singleton.SingleInstance()

    # Setup command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-log_file', type=str, dest='log_file',
                        help='Path to log file', required=True)
    parser.add_argument('-log_level', type=str, dest='log_level',
                        help='Log level as defined in logging module',
                        default='INFO')
    parser.add_argument('-backup_dir', type=str, dest='backup_dir',
                        help='backup directory', default='./backups')
    parser.add_argument('-backup_filename', type=str, dest='backup_filename',
                        help='', default='redis_dump_%Y-%m-%d_%H%M%S')
    parser.add_argument('-redis_host', type=str, dest='redis_host',
                        help='redis host (name or IP address)', default='localhost')
    parser.add_argument('-redis_port', type=int, dest='redis_port',
                        help='redis port', default=6379)
    parser.add_argument('-max_backups', type=int, dest='max_backups',
                        help='maximum number of backups to keep', default=10)
    parser.add_argument('-bgsave_timeout', type=int, dest='bgsave_timeout',
                        help='bgsave timeout in seconds', default=60)
    parser.add_argument('-with_aof', dest="with_aof", help='enable backup aof',
                        action="store_true", default=False)
    parser.add_argument('-aof_filename', dest="aof_filename",
                        default='appendonly.aof', help='aof filename')

    # Parse command line arguments
    args = parser.parse_args()

    args.backup_dir = os.path.abspath(args.backup_dir)

    st = datetime.now()

    backup_dir = args.backup_dir
    backup_filename = args.backup_filename
    max_backups = args.max_backups
    redis_host = args.redis_host
    redis_port = args.redis_port
    bgsave_timeout = args.bgsave_timeout
    with_aof = args.with_aof
    aof_filename = args.aof_filename

    logger = logging.getLogger("main")
    logger.setLevel(args.log_level)

    # create the logging file handler
    fh = logging.FileHandler(args.log_file, mode='w')
    fh.setLevel(args.log_level)

    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    logger.addHandler(fh)

    logger.info('backup begin @ %s' % st)
    logger.info('backup dir: %s' % backup_dir)
    logger.info('backup file: %s' % backup_filename)
    logger.info('max backups: %s' % max_backups)
    logger.info('redis host: %s' % redis_host)
    logger.info('redis port: %s' % redis_port)
    logger.info('bgsave timeout: %s seconds' % bgsave_timeout)

    # Connect to local redis server
    r = redis.Redis(host=redis_host, port=redis_port)
    logger.info('connected to redis server %s:%d' % (redis_host, redis_port))

    # Get where redis saves the RDB file
    rdb = rdb_path(r)
    logger.info('redis rdb file path: %s' % rdb)

    if with_aof:
        aof = aof_path(r, aof_filename)
        logger.info('redis aof file path: %s' % aof)

    # Start bgsave and wait for it to finish
    logger.info('redis bgsave...')
    sys.stdout.flush()
    ret = bgsave_and_wait(r, timeout=timedelta(seconds=args.bgsave_timeout))
    logger.info(ret)

    if ret != 'ok':
        logger.fatal('%s %s\n' % ('backup failed!', datetime.now() - st))
        sys.exit(1)

    logger.info('starting copy rdb...')
    rdb_bak_path = copy_rdb(rdb, backup_dir, backup_filename, redis_port)
    if not rdb_bak_path:
        logger.fatal('%s %s\n' % ('backup failed!', datetime.now() - st))
        sys.exit(1)

    if with_aof:
        logger.info('starting copy aof...')
        aof_bak_path = copy_aof(aof, backup_dir, backup_filename, redis_port)
        if not aof_bak_path:
            os.remove(rdb_bak_path)
            logger.fatal('remove %s\n' % (rdb_bak_path))
            logger.fatal('%s %s\n' % ('backup failed!', datetime.now()-st))
            sys.exit(1)

    clean_rdb_backup(backup_dir, max_backups, redis_port)
    if with_aof:
        clean_aof_backup(backup_dir, max_backups, redis_port)

    logger.info('backup successful! time cost: %s seconds' % (datetime.now() - st).seconds)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Define the APL class to represent and parse a simc profile.

@author: skasch
"""

from collections import OrderedDict

from .actions import ActionList, PrecombatAction
from ..objects.units import Player, Target, Unit
from ..objects.lua import LuaCastable
from .context import Context
from ..abstract.helpers import indent
from ..constants import IGNORED_ACTION_LISTS
from ..database import CLASS_SPECS, TEMPLATES

class APL:
    """
    The main class representing an Action Priority List (or simc profile),
    extracted from its simc string.
    """

    DEFAULT_TEMPLATE = ('{context}'
                        '\n--- ======= ACTION LISTS =======\n'
                        '-- [3] Single Rotation\n'
                        'A[3] = function(icon, isMulti)\n\n'
                        '    --------------------\n'
                        '    --- ROTATION VAR ---\n'
                        '    --------------------\n'
                        '    local isMoving = A.Player:IsMoving()\n'
                        '    local isMovingFor = A.Player:IsMovingTime()\n'
                        '    local inCombat = Unit(player):CombatTime() > 0\n'
	                    '    local combatTime = Unit(player):CombatTime()\n'
                        '    local ShouldStop = Action.ShouldStop()\n'
                        '    local Pull = Action.BossMods_Pulling()\n'
	                    '    local DBM = Action.GetToggle(1, "DBM")\n'
	                    '    local HeartOfAzeroth = Action.GetToggle(1, "HeartOfAzeroth")\n'
	                    '    local Racial = Action.GetToggle(1, "Racial")\n'
	                    '    local Potion = Action.GetToggle(1, "Potion")\n\n'
                        '    ------------------------------------------------------\n'
                        '    ---------------- ENEMY UNIT ROTATION -----------------\n'
                        '    ------------------------------------------------------\n'
                        '    local function EnemyRotation(unit)\n\n'
                        '{action_lists}\n'
                        '{precombat_call}\n\n'
                        '        -- In Combat\n'
                        '        if inCombat and Unit(unit):IsExists() then\n\n'
                        '        {main_actions}\n'
                        '        end\n'
                        '    end\n'
                        '\n{set_apl}')

    def __init__(self):
        self.simc_lines = []
        self.player = None
        self.target = Target()
        self.profile_name = ''
        self.parsed = True
        self.apl_simc = ''
        self.show_comments = True
        self.action_lists_simc = OrderedDict()
        self.context = Context()
        LuaCastable.cid = 0

    def hide_simc_comments(self):
        """
        Hide the default commented simc lines to the printed lua code.
        """
        self.show_comments = False

    def set_simc_lines(self, simc_lines):
        """
        Set the simc_lines attribute of the object to the content of the
        variable simc_lines.
        """
        self.simc_lines = [simc_line for simc_line in simc_lines
                           if not simc_line.startswith('#')]
        self.parsed = False

    def read_profile(self, file_path):
        """
        Read a .simc profile file.
        """
        with open(file_path, 'r') as profile:
            self.set_simc_lines([line.strip() for line in profile.readlines()])

    def read_string(self, multiline_simc):
        """
        Read a simc profile from a multiline string.
        """
        self.set_simc_lines(multiline_simc.split('\n'))

    def process_lua(self):
        """
        Parse the profile read from the simc_lines attribute and print the lua
        code generated by the profile.
        """
        self.parse_profile()
        return self.print_lua()

    def export_lua(self, file_path):
        """
        Parse the profile read from the simc_lines attribute and export the lua
        code generated into file_path.
        """
        self.parse_profile()
        with open(file_path, 'w') as lua_file:
            lua_file.write(self.print_lua())

    def parse_profile(self):
        """
        Parse the profile from the simc_lines attribute.
        """
        if not self.parsed:
            for simc in self.simc_lines:
                self.parse_line(simc)
            self.parsed = True

    def parse_action(self, simc):
        """
        Parse a single line from the simc_lines attribute if this line is an
        action and append it in its action_list in action_lists_simc dict.
        """
        equal_index = simc.find('+=')
        equal_len = 2
        if equal_index == -1:
            equal_index = simc.find('=')
            equal_len = 1
        if equal_index == -1:
            return
        action_call = simc[:equal_index]
        action_simc = simc[equal_index + equal_len:]
        if '.' not in action_call:
            self.apl_simc += action_simc
            return
        action_list_name = action_call.split('.')[1]
        if action_list_name not in IGNORED_ACTION_LISTS:
            if action_list_name in self.action_lists_simc:
                self.action_lists_simc[action_list_name] += action_simc
            else:
                self.action_lists_simc[action_list_name] = action_simc

    def precombat_action(self):
        """
        Get the call to precombat actions.
        """
        return PrecombatAction(self)

    def main_action_list(self):
        """
        Get the ActionList object for the main action list.
        """
        return ActionList(self, self.apl_simc, 'APL')

    def action_lists(self):
        """
        Get the list of ActionList objects from action_lists_simc.
        """
        return [ActionList(self, simc, name)
                for name, simc in self.action_lists_simc.items()]

    def parse_line(self, simc):
        """
        Parse a single line in simc_lines.
        """
        if any(simc.startswith(class_) for class_ in CLASS_SPECS):
            class_, profile_name = simc.split('=')
            self.set_player(class_)
            self.set_profile_name(profile_name)
        elif simc.startswith('spec'):
            _, spec = simc.split('=')
            self.player.set_spec(spec)
        elif simc.startswith('level'):
            _, level = simc.split('=')
            self.player.set_level(level)
        elif simc.startswith('race'):
            _, race = simc.split('=')
            self.player.set_race(race)
        elif simc.startswith('actions'):
            self.parse_action(simc)

    def set_profile_name(self, simc):
        """
        Set the profile name.
        """
        self.profile_name = simc.replace('"', '')

    def set_player(self, simc):
        """
        Set a player as the main actor of the APL.
        """
        self.player = Player(simc, self)
        self.context.set_player(self.player)

    def set_target(self, simc):
        """
        Set the target of the main actor of the APL.
        """
        self.target = Target(simc)

    def print_action_list_names(self):
        """
        Print the definition of action list names in local.
        """
        action_list_names_lua = [action_list.name.print_lua()
                                 for action_list in self.action_lists()]
        if len(action_list_names_lua) > 0:
            return indent('local ' + ', '.join(action_list_names_lua))
        return ''

    def print_action_lists_lua(self):
        """
        Print the lua string of the APL.
        """
        return '\n'.join(indent(action_list.print_lua())
                         for action_list in self.action_lists())

    def print_set_apl(self):
        """
        Print the call to SetAPL to set the APL into HR.
        """
        class_simc = self.player.class_.simc
        spec_simc = self.player.spec.simc
        apl_id = CLASS_SPECS.get(class_simc, {}).get(spec_simc, 0)
        return ('    -- End on EnemyRotation()\n\n'
        '    -- Defensive\n'
        '    --local SelfDefensive = SelfDefensives()\n'
        '    if SelfDefensive then \n'
        '        return SelfDefensive:Show(icon)\n'
        '    end \n\n'

        '    -- Mouseover\n'
        '    if A.IsUnitEnemy("mouseover") then\n'
        '        unit = "mouseover"\n'
        '        if EnemyRotation(unit) then \n'
        '            return true \n'
        '        end \n'
        '    end \n\n'

        '    -- Target  \n'
        '    if A.IsUnitEnemy("target") then \n'
        '        unit = "target"\n'

        '        if EnemyRotation(unit) then \n'
        '            return true\n'
        '        end \n\n'
        '    end\n'
        'end\n'
        '-- Finished\n\n'
        '-- [4] AoE Rotation\n'
        'A[4] = function(icon)\n'
        '    return A[3](icon, true)\n'
        'end\n '
        '-- [5] Trinket Rotation\n'
        '-- No specialization trinket actions \n'
        '-- Passive \n'
        '--[[local function FreezingTrapUsedByEnemy()\n'
        '    if     UnitCooldown:GetCooldown("arena", 3355) > UnitCooldown:GetMaxDuration("arena", 3355) - 2 and\n'
        '    UnitCooldown:IsSpellInFly("arena", 3355) and \n'
        '    Unit("player"):GetDR("incapacitate") >= 50 \n'
        '    then \n'
        '        local Caster = UnitCooldown:GetUnitID("arena", 3355)\n'
        '        if Caster and Unit(Caster):GetRange() <= 40 then \n'
        '            return true \n'
        '        end \n'
        '    end \n'
        'end \n'

        'local function ArenaRotation(icon, unit)\n'
        '    if A.IsInPvP and (A.Zone == "pvp" or A.Zone == "arena") and not Player:IsStealthed() and not Player:IsMounted() then\n'
        '        -- Note: "arena1" is just identification of meta 6\n'
        '        if (unit == "arena1" or unit == "arena2" or unit == "arena3") then \n'
        '            -- Reflect Casting BreakAble CC\n'
        '            if A.NetherWard:IsReady() and A.NetherWard:IsSpellLearned() and Action.ShouldReflect(EnemyTeam()) and EnemyTeam():IsCastingBreakAble(0.25) then \n'
        '                return A.NetherWard:Show(icon)\n'
        '            end \n'
        '        end\n'
        '    end \n'
        'end \n'

        'local function PartyRotation(unit)\n'
        '    if (unit == "party1" and not A.GetToggle(2, "PartyUnits")[1]) or (unit == "party2" and not A.GetToggle(2, "PartyUnits")[2]) then \n'
        '        return false \n'
        '    end\n\n'
        '  	-- SingeMagic\n'
        '    if A.SingeMagic:IsCastable() and A.SingeMagic:AbsentImun(unit, Temp.TotalAndMag) and IsSchoolFree() and Action.AuraIsValid(unit, "UseDispel", "Magic") and not Unit(unit):InLOS() then\n'
        '        return A.SingeMagic:Show(icon)\n'
        '    end\n'

        'end \n\n'
        'A[6] = function(icon)\n'
        '    return ArenaRotation(icon, "arena1")\n'
        'end\n\n'
        'A[7] = function(icon)\n'
        '    local Party = PartyRotation("party1") \n'
        '    if Party then \n'
        '        return Party:Show(icon)\n'
        '    end \n'

        '    return ArenaRotation(icon, "arena2")\n'
        'end\n\n'
        'A[8] = function(icon)\n'
        '    local Party = PartyRotation("party2") \n'
        '    if Party then \n'
        '        return Party:Show(icon)\n'
        '    end     \n'
        '    return ArenaRotation(icon, "arena3")\n'
        'end]]--\n\n'
                )

    def template(self):
        return TEMPLATES.get(self.player.class_.simc+self.player.spec.simc, self.DEFAULT_TEMPLATE)

    def print_lua(self):
        """
        Print the lua string representing the action list.
        """
        function_name = self.main_action_list().name.lua_name()
        action_list_names = self.print_action_list_names()
        action_lists = self.print_action_lists_lua()
        precombat_call = self.precombat_action().print_lua()
        main_actions = self.main_action_list().print_actions_lua()
        context = self.context.print_lua()
        set_apl = self.print_set_apl()
        return self.template().format(
            context=context,
            function_name=function_name,
            action_list_names=indent(action_list_names),
            action_lists=indent(action_lists),
            precombat_call=precombat_call,
            main_actions=indent(indent(main_actions)),
            set_apl=set_apl
        )


#! /usr/bin/env python

import argparse
import os
import sys

import torch
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
import numpy as np
import random
import time
import datetime

from transformers import ElectraTokenizer
from keras.preprocessing.sequence import pad_sequences
from transformers import ElectraForSequenceClassification, AdamW, ElectraConfig
from transformers import get_linear_schedule_with_warmup

parser = argparse.ArgumentParser(description='Get all command line arguments.')
parser.add_argument('--batch_size', type=int, default=32, help='Specify the training batch size')
parser.add_argument('--learning_rate', type=float, default=5e-5, help='Specify the initial learning rate')
parser.add_argument('--adam_epsilon', type=float, default=1e-8, help='Specify the AdamW loss epsilon')
parser.add_argument('--lr_decay', type=float, default=0.85, help='Specify the learning rate decay rate')
parser.add_argument('--dropout', type=float, default=0.1, help='Specify the dropout rate')
parser.add_argument('--n_epochs', type=int, default=1, help='Specify the number of epochs to train for')
parser.add_argument('--n_samples', type=int, default=1, help='Specify the number of negative samples to take')
parser.add_argument('--seed', type=int, default=1, help='Specify the global random seed')
parser.add_argument('--num_topics', type=int, default=379, help='Specify the number of unique topics in training')
parser.add_argument('--reverse', type=bool, default=False, help='If true, then concatenate the response onto prompt instead of other way around')
parser.add_argument('--train_resps_path', type=str, help='Load path to training responses as text')
parser.add_argument('--valid_resps_path', type=str, help='Load path to vaidation responses as text')
parser.add_argument('--unique_prompts_path', type=str, help='Load path to unique prompts as text')
parser.add_argument('--unique_prompts_distribution_path', type=str, help='Load path to distribution of unique prompts')
parser.add_argument('--train_prompts_idxs_path', type=str, help='Load path to training data unique prompt indices (for dynamic shuffling)')
parser.add_argument('--valid_prompts_idxs_path', type=str, help='Load path to valid data unique prompt indices (for dynamic shuffling)')
parser.add_argument('--save_path', type=str, help='Load path to which trained model will be saved')

# Function to calculate the accuracy of our predictions vs labels
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

def format_time(elapsed):
    '''
    Takes a time in seconds and returns a string hh:mm:ss
    '''
    # Round to the nearest second.
    elapsed_rounded = int(round((elapsed)))

    # Format as hh:mm:ss
    return str(datetime.timedelta(seconds=elapsed_rounded))

# Set device
def get_default_device():
    if torch.cuda.is_available():
        print("Got CUDA!")
        return torch.device('cuda')
    else:
        return torch.device('cpu')

def _shuffle(p_id, r, r_msk, topics_dist, NUM_TOPICS, device):
    # Dynamic shuffling in order to generate negative samples
    bs = list(p_id.size())[0]
    y_true_first = np.ones(bs, dtype=int)
    y_true_second = np.zeros(bs, dtype=int)
    y_true = np.concatenate([y_true_first, y_true_second])
    y_true = torch.from_numpy(y_true)
    y_true = y_true.long().to(device)
    new_p_id = np.random.choice(NUM_TOPICS, bs, p=topics_dist)
    for i in range(bs):
        while (new_p_id[i] == p_id[i]):
            new_p_id[i] = np.random.choice(NUM_TOPICS, 1, p=topics_dist)
    new_p_id = torch.from_numpy(new_p_id)
    new_p_id = new_p_id.long().to(device)
    p_id = torch.cat((p_id, new_p_id), 0)
    r = torch.cat((r, r), 0)
    r_msk = torch.cat((r_msk, r_msk), 0)
    return p_id, r, r_msk, y_true, bs

def _get_prompts(p_id, topics, topics_msks):
    p = torch.index_select(topics, 0, p_id)
    p_msk = torch.index_select(topics_msks, 0, p_id)
    return p, p_msk

def _join_pr_resp(p, p_msk, r, r_msk, reverse):
    # Literally concatenate prompt and response without bothering
    # to put all the padding on the end
    if reverse:
        print("Reversing!")
        pr_resp = torch.cat((r, p), 1)
        pr_resp_msk = torch.cat((r_msk, p_msk), 1)
    else:
        print("NOT reversing")
        pr_resp = torch.cat((p, r), 1)
        pr_resp_msk = torch.cat((p_msk, r_msk), 1)
    return pr_resp, pr_resp_msk

def main(args):
    if not os.path.isdir('CMDs'):
        os.mkdir('CMDs')
    with open('CMDs/train.cmd', 'a') as f:
        f.write(' '.join(sys.argv) + '\n')
        f.write('--------------------------------\n')

    # Set the seed value all over the place to make this reproducible.
    seed_val = args.seed
    random.seed(seed_val)
    np.random.seed(seed_val)
    torch.manual_seed(seed_val)
    torch.cuda.manual_seed_all(seed_val)
    # Choose device
    device = get_default_device()

    prompts_train_idxs = np.loadtxt(args.train_prompts_idxs_path, dtype=np.int64)
    topics_dist = np.loadtxt(args.unique_prompts_distribution_path, dtype=np.int32)

    # Normalise
    topics_dist = topics_dist / np.linalg.norm(topics_dist, 1)

    # Load the BERT tokenizer.
    print('Loading BERT tokenizer...')
    tokenizer = ElectraTokenizer.from_pretrained('google/electra-base-discriminator', do_lower_case=True)

    with open(args.unique_prompts_path) as f:
        topics = f.readlines()
    # Remove whitespaces and convert to lowercase
    topics = [x.strip().lower() for x in topics]

    with open(args.train_resps_path) as f:
        responses = f.readlines()
    # Remove whitespaces and convert to lower case
    responses = [x.strip().lower() for x in responses]

    # Tokenize all the prompts and the responses and then map the tokens to their word IDs
    topic_ids = []
    for sent in topics:
        encoded_sent = tokenizer.encode(sent, add_special_tokens=True)
        topic_ids.append(encoded_sent)

    resp_ids = []
    for sent in responses:
        encoded_sent = tokenizer.encode(sent, add_special_tokens=True)
        resp_ids.append(encoded_sent)

    MAX_LEN_topic = max([len(sen) for sen in topic_ids])
    MAX_LEN_resp = max([len(sen) for sen in resp_ids])
    print('\nPadding token: "{:}", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))

    # Pad our input tokens with value 0.
    # "post" indicates that we want to pad and truncate at the end of the sequence,
    # as opposed to the beginning.
    topic_ids = pad_sequences(topic_ids, maxlen=MAX_LEN_topic, dtype="long",
                            value=0, truncating="post", padding="post")

    resp_ids = pad_sequences(resp_ids, maxlen=MAX_LEN_resp, dtype="long",
                            value=0, truncating="post", padding="post")

    # The attention mask simply makes it explicit which tokens are actual words versus which are padding.
    attention_masks_topic = []
    # For each sentence...
    for sent in topic_ids:
        # Create the attention mask.
        #   - If a token ID is 0, then it's padding, set the mask to 0.
        #   - If a token ID is > 0, then it's a real token, set the mask to 1.
        att_mask = [int(token_id > 0) for token_id in sent]
        # Store the attention mask for this sentence.
        attention_masks_topic.append(att_mask)
    attention_masks_resp = []
    for sent in resp_ids:
        # Create the attention mask.
        #   - If a token ID is 0, then it's padding, set the mask to 0.
        #   - If a token ID is > 0, then it's a real token, set the mask to 1.
        att_mask = [int(token_id > 0) for token_id in sent]
        # Store the attention mask for this sentence.
        attention_masks_resp.append(att_mask)

    # Convert to torch tensors

    prompts_train_idxs = torch.from_numpy(prompts_train_idxs)
    prompts_train_idxs = prompts_train_idxs.long()

    topic_ids = torch.tensor(topic_ids)
    topic_ids = topic_ids.long()
    topic_ids = topic_ids.to(device)

    attention_masks_topic = torch.tensor(attention_masks_topic)
    attention_masks_topic = attention_masks_topic.long()
    attention_masks_topic = attention_masks_topic.to(device)

    resp_ids = torch.tensor(resp_ids)
    resp_ids = resp_ids.long()
    resp_ids = resp_ids.to(device)

    attention_masks_resp = torch.tensor(attention_masks_resp)
    attention_masks_resp = attention_masks_resp.long()
    attention_masks_resp = attention_masks_resp.to(device)

    # Create the DataLoader for our training set.
    print(prompts_train_idxs.size(0))
    print(resp_ids.size(0))
    print(attention_masks_resp.size(0))
    train_data = TensorDataset(prompts_train_idxs, resp_ids, attention_masks_resp)
    train_sampler = RandomSampler(train_data)
    train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.batch_size)

    # Load BertForSequenceClassification, the pretrained BERT model with a single
    # linear classification layer on top.
    model = ElectraForSequenceClassification.from_pretrained(
        "google/electra-base-discriminator", # Use the 12-layer BERT model, with an uncased vocab.
        num_labels = 2, # The number of output labels--2 for binary classification.
                        # You can increase this for multi-class tasks.
        output_attentions = False, # Whether the model returns attentions weights.
        output_hidden_states = False, # Whether the model returns all hidden-states.
    )
    model.to(device)

    # Note: AdamW is a class from the huggingface library (as opposed to pytorch)
    # I believe the 'W' stands for 'Weight Decay fix"
    optimizer = AdamW(model.parameters(),
                    lr = args.learning_rate,
                    eps = args.adam_epsilon
                    )

    loss_values = []

    # Total number of training steps is number of batches * number of epochs.
    total_steps = len(train_dataloader) * args.n_epochs
    # Create the learning rate scheduler.
    scheduler = get_linear_schedule_with_warmup(optimizer,
                                                num_warmup_steps = 0, # Default value in run_glue.py
                                                num_training_steps = total_steps)

    for epoch in range(args.n_epochs):
        # Perform one full pass over the training set.
        print("")
        print('======== Epoch {:} / {:} ========'.format(epoch + 1, args.n_epochs))
        print('Training...')
        # Measure how long the training epoch takes.
        t0 = time.time()
        # Reset the total loss for this epoch.
        total_loss = 0
        model.train()
    # For each batch of training data...
        for step, batch in enumerate(train_dataloader):
            # Progress update every 40 batches.
            if step % 40 == 0 and not step == 0:
                # Calculate elapsed time in minutes.
                elapsed = format_time(time.time() - t0)
                # Report progress.
                print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
            p_id = batch[0].to(device)
            r = batch[1].to(device)
            r_msk = batch[2].to(device)
            # Perform dynamic shuffling
            p_id, r, r_msk, y_true, batch_size = _shuffle(p_id, r, r_msk, topics_dist, args.num_topics, device)
            # Get the prompts from the topics
            p, p_msk = _get_prompts(p_id, topic_ids, attention_masks_topic)
            p, p_msk = p.to(device), p_msk.to(device)
            # Concatenate prompts and responses
            pr_resp, pr_resp_msk = _join_pr_resp(p, p_msk, r, r_msk, args.reverse)
            pr_resp, pr_resp_msk = pr_resp.to(device), pr_resp_msk.to(device)
            model.zero_grad()

            # Perform a forward pass (evaluate the model on this training batch).
            # This will return the loss (rather than the model output) because we
            # have provided the `labels`.
            # The documentation for this `model` function is here:
            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
            outputs = model(pr_resp, token_type_ids=None, attention_mask=pr_resp_msk, labels=y_true)

            # The call to `model` always returns a tuple, so we need to pull the
            # loss value out of the tuple.
            loss = outputs[0]
            # Accumulate the training loss over all of the batches so that we can
            # calculate the average loss at the end. `loss` is a Tensor containing a
            # single value; the `.item()` function just returns the Python value
            # from the tensor.
            total_loss += loss.item()
            # Perform a backward pass to calculate the gradients.
            loss.backward()

            # Clip the norm of the gradients to 1.0.
            # This is to help prevent the "exploding gradients" problem.
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            # Update parameters and take a step using the computed gradient.
            # The optimizer dictates the "update rule"--how the parameters are
            # modified based on their gradients, the learning rate, etc.
            optimizer.step()
            # Update the learning rate.
            scheduler.step()
        # Calculate the average loss over the training data.
        avg_train_loss = total_loss / len(train_dataloader)

        # Store the loss value for plotting the learning curve.
        loss_values.append(avg_train_loss)

        print("")
        print("  Average training loss: {0:.2f}".format(avg_train_loss))
        print("  Training epoch took: {:}".format(format_time(time.time() - t0)))

        # NEED TO DO THE VALIDATION CODE NOW - see the rest of the tutorial at
        # https://medium.com/@aniruddha.choudhury94/part-2-bert-fine-tuning-tutorial-with-pytorch-for-text-classification-on-the-corpus-of-linguistic-18057ce330e1

    # Save the model to a file
    file_path = args.save_path+'electra_classifier_seed'+str(args.seed)+'.pt'
    torch.save(model, file_path)

if __name__ == '__main__':
    args = parser.parse_args()
    main(args)


from enum import Enum
import math
import MLKit

class ColumnAttributes:

    class Type(Enum):

        string = 0
        numeric = 1

        @staticmethod
        def type_of_value(value):
            try:
                float(value)
                return ColumnAttributes.Type.numeric
            except ValueError:
                pass

            return ColumnAttributes.Type.string

        @staticmethod
        def name_of_type(column_type):
            if column_type == ColumnAttributes.Type.string:
                return "String"
            elif column_type == ColumnAttributes.Type.numeric:
                return "Numeric"
            return None

    def __init__(self, column):
        self.count = 0
        self.mean = None
        self.minimum = float("inf")
        self.maximum = -float("inf")
        self.percent_25 = None
        self.percent_50 = None
        self.percent_75 = None
        self.std = None
        self.type = None
        self.numeric_values = {}
        self.__compute_attributes(column)

    def __compute_attributes(self, column):
        values_sum = 0

        for value in column.values:
            if value == None:
                continue

            if not self.__is_type_correct_for_value(value):
                MLKit.Display.warning("Column " + column.name + " contains different value types.")
                self.minimum = None
                self.maximum = None
                return

            self.__transform_value_to_numeric_if_needed(value)

            float_value = self.numeric_value_for_value(value)
            values_sum += float_value
            self.count += 1

            if float_value < self.minimum:
                self.minimum = float_value
            if float_value > self.maximum:
                self.maximum = float_value

        if self.count == 0:
            return

        self.mean = values_sum / self.count
        self.percent_25 = 0
        self.percent_50 = 0
        self.percent_75 = 0
        squared_sum = 0

        replaced_values = []
        for value in column.values:
            if value is None:
                continue
            replaced_values.append(self.numeric_value_for_value(value))
        sorted_values = sorted(replaced_values)

        for (index, value) in enumerate(sorted_values):
            if value == None:
                continue

            squared_sum += (self.mean - value) ** 2

            if index == round(self.count / 4):
                self.percent_25 = value
            if index == round(self.count / 2):
                self.percent_50 = value
            if index == round(3 * self.count / 4):
                self.percent_75 = value

        variance = squared_sum / self.count
        self.std = math.sqrt(variance)

    def __is_type_correct_for_value(self, value):
        if value == None:
            return True

        if self.type == ColumnAttributes.Type.string and value == "Nan":
            return True
        elif self.type == None:
            self.type = ColumnAttributes.Type.type_of_value(value)
            return True
        elif self.type == ColumnAttributes.Type.type_of_value(value):
            return True
        else:
            return False

    def __transform_value_to_numeric_if_needed(self, value):
        if value == None or self.type == ColumnAttributes.Type.numeric:
            return

        if self.numeric_values.get(value) == None:
            self.numeric_values[value] = len(self.numeric_values.keys())

    def numeric_value_for_value(self, value):
        if self.type == ColumnAttributes.Type.numeric:
            return float(value)

        return self.numeric_values[value]

    def value_for_key(self, name):
        if name == "Type":
            return ColumnAttributes.Type.name_of_type(self.type)
        elif name == "Count":
            return self.count
        elif name == "Mean":
            return self.mean
        elif name == "Std":
            return self.std
        elif name == "Min":
            return self.minimum
        elif name == "25%":
            return self.percent_25
        elif name == "50%":
            return self.percent_50
        elif name == "75%":
            return self.percent_75
        elif name == "Max":
            return self.maximum
        else:
            return None

    @staticmethod
    def all():
        return ["Type", "Count", "Mean", "Std", "Min", "25%", "50%", "75%", "Max"]


# script.py
# Author : Victor Greiner (2016)

import configparser
import time
import os
from math import pi
from run import simulation
import random

def ask(question, answers=['y', 'n']):
    """Ask a question to the user until the input result belongs to the answers list
    question : str
    answers : str list
    returns the chosen answer"""
    choice = input(question)
    while choice not in answers:
        choice = input(question)
    return choice

def get_standard_options():
    """Return a dictionary containing the default parameters"""
    initial_config = configparser.ConfigParser()
    initial_config.read("config.cfg")

    config = configparser.ConfigParser()
    config.read("config.cfg")
    options = config['Options']

    options['LogFileName'] = 'last.json'
    options['WeightFileName'] = 'weights.json'
    options['NeuronFileName'] = 'last.json'

    options['RealRobot'] = 'n'
    options['Description'] = 'No Description'
    options['HiddenNeuronNumber'] = '3'
    options['LearningStep'] = '0.05'
    options['ThetaShiftRatio'] = '10'
    options['Size'] = '4'
    options['Prediction'] = '1'
    options['ExperimentalTheta'] = '1'

    options["Gain"] = '1'
    options["RestrictThetaShift"] = '1'
    options["RestrictPropagation"] = '0'
    options["InvertRestriction"] = '0'
    options["NewSigmoid"] = '0'
    options["RandomRatio"] = '0'

    options['Verbose'] = 'n'

    options['SingleSimulation'] = '1'
    options['MaximumDuration'] = '150'
    options['Tick'] = '0.020'

    options['Learn'] = 'y'
    options['Load'] = 'n'

    options['FixedStartingPosition'] = '1'
    options['StartingPositionX'] = '1'
    options['StartingPositionY'] = '1'
    options['StartingPositionTheta'] = '0'

    options['FixedTargetPosition'] = '1'
    options['TargetPositionX'] = '0'
    options['TargetPositionY'] = '0'
    options['TargetPositionTheta'] = '0'

    options['StopAutomatically'] = 'y'
    options['CriterionPercent'] = '0.03'
    options['CriterionDuration'] = '10'

    return options

options = get_standard_options()

choices = ["first_lessons", "100_lessons", "100_lessons_old", "3_lessons", "9_lessons", "4_lessons", "4_lessons_tests", "18_lessons", "testxy"]
choice = ask("Which script do you want to launch ? "+str(choices)+" -> ", choices)

LogFileName = choice + "/logs"

if choice=="first_lessons": # 10 independant first lessons
    nmax = 10
    for i in range(nmax):
        options['LogFileName'] = LogFileName+str(i+1)+".json"
        print("Running simulation "+str(i+1)+"/"+str(nmax)+"...")
        simulation(options)
        time.sleep(1)

elif choice=="100_lessons": # 100 successive lessons
    nmax = 100
    options['StartingPositionX'] = '1'
    options['StartingPositionY'] = '1'
    options['StartingPositionTheta'] = '0'
    options['TargetPositionX'] = '0'
    options['TargetPositionY'] = '0'
    for i in range(nmax):
        options['LogFileName'] = LogFileName+str(i+1)+".json"
        print("Running simulation "+str(i+1)+"/"+str(nmax)+"...")
        simulation(options)
        time.sleep(1)
        options['Load'] = 'y'

elif choice=="100_lessons_old": # 100 successive lessons with the old gradient formula
    nmax = 100
    options['Prediction'] = '0'
    options['ExperimentalTheta'] = '1'
    for i in range(nmax):
        options['LogFileName'] = LogFileName+str(i+1)+".json"
        print("Running simulation "+str(i+1)+"/"+str(nmax)+"...")
        simulation(options)
        time.sleep(1)
        options['Load'] = 'y'

elif choice=="3_lessons": # 3 lessons

    options['ExperimentalTheta'] = '1'

    # First lesson :
    options['StartingPositionX'] = '2'
    options['StartingPositionY'] = '2'
    options['StartingPositionTheta'] = '0'
    options['LogFileName'] = "3_lessons/1.json"
    print("Running simulation 1/4...")
    simulation(options)
    time.sleep(1)

    # Second lesson :
    options['Load'] = 'y'
    options['LogFileName'] = "3_lessons/2.json"
    print("Running simulation 2/4...")
    simulation(options)
    time.sleep(1)

    # Third lesson :
    options['StartingPositionX'] = '2'
    options['StartingPositionY'] = '0'
    options['StartingPositionTheta'] = '3.14'
    options['LogFileName'] = "3_lessons/3.json"
    print("Running simulation 3/4...")
    simulation(options)
    time.sleep(1)

    # Test :
    options['Learn'] = 'n'
    options['StartingPositionX'] = '2'
    options['StartingPositionY'] = '0'
    options['StartingPositionTheta'] = '1.57'
    options['LogFileName'] = "3_lessons/4.json"
    print("Running simulation 4/4...")
    simulation(options)
    time.sleep(1)

elif choice=="4_lessons": # 4 lessons

    n = 0
    nmax = 5

    for x in [-1, 1]:
        for y in [-1, 1]:
            n += 1
            options['LogFileName'] = LogFileName+str(n)+".json"
            options['StartingPositionX'] = str(x)
            options['StartingPositionY'] = str(y)
            options['StartingPositionTheta'] = str(1.57)
            print("Running simulation "+str(n)+"/"+str(nmax)+"...")
            simulation(options)
            time.sleep(1)
            options['Load'] = 'y'

    # Test :
    options['LogFileName'] = LogFileName+str(5)+".json"
    options['Learn'] = 'n'
    options['StartingPositionX'] = '-1'
    options['StartingPositionY'] = '1'
    options['StartingPositionTheta'] = '1'
    print("Now testing")
    print("Running simulation "+str(nmax)+"/"+str(nmax)+"...")
    simulation(options)

elif choice=="4_lessons_tests": # 4 lessons avec 10 tests

    n = 0
    nmax = 4

    for x in [-1, 1]:
        for y in [-1, 1]:
            n += 1
            options['LogFileName'] = LogFileName+str(n)+".json"
            options['StartingPositionX'] = str(x)
            options['StartingPositionY'] = str(y)
            options['StartingPositionTheta'] = str(1.57)
            print("Running simulation "+str(n)+"/"+str(nmax)+"...")
            simulation(options)
            time.sleep(1)
            options['Load'] = 'y'

    # Test :
    for i in range (0, 10):
        x = random.randint(-1, 1)
        y = random.randint(-1, 1)
        n += 1
        options['LogFileName'] = LogFileName+str(n)+".json"
        options['Learn'] = 'n'
        options['StartingPositionX'] = str(x)
        options['StartingPositionY'] = str(y)
        options['StartingPositionTheta'] = '1'
        print("Running simulation "+str(nmax)+"/"+str(nmax)+"...")
        simulation(options)

elif choice=="9_lessons": # 9 lessons

    n = 0
    nmax = 10
    options['Size'] = '3'
    for x in [-2, 0, 2]:
        for y in [-2, 0, 2]:
            options['LogFileName'] = LogFileName+str(n)+".json"
            options['StartingPositionX'] = str(1)
            options['StartingPositionY'] = str(1)
            options['StartingPositionTheta'] = str(0)
            n += 1
            print("Running simulation "+str(n)+"/"+str(nmax)+"...")
            simulation(options)
            time.sleep(1)
            options['Load'] = 'y'

    # Test :
    options['LogFileName'] = LogFileName+str(5)+".json"
    options['Learn'] = 'n'
    options['StartingPositionX'] = '1'
    options['StartingPositionY'] = '1'
    options['StartingPositionTheta'] = '1'
    print("Running simulation "+str(nmax)+"/"+str(nmax)+"...")
    simulation(options)

elif choice=="18_lessons": # 18 lessons

    n = 0
    nmax = 19
    options['Size'] = '3'
    for i in range (0, 2) :
    	for x in [-2, 0, 2]:
        	for y in [-2, 0, 2]:
            		options['LogFileName'] = LogFileName+str(n)+".json"
            		options['StartingPositionX'] = str(x)
            		options['StartingPositionY'] = str(y)
            		options['StartingPositionTheta'] = str(0)
            		n += 1
            		print("Running simulation "+str(n)+"/"+str(nmax)+"...")
            		simulation(options)
            		time.sleep(1)
            		options['Load'] = 'y'

    # Test :
    options['LogFileName'] = LogFileName+str(5)+".json"
    options['Learn'] = 'n'
    options['StartingPositionX'] = '1'
    options['StartingPositionY'] = '1'
    options['StartingPositionTheta'] = '1'
    print("Running simulation "+str(nmax)+"/"+str(nmax)+"...")
    simulation(options)

elif choice=="testxy": # un run avec X = x et Y = y
    x = 1
    y = 1
    theta = 0
    options['LogFileName'] = LogFileName+str(1)+".json"
    options['TargetPositionX'] = str(x)
    options['TargetPositionY'] = str(y)
    options['TargetPositionTheta'] = str(theta)
    print("Running simulation ...")
    simulation(options)

else:
    print("Incorrect choice")


import importlib
import datetime
from geopy.geocoders import Nominatim
import requests
import urllib.parse
from time import sleep

api_url = 'https://beat-magazine-map.herokuapp.com'
beat_url = "http://www.beat.com.au/"

def PostPayload(objectId,payload, apiEndpoint):
    retry_max = 10
    retry_count = 0
    request_success = False
    this_endpoint = api_url + '/' + apiEndpoint + '/push/' + urllib.parse.quote_plus(objectId)
    while(request_success == False and retry_count < retry_max):
        try:
            r = requests.post(this_endpoint, json = payload)
            request_success = True
        except:
            print("Venue scrape - cannot find coordinates")
            retry_count += 1
            r = 'Failure'
            sleep(retry_count)
    return(r)

def GetAddressLatLon(address):
    retry_max = 10
    retry_count = 0
    request_success = False
    geolocator = Nominatim()
    returnObject = {
        "latitude":None,
        "longitude":None
    }
    while(request_success == False and retry_count < retry_max):
        try:
            location = geolocator.geocode(address)
            request_success = True
        except:
            print("Geolocator failed")
            retry_count += 1
            sleep(retry_count)
    if request_success == True:
        try:
            returnObject["latitude"] = location.latitude
            returnObject["longitude"] = location.longitude
        except:
            print("No lat lon for address " + address)
    return(returnObject)

bandcampScrape = importlib.import_module("bandcampScrape")
beatScrape = importlib.import_module("beatScrape")

#scrape a page of the gig guide to get gig links, gig genres

def scrapeBeatDay(date):
    allGigs = []
    #maintain an overall list of venues, headline artists and support artists so those only get scraped once
    allHeadlineArtist = []
    allSupportArtist=[]
    allVenue=[]
    this_link = beat_url + "gig-guide/" + date.strftime("%Y-%m-%d")
    print(this_link)
    gigGuideGigs = beatScrape.BeatGigGuideScrape(this_link)
    #loop through and get the gig details
    for i in range(len(gigGuideGigs)):
    #for i in range(1):
        gig_link = gigGuideGigs[i].get("gigLink")
        print(gig_link)
        gigDetails = beatScrape.BeatGigScrape(beat_url + gig_link)
        gigGuideGigs[i]["gigPrice"] = gigDetails.get("gigPrice")
        gigGuideGigs[i]["gigHeadlineArtist"] = gigDetails.get("gigHeadlineArtist")
        gigGuideGigs[i]["gigSupportArtist"] = gigDetails.get("gigSupportArtist")
        gigGuideGigs[i]["gigVenue"] = gigDetails.get("gigVenue")
        gigGuideGigs[i]["gigDatetime"] = gigDetails.get("gigDatetime")
        gigGuideGigs[i]["gigDate"] = this_date.strftime('%Y-%m-%d')
        #append to the overall arrays
        allHeadlineArtist.extend(gigDetails.get("gigHeadlineArtist"))
        allSupportArtist.extend(gigDetails.get("gigSupportArtist"))
        allVenue.append(gigDetails.get("gigVenue"))
        allGigs.extend(gigGuideGigs)
        #reduce the all lists to unique values
    allHeadlineArtist = list(set(allHeadlineArtist))
    allSupportArtist = list(set(allSupportArtist))
    allVenue = list(set(allVenue))

    allHeadlineArtist = [x for x in allHeadlineArtist if x is not None]
    allSupportArtist = [x for x in allSupportArtist if x is not None]
    allVenue = [x for x in allVenue if x is not None]
    #perhaps should check which of these are new before scraping??

    #maybe not, could be a chance to update reference information
    #only artists who have gigs get updated then... hmmm..

    #loop through the headline artists
    for i in range(len(allHeadlineArtist)):
    #for i in range(50,60):
        artistId = allHeadlineArtist[i]
        artistLinks = {}
        headlineArtistLink = beat_url + artistId
        print(artistId)
        beatArtist = beatScrape.BeatHeadlineArtistScrape(headlineArtistLink)
        try:
            bandcampArtist = bandcampScrape.BandcampBandSearch(beatArtist.get("artistName"))
            if bandcampArtist.get("bandcampLink") != None:
                artistLinks["bandcamp"] = {
                    "bandcampPage":bandcampArtist.get("bandcampLink"),
                    "bandcampTracks":bandcampArtist.get("bandcampTracks")
                }
        except:
            print("Bandcamp scrape failed")
        try:
            #append this new artist to the total payload
            payload = {
                "beatArtistType":"headline",
                "artistName":beatArtist.get("artistName"),
                "artistLinks":artistLinks
            }
            PostPayload(objectId = artistId,payload = payload, apiEndpoint = "artist")
        except:
            print("Artist " + artistId + " post failed")

    #loop through support artists
    for i in range(len(allSupportArtist)):
    #for i in range(50,60):
        artistId = allSupportArtist[i]
        artistLinks = {}
        headlineArtistLink = beat_url + artistId
        print(artistId)
        beatArtist = beatScrape.BeatHeadlineArtistScrape(headlineArtistLink)
        try:
            bandcampArtist = bandcampScrape.BandcampBandSearch(beatArtist.get("artistName"))
            if bandcampArtist.get("bandcampLink") != None:
                artistLinks["bandcamp"] = {
                    "bandcampPage":bandcampArtist.get("bandcampLink"),
                    "bandcampTracks":bandcampArtist.get("bandcampTracks")
                }
        except:
            print("Bandcamp scrape failed")
        try:
            #append this new artist to the total payload
            payload = {
                "beatArtistType":"headline",
                "artistName":beatArtist.get("artistName"),
                "artistLinks":artistLinks
            }
            PostPayload(objectId = artistId,payload = payload, apiEndpoint = "artist")
        except:
            print("Artist " + artistId + " post failed")

    #loop through venues
    for i in range(len(allVenue)):
        venueId = allVenue[i]
        venueUrl=beat_url + venueId
        print(venueId)
        venueDetails = beatScrape.BeatVenueScrape(venueUrl)
        #try to use venueLocation
        #otherwise try to use google
        venueLocation = GetAddressLatLon(venueDetails.get("venueAddress"))
        lat = venueLocation.get("latitude")
        lon = venueLocation.get("longitude")
        try:
            #append this new artist to the total payload
            payload = {
                "venueName":venueDetails.get("venueName"),
                "venueAddress":venueDetails.get("venueAddress"),
                "lat":lat,
                "lon":lon
            }
            PostPayload(objectId = venueId,payload = payload, apiEndpoint = "venue")
        except:
            print("Venue " + venueId + " post failed")

    #now finally post the gigs
    for i in range(len(allGigs)):
        thisGig = allGigs[i]
        gigId = thisGig.get("gigLink")
        print(gigId)
        try:
            payload = {
                "gigGenre":thisGig.get("gigGenre"),
                "gigDatetime":thisGig.get("gigDatetime"),
                "venueId":thisGig.get("gigVenue"),
                "headlineArtist":thisGig.get("gigHeadlineArtist"),
                "supportArtist":thisGig.get("gigSupportArtist"),
                "gigPrice":thisGig.get("gigPrice")
            }
            PostPayload(objectId = gigId,payload = payload, apiEndpoint = "gig")
        except:
            print("Gig " + gigId + " post failed")

#scrape between a date range
n_days = 14
start_date = datetime.datetime.now().date()
end_date = start_date + datetime.timedelta(n_days)

delta = end_date - start_date

for j in range(delta.days + 1):
    this_date = start_date + datetime.timedelta(j)
    scrapeBeatDay(this_date)


# Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
# with the License. A copy of the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys

from setuptools import find_packages, setup

def readme():
    """Read the README file and use it as long description."""
    with open(os.path.join(os.path.dirname(__file__), "README")) as f:
        return f.read()

VERSION = "2.2.1"
REQUIRES = ["boto3>=1.9.48,<=1.9.101", "future>=0.16.0,<=0.17.1", "tabulate>=0.8.2,<=0.8.3"]

if sys.version_info[:2] == (2, 6):
    # For python2.6 we have to require argparse since it
    # was not in stdlib until 2.7.
    REQUIRES.append("argparse==1.4.0")

if sys.version_info[0] == 2:
    REQUIRES.append("configparser>=3.5.0,<=3.5.3")

setup(
    name="aws-parallelcluster",
    version=VERSION,
    author="Amazon Web Services",
    description="AWS ParallelCluster is an AWS supported Open Source cluster management tool to deploy "
    "and manage HPC clusters in the AWS cloud.",
    url="https://github.com/aws/aws-parallelcluster",
    license="Apache License 2.0",
    packages=find_packages(),
    install_requires=REQUIRES,
    entry_points={
        "console_scripts": [
            "pcluster = pcluster.cli:main",
            "awsbqueues = awsbatch.awsbqueues:main",
            "awsbhosts = awsbatch.awsbhosts:main",
            "awsbstat = awsbatch.awsbstat:main",
            "awsbkill = awsbatch.awsbkill:main",
            "awsbsub = awsbatch.awsbsub:main",
            "awsbout = awsbatch.awsbout:main",
        ]
    },
    include_package_data=True,
    zip_safe=False,
    package_data={"": ["examples/config"]},
    long_description=readme(),
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Console",
        "Programming Language :: Python",
        "Topic :: Scientific/Engineering",
        "License :: OSI Approved :: Apache Software License",
    ],
)


import graphene
from datetime import date
from core.models import Company, Version
from helpers.tst_helper import GraphQLTestCase
from helpers.functions import to_dict
from helpers.graphene import filter_fields_args

class HelperFunctionsTest(GraphQLTestCase):

    @classmethod
    def setUpTestData(cls):
        company = Company.objects.create(shortname="X", name="X-Company")
        Version.objects.create(company=company, shortname="V1", reporting_date=date(2010, 12, 31))
        super().setUpTestData()

    def test_toDict_with_existing_instance(self):
        print(""" - to_dict() works as expected with existing model instance """)
        version = Version.objects.all()[0]
        expected = {
            "pk": version.id,
            "id": version.id,
            "shortname": "V1",
            "reporting_date": version.reporting_date,
            "company": version.company,
            "compare_version": None,
            "matching_version": None,
            "copy_version": None,
            "description": "",
            "archived": False,
            "locked": False,
            "created_at": version.created_at,
            "updated_at": version.updated_at
        }
        self.assertEqual(to_dict(version), expected)

class HelperGrapheneTest(GraphQLTestCase):

    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()

    def test_filterFieldsArgs(self):
        print(""" - filter_fields_args() works as expected """)
        filter_fields = {
            "company": {
                "field_type": graphene.String(),
                "filter": ["exact"]
            },
            "reporting_date__year": {
                "field_type": graphene.Int(),
                "filter": ["gt", "gte"]
            },
            "shortname": {
                "field_type": graphene.String(),
                "filter": ["icontains"]
            }
        }
        expected = {
            "company__exact": graphene.String(),
            "shortname__icontains": graphene.String(),
            "reporting_date__year__gt": graphene.Int(),
            "reporting_date__year__gte": graphene.Int()
        }
        self.assertEqual(filter_fields_args(filter_fields), expected)


from interference.clusters.processor import Processor
import numpy as np
from typing import Any, Dict, Sequence, Tuple
from scipy.spatial.distance import mahalanobis

class ClusterNode:

    def __init__(self, id, embedding: np.ndarray, initial_std: float, dimensions: int) -> None:

        self.id = id
        self.dimensions = dimensions

        self.cov_matrix = np.eye(dimensions)
        self.std = initial_std
        self.mean = embedding
        self.instances = [embedding]

        i_observation = embedding.reshape((dimensions, 1))

        self.observations = i_observation

    def add_embedding(self, embedding) -> None:

        self.instances.append(embedding)

        self.observations = np.hstack([self.observations, embedding.reshape((self.dimensions, 1))])

        self.cov_matrix = np.cov(self.observations)

        self.mean = np.mean(self.instances, axis=0)

        std_vector = np.std(self.instances, axis=0)

        self.std = np.linalg.norm(std_vector)

class CovarianceCluster(Processor):

    def __init__(self, dimensions: int, initial_std: float = 0.01) -> None:

        self.initial_std = initial_std
        self.tag_to_cluster: Dict[str, int] = {}
        self.id = 0
        self.clusters: Dict[int, ClusterNode] = {}
        self.dimensions = dimensions

    def add_to_cluster(self, tag: str, embedding: np.ndarray) -> None:

        id = -1

        if len(self.clusters) == 0:

            id = self._create_node(embedding)

        else:

            distance, node = self.brute_search(embedding)

            if distance < node.std:

                node.add_embedding(embedding)
                id = node.id

            else:

                id = self._create_node(embedding)

        self.tag_to_cluster[tag] = id

    def remove_from_cluster(self, tag: str) -> None:

        pass

    def stat_distance(self, embedding: np.ndarray, node: ClusterNode) -> float:

        return mahalanobis(embedding, node.mean, node.cov_matrix)

    def brute_search(self, embedding: np.ndarray) -> Tuple[float, ClusterNode]:

        nodes = list(self.clusters.values())

        curr_node = nodes[0]
        distance = self.stat_distance(embedding, nodes[0])

        for node in nodes[1:]:

            c_distance = self.stat_distance(embedding, node)

            if c_distance < distance:

                distance = c_distance
                curr_node = node

        return (distance, curr_node)

    def _create_node(self, embedding: np.ndarray) -> int:

        id = self.id
        self.id += 1

        new_node = ClusterNode(id, embedding, self.initial_std, self.dimensions)

        self.clusters[id] = new_node

        return id

    def process(self, tag: str, embedding: np.ndarray) -> None:

        self.add_to_cluster(tag, embedding)

    def update(self, tag: str, embedding: np.ndarray) -> None:

        self.remove(tag)

        self.process(tag, embedding)

    def remove(self, tag: str) -> None:

        self.remove_from_cluster(tag)

    def get_cluster_by_tag(self, tag: str) -> int:

        return self.tag_to_cluster[tag]

    def get_tags_in_cluster(self, cluster_id: int) -> Sequence[str]:

        return [tag for tag, id in self.tag_to_cluster.items() if id ==
                cluster_id]

    def get_cluster_ids(self) -> Sequence[int]:

        return [
            id for id
            in self.clusters.keys()
        ]

    def predict(self, embedding: np.ndarray) -> int:

        return self.brute_search(embedding)[1].id

    def describe(self) -> Dict[str, Any]:

        return {
            "name": "Covariance Cluster",
            "parameters": {
                "initial_std": self.initial_std
            }
        }

    def safe_file_name(self) -> str:

        return f"CovCluster = initial_std={self.initial_std}"


# -*- coding: utf-8 -*-
import logging

from scrapy.settings import Settings
from stocks import settings

__author__ = 'lzhao'
__date__ = '5/15/16'
__time__ = '11:14 PM'

logging.basicConfig(filename=None, level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s")

# logging.disable(logging.CRITICAL)

class emailSettings(Settings):
	def __init__(self):
		super(emailSettings, self).__init__()
		self.setmodule(settings, priority='default')


__author__ = 'Tania'

from datetime import datetime

def is_in_range(date, daterange):
    d1 = datetime.strptime(date[:10], "%Y/%m/%d")
    for i in range(len(daterange)):
        # dStart = datetime.strptime(daterange[i][0], "%Y-%m-%d")
        # dEnd = datetime.strptime(daterange[i][1], "%Y-%m-%d")
        dStart = daterange[i][0]
        dEnd = daterange[i][1]
        if dStart < d1 < dEnd:
            #print(daterange, i)
            return i
    return -1

def generate_log(filter, pacientes, file, center, decompensation_window_DC):
    filtro = filter
    bigLog = open(file, 'r')
    filteredLog = open('Log/' + filtro + '_' + str(decompensation_window_DC) + 'm_' + center + '.csv', 'w')
    filteredLog.write(bigLog.readline())
    for line in bigLog:
        tags = line.split(';')
        if int(tags[0]) in pacientes.keys():
            x = is_in_range(tags[3], pacientes[int(tags[0])][filtro])
            if x != -1:
                filteredLog.write(str(x) + '_' + line)


from django import forms
from django.contrib.auth import get_user_model
from django.contrib.auth.forms import UserCreationForm, UserChangeForm
from .models import Userhistory, Uservisitedreview

class CustomUserChangeForm(UserChangeForm):
    class Meta:
        model = get_user_model()
        fields = ['username', 'first_name', 'last_name', 'email']

class CustomUserCreationForm(UserCreationForm):
    rrn = forms.IntegerField(
        help_text='11자리를 입력하세요',
        label='주민등록번호',
        )

    class Meta:
        model = get_user_model()
        fields = ['username', 'rrn', 'age', 'nickname']
        exclude = ['age',]

class UserhistoryForm(forms.ModelForm):
    class Meta:
        model = Userhistory
        fields = '__all__'
        exclude = ['user', 'movie_pk']

class UservisitedreviewForm(forms.ModelForm):
    class Meta:
        model = Uservisitedreview
        fields = '__all__'
        exclude = ['user', 'review', 'visited_time']


import numpy as np
import matplotlib.pyplot as plt
import cv2

def crop_and_shrink(img_name, input_dir, output_dir):

    img = cv2.imread(input_dir+img_name+".png")
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    h, w = img.shape[0], img.shape[1]

    x = 118*2
    y = 124*2

    mid_h = h//2
    mid_w = w//2

    new = img[mid_h-2*x+75:mid_h+75,mid_w-y+50:mid_w+y+50]
    new = cv2.cvtColor(new, cv2.COLOR_RGB2BGR)
    new_lr = cv2.resize(new, (124,118))
    cv2.imwrite(output_dir+img_name+"cropped_lr.png",new_lr)


#!/usr/bin/env python
# coding: utf-8

from elasticsearch import Elasticsearch
from elasticsearch import helpers

# configure elasticsearch
es = Elasticsearch(httpCompress=True)
request_body = {
        "settings" : {
                "number_of_shards": 1,
                "number_of_replicas": 1
            },

            'mappings': {
                 'properties': {
                    'name': {'type': 'text'},
                    'metadata': {'type': 'text'},
                    'content': {'type': 'text'},
                  }}
        }
print("creating 'example_index' index...")
es.indices.create(index = 'index-pdf', body = request_body)


#!/usr/bin/env python3.6

from __future__ import print_function

import os
import shutil
import zipfile
import xlrd
from mako.template import Template

def extract_docx(extracted_folder, template):
    with zipfile.ZipFile(template, 'r') as zipObj:
        zipObj.extractall(extracted_folder)
    shutil.copy(f'{extracted_folder}/word/document.xml', 'document.xml')

def gen_docx(extracted_folder, app_no, row_tuple):
    s = '%.2f' % ((float(row_tuple[1]) - float(row_tuple[0])) * 1.68)
    if s == 0:
        return
    with open(f'document.xml', 'r') as document:
        content = document.read()
        t = Template(content)

    with open(f'{extracted_folder}/word/document.xml', 'w+') as document:
        document.write(t.render(
            app=app_no,
            data_water=row_tuple[1].replace(".0", ""),
            electricity=row_tuple[2].replace(".0", ""),
            plumb_water=row_tuple[3].replace(".0", ""),
            plumb_el=row_tuple[4].replace(".0", ""),
            counter_el=row_tuple[5].replace(".0", ""),
        ))

    shutil.make_archive(f'el-{app_no}.docx', 'zip', f'{extracted_folder}')
    shutil.copy(f'el-{app_no}.docx.zip', f'docx/el-{app_no}.docx')
    os.remove(f'el-{app_no}.docx.zip')

if __name__ == '__main__':
    template_name = 'contract_1.docx'
    folder = 'temp'
    try:
        extract_docx(folder, template_name)
        workbook = xlrd.open_workbook('data.xlsx')
        worksheet = workbook.sheet_by_index(0)
        for i in range(0, 90):
            gen_docx(
                folder,
                i + 1,
                [str(worksheet.cell(i, x).value).replace(',', '.') for x in range(0, 6)],
            )
    finally:
        shutil.rmtree(folder)
        os.remove('document.xml')

    template_name = 'contract_2.docx'
    folder = 'temp'
    try:
        extract_docx(folder, template_name)
        workbook = xlrd.open_workbook('data.xlsx')
        worksheet = workbook.sheet_by_index(0)
        for i in range(91, 180):
            gen_docx(
                folder,
                i + 1,
                [str(worksheet.cell(i, x).value).replace(',', '.') for x in range(0, 6)],
            )
    finally:
        shutil.rmtree(folder)
        os.remove('document.xml')


import os
import zlib
from itertools import chain
from itertools import repeat
from typing import Callable
from typing import cast
from typing import Generator
from typing import Optional
from typing import Tuple

import colorlog
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
from cryptography.hazmat.primitives.ciphers import Cipher
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.ciphers.modes import CTR
from cryptography.hazmat.primitives.hashes import SHA256
from cryptography.hazmat.primitives.hmac import HMAC

from backuppy.exceptions import BackupCorruptedError
from backuppy.io import IOIter
from backuppy.options import OptionsDict

logger = colorlog.getLogger(__name__)
AES_KEY_SIZE = 32    # 256 bits
AES_BLOCK_SIZE = 16  # 128 bits
RSA_KEY_SIZE = 512   # 4096 bits
RSA_KEY_SIZE_BITS = RSA_KEY_SIZE * 8

def identity(x: bytes) -> bytes:
    return x

def compress_and_encrypt(
    input_file: IOIter,
    output_file: IOIter,
    key_pair: Optional[bytes],
    options: OptionsDict,
) -> bytes:
    """ Read data from an open file descriptor, and write the compressed, encrypted data to another
    file descriptor; compute the HMAC of the encrypted data to ensure integrity

    :param input_file: an IOIter object to read plaintext data from
    :param output_file: an IOIter object to write compressed ciphertext to
    """
    key, nonce = (key_pair[:AES_KEY_SIZE], key_pair[AES_KEY_SIZE:]) if key_pair else (b'', b'')
    compressobj = zlib.compressobj()
    zip_fn: Callable[[bytes], bytes] = (  # type: ignore
        compressobj.compress if options['use_compression'] else identity
    )
    encrypt_fn: Callable[[bytes], bytes] = (
        Cipher(AES(key), CTR(nonce), backend=default_backend()).encryptor().update
        if options['use_encryption'] else identity
    )
    hmac = HMAC(key, SHA256(), default_backend())

    def last_block() -> Generator[Tuple[bytes, bool], None, None]:
        yield (compressobj.flush(), False) if options['use_compression'] else (b'', False)

    writer = output_file.writer(); next(writer)
    logger.debug2('starting to compress')
    for block, needs_compression in chain(zip(input_file.reader(), repeat(True)), last_block()):
        if needs_compression:
            block = zip_fn(block)
        logger.debug2(f'zip_fn returned {len(block)} bytes')
        block = encrypt_fn(block)
        logger.debug2(f'encrypt_fn returned {len(block)} bytes')
        if options['use_encryption']:
            hmac.update(block)
        writer.send(block)

    if options['use_encryption']:
        return hmac.finalize()
    else:
        return b''

def decrypt_and_unpack(
    input_file: IOIter,
    output_file: IOIter,
    key_pair: Optional[bytes],
    options: OptionsDict,
) -> None:
    """ Read encrypted, GZIPed data from an open file descriptor, and write the decoded data to
    another file descriptor; verify the HMAC of the encrypted data to ensure integrity

    :param input_file: an IOIter object to read compressed ciphertext from
    :param output_file: an IOIter object to write plaintext data to
    """
    key, nonce, signature = (
        key_pair[:AES_KEY_SIZE],
        key_pair[AES_KEY_SIZE:AES_KEY_SIZE + AES_BLOCK_SIZE],
        key_pair[AES_KEY_SIZE + AES_BLOCK_SIZE:]
    ) if key_pair else (b'', b'', b'')
    decrypted_data = b''
    decrypt_fn: Callable[[bytes], bytes] = (
        Cipher(AES(key), CTR(nonce), backend=default_backend()).decryptor().update
        if options['use_encryption'] else identity
    )
    decompress_obj = zlib.decompressobj()
    unzip_fn: Callable[[bytes], bytes] = (
        decompress_obj.decompress  # type: ignore
        if options['use_compression'] else identity
    )
    hmac = HMAC(key, SHA256(), default_backend())
    writer = output_file.writer(); next(writer)
    for encrypted_data in input_file.reader():
        if options['use_encryption']:
            hmac.update(encrypted_data)
        decrypted_data += decrypt_fn(encrypted_data)
        logger.debug2(f'decrypt_fn returned {len(decrypted_data)} bytes')

        block = unzip_fn(decrypted_data)
        logger.debug2(f'unzip_fn returned {len(block)} bytes')
        writer.send(block)
        decrypted_data = decompress_obj.unused_data

    # Decompress and write out the last block
    if decrypted_data:
        block = unzip_fn(decrypted_data)
        logger.debug2(f'unzip_fn returned {len(block)} bytes')
        writer.send(block)

    try:
        if options['use_encryption']:
            hmac.verify(signature)
    except InvalidSignature as e:
        raise BackupCorruptedError("The file's signature did not match the data") from e

def generate_key_pair(options: OptionsDict) -> bytes:
    if not options['use_encryption']:
        return b''
    return os.urandom(AES_KEY_SIZE + AES_BLOCK_SIZE)

def encrypt_and_sign(data: bytes, private_key_filename: str) -> bytes:
    """ Use an RSA private key to encrypt and sign some data

    :param data: the bytes to encrypt
    :param private_key_filename: the location of the RSA private key file in PEM format
    :returns: the encrypted data with signature appended
    """
    private_key = _get_key(private_key_filename)

    # the public key is used to encrypt, private key to decrypt
    encrypted_key_pair = private_key.public_key().encrypt(
        data,
        padding.OAEP(padding.MGF1(SHA256()), SHA256(), label=None),
    )
    # the _private_ key is used to sign, the public key to verify
    signature = private_key.sign(
        data,
        padding.PSS(padding.MGF1(SHA256()), padding.PSS.MAX_LENGTH),
        SHA256(),
    )
    return encrypted_key_pair + signature

def decrypt_and_verify(data: bytes, private_key_filename: str) -> bytes:
    """ Use an RSA private key to decrypt and verify some data

    :param data: encrypted data with a signature appended
    :param private_key_filename: the location of the RSA private key file in PEM format
    :returns: the unencrypted data
    :raises BackupCorruptedError: if the signature cannot be verified
    """

    private_key = _get_key(private_key_filename)
    message, signature = data[:RSA_KEY_SIZE], data[RSA_KEY_SIZE:]
    key_pair = private_key.decrypt(
        message,
        padding.OAEP(padding.MGF1(SHA256()), SHA256(), label=None),
    )
    try:
        private_key.public_key().verify(
            signature,
            key_pair,
            padding.PSS(padding.MGF1(SHA256()), padding.PSS.MAX_LENGTH),
            SHA256(),
        )
    except InvalidSignature as e:
        raise BackupCorruptedError('Could not decrypt archive') from e

    return key_pair

def _get_key(private_key_filename: str) -> RSAPrivateKey:
    with open(private_key_filename, 'rb') as priv_kf:
        private_key = cast(
            RSAPrivateKey,
            serialization.load_pem_private_key(priv_kf.read(), None, default_backend()),
        )

    if private_key.key_size != RSA_KEY_SIZE_BITS:
        raise ValueError(
            f'Backuppy requires a {RSA_KEY_SIZE_BITS}-bit private key, '
            f'this is {private_key.key_size} bits'
        )

    return private_key


import io
import os

# Imports the Google Cloud client library
from google.cloud import vision
from google.cloud.vision import types

#Perform VisionAI Image Recognition on Image BEFORE

# Instantiates a client
client = vision.ImageAnnotatorClient()

# The name of the image file to annotate
file_nameB = os.path.abspath('images/vegetables_before.jpg')

# Loads the image into memory
with io.open(file_nameB, 'rb') as image_file:
    content = image_file.read()

image = types.Image(content=content)

# Performs label detection on the image file
response = client.label_detection(image=image)
itemsBefore = response.label_annotations

#Repeat process for Image AFTER
# The name of the image file to annotate
file_nameA = os.path.abspath('images/vegetables_after.jpg')

# Loads the image into memory
with io.open(file_nameA, 'rb') as image_file:
    content = image_file.read()

image = types.Image(content=content)

# Performs label detection on the image file
response = client.label_detection(image=image)
itemsAfter = response.label_annotations

missingItems = [item for item in itemsBefore if item not in itemsAfter]
newItems = [item for item in itemsAfter if item not in itemsBefore]

print('Labels Before:')
for label in itemsBefore:
    print(label.description)

print('\n')
print('Labels After:')
for label in itemsAfter:
    print(label.description)

print('\n')

print('Items Missing:')
for label in missingItems:
    print(label.description)

print('\n')
print('Items Newly Added:')
for label in newItems:
    print(label.description)