#_*_coding:utf-8_*_

from django.shortcuts import  render,redirect

class BaseKingAdmin(object):
    list_display = []
    list_filters = []
    search_fields = []
    list_per_page = 20
    ordering = None
    filter_horizontal = []
    list_editable = []
    readonly_fields = []
    actions = ["delete_selected_objs",]
    readonly_table = False
    modelform_exclude_fields = []
    add_form = None

    def delete_selected_objs(self,request,querysets):
        app_name = self.model._meta.app_label
        model_name = self.model._meta.model_name
        if self.readonly_table:
            errors = {"readonly_table": "This table is readonly ,cannot be deleted or modified!" }
        else:
            errors = {}
        if request.POST.get("_delete_confirm") == "yes":
            if not self.readonly_table:
                querysets.delete()
            return redirect("/kingadmin/%s/%s/" % (app_name,model_name))
        selected_ids =  ','.join([str(i.id) for i in querysets])
        return render(request,"kingadmin/table_objs_delete.html",{"objs":querysets,
                                                              "admin_class":self,
                                                              "app_name": app_name,
                                                              "model_name": model_name,
                                                              "model_verbose_name": self.model._meta.verbose_name,
                                                              "selected_ids":selected_ids,
                                                              "admin_action":request._admin_action,
                                                              "errors":errors
                                                              })

    def default_form_validation(self):
        '''用户可以在此进行自定义的表单验证，相当于django form的clean方法'''
        pass

class AdminAlreadyRegistered(Exception):
    def __init__(self,msg):
        self.message = msg

class AdminSite(object):
    def __init__(self, name='admin'):
        self.enabled_admins = {}  # model_class class -> admin_class instance
        #self.default_actions = {'delete_selected': actions.delete_selected}
        #self._global_actions = self._actions.copy()

    def register(self,model_class,admin_class=None):
        if model_class._meta.app_label not in self.enabled_admins:
            self.enabled_admins[model_class._meta.app_label] = {} #enabled_admins['crm'] = {}
        # else:
        #     print(self.enabled_admins)
        #     raise AdminAlreadyRegistered("model %s has registered already"% model_class._meta.model_name)
        #admin_obj = admin_class()
        if not admin_class:#no custom admin class , use BaseAdmin
            admin_class = BaseKingAdmin()
        admin_class.model = model_class #绑定model 对象和admin 类

        self.enabled_admins[model_class._meta.app_label][model_class._meta.model_name] = admin_class
        #enabled_admins['app']['tablename'] = tableadmin

site = AdminSite()


import numpy as np
import sklearn.tree
import random

def read_file(filename):
     ## read X
     contexts = []
     d = 0
     fX = open(filename+".X", 'r')
     for line in fX:
         linesp = line.split()
         if d==0: d = len(linesp)
         else: assert(d==len(linesp))
         linesp = [float(x) for x in linesp]
         contexts.append(linesp)

     ## read Y
     labels = []
     all_class = []
     fY = open(filename+".Y", 'r')
     for line in fY:
         linesp = line.split()
         c = int(linesp[0])
         if c not in all_class:
             all_class.append(c)
         labels.append(c)
     K = len(all_class)
     assert(len(contexts)==len(labels))
     print("size=", len(contexts), " #feat=", d, " #class=", K)
     print(all_class)

     return contexts, labels

fn="data/Sensorless/Sensorless.scale"
X,Y = read_file(fn)

Z = list(zip(X, Y))
random.shuffle(Z)
X, Y=zip(*Z)

h = len(X)
tmp = 4*h//5
trainX = X[:tmp]
trainY = Y[:tmp]
testX = X[tmp:]
testY = Y[tmp:]

learning_alg = lambda: sklearn.tree.DecisionTreeClassifier(max_depth=5)

p=0
while np.power(2,p) < len(trainY):
   tmp_list = []
   for r in range(20):
      train_size = int(np.power(2,p))
      train_idx = np.random.choice(len(trainX), train_size, replace=False)
      train_setX = [trainX[i] for i in train_idx]
      train_setY = [trainY[i] for i in train_idx]
      pred = learning_alg()
      pred.fit(train_setX, train_setY)
      result = pred.predict(testX).astype(int)
      accu = 0
      for j in range(len(result)):
         if result[j]==testY[j]:
            accu+=1
      #print(accu/len(result))
      tmp_list.append(accu/len(result))
   print("2^"+str(p), np.mean(tmp_list))
   p+=1


#!/usr/bin/env python
# $Id$
# -*- coding: utf-8

import time
import tempfile
from functools import reduce
import numpy

from pyscf.lib import logger
from pyscf import ao2mo

'''
spin-adapted MP2
t2[i,j,b,a] = (ia|jb) / D_ij^ab
'''

# the MO integral for MP2 is (ov|ov). The most efficient integral
# transformation is
# (ij|kl) => (ij|ol) => (ol|ij) => (ol|oj) => (ol|ov) => (ov|ov)
#   or    => (ij|ol) => (oj|ol) => (oj|ov) => (ov|ov)

def kernel(mp, mo_energy, mo_coeff, nocc, verbose=None):
    nvir = len(mo_energy) - nocc
    eia = mo_energy[:nocc,None] - mo_energy[None,nocc:]
    t2 = numpy.empty((nocc,nocc,nvir,nvir))
    emp2 = 0

    with mp.ao2mo(mo_coeff, nocc) as ovov:
        for i in range(nocc):
            djba = (eia.reshape(-1,1) + eia[i].reshape(1,-1)).ravel()
            gi = numpy.array(ovov[i*nvir:(i+1)*nvir], copy=False)
            gi = gi.reshape(nvir,nocc,nvir).transpose(1,2,0)
            t2[i] = (gi.ravel()/djba).reshape(nocc,nvir,nvir)
            # 2*ijab-ijba
            theta = gi*2 - gi.transpose(0,2,1)
            emp2 += numpy.einsum('jab,jab', t2[i], theta)

    return emp2, t2

def make_rdm1(mp, mo_energy, mo_coeff, nocc, verbose=None):
    nmo = len(mo_energy)
    nvir = nmo - nocc
    dm1occ = numpy.zeros((nocc,nocc))
    dm1vir = numpy.zeros((nvir,nvir))
    eia = mo_energy[:nocc,None] - mo_energy[None,nocc:]
    emp2 = 0
    with mp.ao2mo(mo_coeff, nocc) as ovov:
        for i in range(nocc):
            djba = (eia.reshape(-1,1) + eia[i].reshape(1,-1)).ravel()
            gi = numpy.array(ovov[i*nvir:(i+1)*nvir]).reshape(nvir,nocc,nvir)
            gi = gi.transpose(1,2,0)
            t2i = (gi.ravel()/djba).reshape(nocc,nvir,nvir)
            # 2*ijab-ijba
            theta = gi*2 - gi.transpose(0,2,1)
            emp2 += numpy.einsum('jab,jab', t2i, theta)

            dm1vir += numpy.einsum('jca,jcb->ab', t2i, t2i) * 2 \
                    - numpy.einsum('jca,jbc->ab', t2i, t2i)
            dm1occ += numpy.einsum('iab,jab->ij', t2i, t2i) * 2 \
                    - numpy.einsum('iab,jba->ij', t2i, t2i)

    rdm1 = numpy.zeros((nmo,nmo))
# *2 for beta electron
    rdm1[:nocc,:nocc] =-dm1occ * 2
    rdm1[nocc:,nocc:] = dm1vir * 2
    rdm1 = reduce(numpy.dot, (mo_coeff, rdm1, mo_coeff.T))
    return rdm1

class MP2(object):
    def __init__(self, mf):
        self.mol = mf.mol
        self._scf = mf
        self.verbose = self.mol.verbose
        self.stdout = self.mol.stdout
        self.max_memory = mf.max_memory

        self.emp2 = None
        self.t2 = None

    def kernel(self, mo_energy=None, mo_coeff=None, nocc=None):
        if mo_coeff is None:
            mo_coeff = self._scf.mo_coeff
        if mo_energy is None:
            mo_energy = self._scf.mo_energy
        if nocc is None:
            nocc = self.mol.nelectron // 2

        self.emp2, self.t2 = \
                kernel(self, mo_energy, mo_coeff, nocc, verbose=self.verbose)
        logger.log(self, 'RMP2 energy = %.15g', self.emp2)
        return self.emp2, self.t2

    # return eri_ovov array[nocc*nvir,nocc*nvir]
    def ao2mo(self, mo_coeff, nocc):
        log = logger.Logger(self.stdout, self.verbose)
        time0 = (time.clock(), time.time())
        log.debug('transform (ia|jb)')
        nmo = mo_coeff.shape[1]
        nvir = nmo - nocc
        co = mo_coeff[:,:nocc]
        cv = mo_coeff[:,nocc:]
        if self._scf._eri is not None and \
           (nocc*nvir*nmo**2/2*8 + (nocc*nvir)**2*8 \
            + self._scf._eri.nbytes)/1e6 < self.max_memory:
            eri = ao2mo.incore.general(self._scf._eri, (co,cv,co,cv))
        else:
            erifile = tempfile.NamedTemporaryFile()
            ao2mo.outcore.general(self.mol, (co,cv,co,cv), erifile.name,
                                  verbose=self.verbose)
            eri = erifile
        time1 = log.timer('Integral transformation', *time0)
        return ao2mo.load(eri)

if __name__ == '__main__':
    from pyscf import scf
    from pyscf import gto
    mol = gto.Mole()
    mol.verbose = 5
    mol.output = 'out_h2o'
    mol.atom = [
        [8 , (0. , 0.     , 0.)],
        [1 , (0. , -0.757 , 0.587)],
        [1 , (0. , 0.757  , 0.587)]]

    mol.basis = {'H': 'cc-pvdz',
                 'O': 'cc-pvdz',}
    mol.build()
    mf = scf.RHF(mol)
    print(mf.scf())

    nocc = mol.nelectron//2
    nmo = mf.mo_energy.size
    nvir = nmo - nocc

    co = mf.mo_coeff[:,:nocc]
    cv = mf.mo_coeff[:,nocc:]
    g = ao2mo.incore.general(mf._eri, (co,cv,co,cv)).ravel()
    eia = mf.mo_energy[:nocc,None] - mf.mo_energy[nocc:]
    t2ref0 = g/(eia.reshape(-1,1)+eia.reshape(-1)).ravel()
    t2ref0 = t2ref0.reshape(nocc,nvir,nocc,nvir).transpose(0,2,3,1)

    pt = MP2(mf)
    emp2, t2 = pt.kernel()
    print(emp2 - -0.204019967288338)
    print('incore', numpy.allclose(t2, t2ref0))
    pt.max_memory = 1
    print('direct', numpy.allclose(pt.kernel()[1], t2ref0))

    t2s = numpy.zeros((nocc*2,nocc*2,nvir*2,nvir*2))
    t2s[ ::2, ::2, ::2, ::2] = t2ref0 - t2ref0.transpose(0,1,3,2)
    t2s[1::2,1::2,1::2,1::2] = t2ref0 - t2ref0.transpose(0,1,3,2)
    t2s[ ::2,1::2,1::2, ::2] = t2ref0
    t2s[1::2, ::2, ::2,1::2] = t2ref0
    t2s[ ::2,1::2, ::2,1::2] = -t2ref0.transpose(0,1,3,2)
    t2s[1::2, ::2,1::2, ::2] = -t2ref0.transpose(0,1,3,2)
    dm1occ =-.5 * numpy.einsum('ikab,jkab->ij', t2s, t2s)
    dm1vir = .5 * numpy.einsum('ijac,ijbc->ab', t2s, t2s)
    dm1ref = numpy.zeros((nmo,nmo))
    dm1ref[:nocc,:nocc] = dm1occ[ ::2, ::2]+dm1occ[1::2,1::2]
    dm1ref[nocc:,nocc:] = dm1vir[ ::2, ::2]+dm1vir[1::2,1::2]
    dm1ref = reduce(numpy.dot, (mf.mo_coeff, dm1ref, mf.mo_coeff.T))
    rdm1 = make_rdm1(pt, mf.mo_energy, mf.mo_coeff, nocc)
    print(numpy.allclose(rdm1, dm1ref))


import psutil
import cleanvideos
import cleanfiles
import cleanaudio
import cleanall
import time
import sys
import os

print("ATTENZIONE: Leggere per bene il file help.txt!\n\nCosa devo fare?\n1. Eliminare File\n2. Eliminare Video\n3. Eliminare Audio\n4. Eliminare File, video e audio\n5. Terminare App in esecuzione")
try:
    scelta = int(input("Numero: "))
except:
    print("Devi inserire un valore numerico di tipo intero!")

if scelta == 1:
    path = input("Quale PATH devo scansionare? : ")
    cleanfiles.filechecker(path)
elif scelta == 2:
    path = input("Quale PATH devo scansionare? : ")
    cleanvideos.filechecker(path)
elif scelta == 3:
    path = input("Quale PATH devo scansionare? : ")
    cleanaudio.filechecker(path)
elif scelta == 4:
    path = input("Quale PATH devo scansionare? : ")
    cleanall.filechecker(path)
elif scelta == 5:
    app_chiuse = 0
    rimuovo = input("Desideri terminare ogni applicazione in esecuzione?\n(Ricordo che termina tutto che utilizza la CPU al di sopra dell'1)\ny/n: ")
    if rimuovo.lower() == 'y':
        for processo in psutil.process_iter():
            processName = processo.name()
            processID = processo.pid
            processCPU = processo.memory_percent()
            f = open("tm.txt", "a")
            f.write(f"\n{processName} : {processID}, CPU: {processCPU}")
            f.close()
            if processCPU > 1.0:
                f = open("attivita_chiuse", "a")
                f.write(f"\n{processName} ha {processCPU} di CPU, attività terminata!")
                f.close()
                print("Aspetta qualche secondo...")
                try:
                    processo.kill()
                except:
                    print("Permesso Negato!")
                app_chiuse+=1
    print(f"Terminate {app_chiuse} attività\nApri il file 'attivita_chiuse.txt' per vedere i processi terminati")

else:
    print("Non ho capito bene! Chiudi e riprova!")


# -*- coding: utf-8 -*-
from datetime import datetime

import pytest
from twilio.rest.exceptions import TwilioRestException

from ..models import *
from ..tasks import *

@pytest.mark.django_db
class TestTasks:
    def test_send_recipient_ok(self, recipients):
        # test sending via recipient
        recipient_send_message_task(recipients['calvin'].id,
                                    "This is a test",
                                    None,
                                    'test')

    def test_send_recipient_blacklist(self, recipients):
        recipient_send_message_task(recipients['john_owen'].id,  # doesn't actually test blacklisted number handling :(
                                    "This is a test to a blacklisted number...",
                                    None,
                                    'test')

    def test_send_fail_number(self, recipients):
        with pytest.raises(TwilioRestException):
            recipient_send_message_task(recipients['thomas_chalmers'].id,
                                        "This is a test to a number that will fail",
                                        None,
                                        'test')

    def test_send_group(self):
        # test sending via group
        group_send_message_task("This is another test",
                                "Test group",
                                'test',
                                eta=None)

    def test_import_incoming_sms_task(self):
        import_incoming_sms_task()

    def test_import_outgoing_sms_task(self):
        import_outgoing_sms_task()

    def test_check_log_consistent(self):
        with pytest.raises(TwilioRestException):
            check_log_consistent('0')

    def test_check_outgoing_log_consistent(self):
        with pytest.raises(TwilioRestException):
            check_recent_outgoing_log('0')

    def test_send_keyword_digest(self, keywords, smsin, users):
        send_keyword_digest()
        assert Keyword.objects.get(keyword='test').last_email_sent_time is not None
        send_keyword_digest()
        # add a new sms:
        sms = SmsInbound.objects.create(content='test message',
                                        time_received=timezone.now(),
                                        sender_name="John Calvin",
                                        sender_num="+447927401749",
                                        matched_keyword="test",
                                        sid='123fasdfdfaw45')
        sms.save()
        keywords['test'].subscribed_to_digest.add(users['staff'])
        keywords['test'].save()
        send_keyword_digest()

    def test_log_msg_in(self, recipients):
        calvin = recipients['calvin']
        p = {'Body': 'New test message',
             'MessageSid': 'thisisreallyauuid',
             'From': calvin.number}
        log_msg_in(p, datetime.now(), calvin.pk)

        assert SmsInbound.objects.filter(content="New test message").count() == 1

    def test_warn_on_blacklist_receipt(self, recipients):
        warn_on_blacklist_receipt(recipients['wesley'].pk, 'stop')


import re
import uuid
from base64 import b64decode
from django import forms
from django.forms import ValidationError
from django.utils.translation import ugettext as _
from django.core.files.base import ContentFile
from django.core.files.uploadedfile import UploadedFile
from snowpenguin.django.recaptcha2.fields import ReCaptchaField
from snowpenguin.django.recaptcha2.widgets import ReCaptchaWidget
from .models import Questions, Participants, Locations

IMAGE_TYPE_EXTENSION_MAP = {
    'png': 'png',
    'jpeg': 'jpg',
}

BASE64_CONTENT_PATTERN = re.compile(r'^data:(?P<content_type>image/(?P<type>{}));base64,(?P<data>.*)$'.format(
    r'|'.join(IMAGE_TYPE_EXTENSION_MAP.keys())))

class RegistrationForm(forms.ModelForm):
    photo_raw = forms.CharField(max_length=8589934592, widget=forms.HiddenInput, required=False)

    #captcha = ReCaptchaField(label='', required=True, widget=ReCaptchaWidget())

    locations = forms.ModelMultipleChoiceField(
        queryset=Locations.objects.all(),
        widget=forms.CheckboxSelectMultiple,
        required=True
    )

    class Meta:
        model = Participants
        fields = ['photo', 'email', 'first_name', 'last_name', 'city', 'phone', 'birthday', 'locations', 'view']
        exclude = ['subdomain']

    def __init__(self, *args, **kwargs):
        extra = kwargs.pop('extra')
        super(RegistrationForm, self).__init__(*args, **kwargs)

        for i, question in enumerate(extra):
            if question.type == Questions.TYPE_STRING:
                self.fields['question_%s' % question.id] = forms.CharField(
                    label=question.get_question(),
                    required=False)

            if question.type == Questions.TYPE_TEXTAREA:
                self.fields['question_%s' % question.id] = forms.CharField(
                    label=question.get_question(),
                    required=False,
                    widget=forms.Textarea)

            if question.type == Questions.TYPE_BOOLEAN or question.type == Questions.TYPE_CHECKEN:
                self.fields['question_%s' % question.id] = forms.BooleanField(
                    label=question.get_question(),
                    required=False)

    def extra_answers(self):
        for name, value in self.cleaned_data.items():
            if name.startswith('question_'):
                yield (name[-36:], value, self.fields[name].label)

    def clean_photo_raw(self):
        data = self.cleaned_data['photo_raw']
        if not data:
            return data

        match = BASE64_CONTENT_PATTERN.match(data)
        if not match:
            raise ValidationError(_("Not a valid image file."))

        content_type = match.group('content_type')
        data = b64decode(match.group('data'))
        filename = uuid.uuid4()
        extension = IMAGE_TYPE_EXTENSION_MAP.get(match.group('type'))
        filename = '{}.{}'.format(filename, extension)

        content_file = ContentFile(data, name=filename)
        uploaded_file = UploadedFile(content_file, filename, content_type, size=content_file.size)

        return forms.ImageField().to_python(uploaded_file)

    def clean_email(self):
        email = self.cleaned_data['email']

        if Participants.objects.filter(email=email, subdomain=self.subdomain).exists():
            raise ValidationError('Такой адрес электронной почты уже существует')

        return email

    def clean(self):
        cleaned_data = super(RegistrationForm, self).clean()

        photo = cleaned_data.get('photo')
        photo_raw = cleaned_data.get('photo_raw')

        if photo_raw:
            cleaned_data['photo'] = photo_raw
        #elif not photo:
            #raise ValidationError(_('File-picker image or drag-and-drop image is required.'))

        return cleaned_data


import pytest

from lib import calculate_score

class Repo:
    def __init__(self, forks_count, watchers_count, stargazers_count, open_issues_count, updated_at, created_at, description):
        self.forks_count = forks_count
        self.watchers_count = watchers_count
        self.stargazers_count = stargazers_count
        self.open_issues_count = open_issues_count
        self.updated_at = updated_at
        self.created_at = created_at
        self.description = description
        self._InnerSourceMetadata = {
            'participation': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
            'motivation': "",
            'guidelines': ""}

@pytest.fixture
def empty_repo():
    return Repo(0, 0, 0, 0, "2020-04-17T16:09:26Z", "2020-01-27T09:32:01Z", "Description")

@pytest.fixture
def small_activity_repo():
    return Repo(1, 1, 1, 1, "2020-04-18T16:09:26Z", "2020-01-25T09:32:01Z", "Description")

def test_calculate_score_empty(empty_repo):
    """
    Test calculate_actvitiy if no data is available
    """
    assert calculate_score(empty_repo) == 0

def test_calculate_score_small(small_activity_repo):
    """
    Test calculate_actvitiy if no data is available
    """
    assert calculate_score(small_activity_repo) == 7


#!/usr/bin/env python
import ConfigParser
import PIL
import hashlib
import sys
import urllib2
import urllib
import sys
import os
from PIL import Image
from django.conf import settings

SMALL_WIDTH = 200
BIG_WIDTH = 360

BASEFOLDER = settings.PRODUCT_IMAGE_FOLDER

def writeImages(image):
	# finding hashed file name
	filename = hashlib.md5( image ).hexdigest()

	# filename for both image sizes
	originalImageFile = BASEFOLDER + str(filename) + '-orig.jpg'
	smallImageFile = BASEFOLDER + str(filename) + '-small.jpg'
	bigImageFile = BASEFOLDER + str(filename) + '-big.jpg'
	cropImageFile = BASEFOLDER + str(filename) + '-crop.jpg'
	thumbImageFile = BASEFOLDER + str(filename) + '-thumb.jpg'
	# Store tiny image for small thumbnails
	tinyImageFile = BASEFOLDER + str(filename) + '-tiny.jpg'

	try:
		with open( originalImageFile ) as fb:
			with open( bigImageFile ) as fb:
				with open( smallImageFile ) as fb:
					with open( cropImageFile ) as fb:
						with open( thumbImageFile ) as fb:
							with open( tinyImageFile ) as fb:
								img = Image.open(originalImageFile)
								origWidth = float(img.size[0])
								if (origWidth > BIG_WIDTH and validAspect(image)):
									return 1
								else:
									return 0
	except IOError as e:
		f = open( originalImageFile, 'wb')
		f.write( urllib.urlopen( image ).read() )
		f.close()

		# Store image for prodList page
		img = Image.open(originalImageFile)
		origWidth = float(img.size[0])

		basewidth = SMALL_WIDTH # define the fixed width
		wpercent = (basewidth / float(img.size[0]))
		hsize = int((float(img.size[1]) * float(wpercent))) # find height which preserves the aspect ratio
		img = img.resize((basewidth, hsize), PIL.Image.ANTIALIAS)

		if img.mode != "RGB":
			img = img.convert("RGB")
		# Writing image to resized file
		img.save(smallImageFile)

		# Store image for product page
		img = Image.open(originalImageFile)

		basewidth = BIG_WIDTH # define the fixed width
		wpercent = (basewidth / float(img.size[0]))
		hsize = int((float(img.size[1]) * float(wpercent))) # find height which preserves the aspect ratio
		img = img.resize((basewidth, hsize), PIL.Image.ANTIALIAS)
		if img.mode != "RGB":
			img = img.convert("RGB")
		# Writing image to resized file
		img.save(bigImageFile)

		# save cropped image
		cropOrigImage(image)

		# check aspect ratio to determine validity

		if (origWidth > BIG_WIDTH and validAspect(image) ):
			return 1
		else:
			return 0

def validAspect(image):
	# finding hashed file name
	filename = hashlib.md5( image ).hexdigest()

	# filename for both image sizes
	bigImageFile = BASEFOLDER + str(filename) + '-big.jpg'

	# Store image for prodList page
	img = Image.open(bigImageFile)
	aspect = float(img.size[0])/img.size[1]
	return ( aspect < 1.6 and aspect > 0.625 )

def cropOrigImage(image):
	# finding hashed file name
	filename = hashlib.md5( image ).hexdigest()

	# filename for both image sizes
	bigImageFile = BASEFOLDER + str(filename) + '-orig.jpg'
	# Store image for prodList page
	cropImageFile = BASEFOLDER + str(filename) + '-crop.jpg'
	# Store image for mobile/tablet page
	thumbImageFile = BASEFOLDER + str(filename) + '-thumb.jpg'
	# Store tiny image for small thumbnails
	tinyImageFile = BASEFOLDER + str(filename) + '-tiny.jpg'

	img = Image.open(bigImageFile)

	width = img.size[0]
	height = img.size[1]
	if width > height:
		diff = width-height
		img = img.crop((diff/2, 0, (width-diff/2), height))
	else:
		diff = height-width
		img = img.crop((0, diff/2, width, (height-diff/2)))
	img.thumbnail((360,360),PIL.Image.ANTIALIAS)
	if img.mode != "RGB":
		img = img.convert("RGB")
	# Writing image to resized file
	img.save(cropImageFile)

	img.thumbnail((240,240),PIL.Image.ANTIALIAS)
	if img.mode != "RGB":
		img = img.convert("RGB")
	# Writing image to resized file
	img.save(thumbImageFile)

	img.thumbnail((60,60),PIL.Image.ANTIALIAS)
	if img.mode != "RGB":
		img = img.convert("RGB")
	img.save(tinyImageFile)


import sip
sip.setapi('QString', 2)
sip.setapi('QVariant', 2)

import sys, os, images_rc, util

from odbcview import ODBCView
from switchplu import SwitchPLU

from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyQt4.QtSql import *

MSL = util.msale_dir()

QUERY_DIR = 'c:/RSS/SavedQueries'

try:
    os.makedirs(QUERY_DIR)
except OSError:
    pass

class MainWindow(QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()

        ## main window setup
        self.setMinimumSize(1100, 700)
        self.setWindowTitle("Database Viewer")
        self.centralWidget = QWidget()
        self.setCentralWidget(self.centralWidget)

        ## layout settings
        self.layout = QVBoxLayout()
        gridLayout = QGridLayout()
        self.layout.addLayout(gridLayout)
        self.centralWidget.setLayout(self.layout)

        ## widgets
        self.tableList = QComboBox(self)
        self.clearQueryButton = QPushButton("Clear Queries")

        self.submitButton = QPushButton("Submit")
        self.submitButton.clicked.connect(self.submitChanges)
        self.submitButton.setDisabled(True)
        self.revertButton = QPushButton("Revert")
        self.revertButton.clicked.connect(self.revertChanges)
        self.revertButton.setDisabled(True)

        ## Edit Type selector. On Row Change and On Field Change are automatic updates
        ## whereas On Manual Submit will cache changes until Submit is clicked.
        self.editType = QComboBox()
        self.editType.addItems([ 'On Row Change', 'On Field Change', 'On Manual Submit' ])
        self.editType.activated.connect(self.changeMode)
        self.editType.setDisabled(True)
        self.dbTable = QTableView(self)
        self.tableList.activated.connect(self.loadTable)

        self.clearQueryButton.clicked.connect(self.clearQuery)
        self.dbTable.setEditTriggers(QAbstractItemView.NoEditTriggers)

        gridLayout.addWidget(QLabel("Tables"), 0, 0, 1, 7)
        gridLayout.addWidget(QLabel("Edit Table"), 2, 0, 1, 1)
        gridLayout.addWidget(QLabel("Update Type"), 4, 1, 1, 4)
        gridLayout.addWidget(self.editType, 4, 0, 1, 4)
        gridLayout.addWidget(self.submitButton, 6, 0, 1, 4)
        gridLayout.addWidget(self.revertButton, 7, 0, 1, 4)
        gridLayout.addWidget(self.tableList, 1, 0, 1, 7)
        gridLayout.addWidget(self.clearQueryButton, 2, 0, 1, 4)
        gridLayout.addWidget(self.dbTable, 0, 8, 50, 50)

        self.currentDb = None

        self.fillMenuBar()
        self.statusBar()

        if len(sys.argv) >= 2:
            self.accessDB(' '.join(sys.argv[1:]))

    def fillMenuBar(self):
        fileMenu = self.menuBar().addMenu("File")
        openAction = QAction("Open", self)
        openAction.setShortcut(QKeySequence.Open)
        openAction.triggered.connect(self.openDB)
        fileMenu.addAction(openAction)

        editMenu = self.menuBar().addMenu("Edit")
        self.editAction = QAction("Edit Table", self)
        self.editAction.triggered.connect(self.changeMode)
        self.editAction.setCheckable(True)
        editMenu.addAction(self.editAction)

        toolsMenu = self.menuBar().addMenu("Tools")
        queryCreatorAction = QAction("Query Creator", self)
        queryCreatorAction.triggered.connect(self.queryCreator)
        toolsMenu.addAction(queryCreatorAction)

        helpMenu = self.menuBar().addMenu("Help")
        aboutAction = QAction("About", self)
        aboutAction.triggered.connect(self.about)
        helpMenu.addAction(aboutAction)

        switchPLUAct = QAction("Switch PLU", self)
        switchPLUAct.triggered.connect(self.launchSwitchPLU)
        toolsMenu.addAction(switchPLUAct)

    def about(self):
        QMessageBox.information(self, 'About', 'BAKING SODA!', QMessageBox.Ok)
        return

    def launchSwitchPLU(self):
        sp = SwitchPLU(self)
        sp.setAttribute(Qt.WA_DeleteOnClose)
        sp.exec_()

    def toggleEditButtons(self, on_off):
        self.submitButton.setEnabled(on_off)
        self.revertButton.setEnabled(on_off)

    def backupPrompt(self, state):

        if state == Qt.Checked:
            if self.editType.currentIndex() == 2:
                self.toggleEditButtons(False)
            self.editType.setDisabled(False)
        elif state == Qt.Unchecked:
            self.toggleEditButtons(True)
            self.editType.setDisabled(True)

    def submitChanges(self):

        pass

    def revertChanges(self):

        pass

    def changeMode(self, on_off):

        if self.currentDb == None:
            self.editAction.setChecked(False)
            self.statusBar().showMessage("You must open a database before you can edit a table.")
            qDebug("changeMode() open a database first")
        else:
            if self.editAction.isChecked():
                qDebug("changeMode() do you want to edit?")
                reply = QMessageBox.question(self, 'Edit Table', 'Are you sure you want to edit this table?', QMessageBox.Yes | QMessageBox.No)

                if reply == QMessageBox.Yes:
                    self.editAction.setChecked(True)
                    self.toggleEditButtons(on_off)
                    self.dbTable.setEditTriggers(QAbstractItemView.DoubleClicked)
                    qDebug("changeMode() turned editing on")
                else:
                    qDebug("changeMode() canceled editing prompt")
                    self.editAction.setChecked(False)
            else:
                self.editAction.setChecked(False)
                self.toggleEditButtons(on_off)
                self.dbTable.setEditTriggers(QAbstractItemView.NoEditTriggers)
                qDebug("changeMode() turned editing off")

    def openDB(self):

        filename = QFileDialog.getOpenFileName(self, "Open Access Database", MSL, "Microsoft Access (*.mdb *.accdb);;Micro$ale Compacted Database (*.old *.bak)")
        if filename:
            self.accessDB(filename)

    def accessDB(self, dbpath):

        if self.currentDb is not None:
            self.currentDb.close()

        db = QSqlDatabase.addDatabase('QODBC', dbpath)
        db.setDatabaseName(util.odbc_conn_str + dbpath)
        self.currentDb = db

        if not db.open():
            QMessageBox.critical(self, dbpath, "Could not open database: %s" % db.lastError().text())
            return

        self.tableList.clear()
        #self.tableList.addItems(db.tables())

        for tbl in db.tables():
            query = QSqlQuery('SELECT COUNT(*) FROM [%s]' % tbl, self.currentDb)
            query.next()

            self.tableList.addItem('%s (%s)' % (tbl, query.value(0)), tbl)

    def loadTable(self, index):

        qDebug("loadTable() #%d" % index)
        if self.currentDb == None:
            self.statusBar().showMessage("Open a database first before loading tables.")

        tblName = self.tableList.itemData(self.tableList.currentIndex())
        #tblName = self.tableList.currentText()

        qDebug("loadTable() loading table '%s'" % tblName)
        self.model = ODBCView(self)
        self.model.setQuery("SELECT * FROM [%s]" % tblName, self.currentDb)

        if self.model.lastError().isValid():
            QMessageBox.critical(self, "Error", self.model.lastError().text())

        self.statusBar().showMessage("%d records in table." % self.model.rowCount())

        #if not self.model.select():
        #    QMessageBox.critical(self, "Error", self.model.lastError().text())

        self.dbTable.setModel(self.model)
        self.dbTable.show()

        #qDebug("loadTable() select statement: %s" % self.model.selectStatement())

        self.dbTable.resizeColumnsToContents()
        self.dbTable.horizontalHeader().setStretchLastSection(True)
        self.dbTable.show()

    def loadQueries(self):

        ''' load all SQL queries into memory, and add them
        to the combo box with the file contents as the item data.
        we could run into problems if there are several gigs of
        queries to load. tested loading 1,888 queries without delay '''

        self.savedCombo.clear()

        savePath = QDir(QUERY_DIR)

        savePath.setNameFilters(['*.sql'])

        saved = savePath.entryList(QDir.Files)

        self.savedCombo.addItem('', '')
        failedToLoad = [ ]
        for filename in saved:
            try:
                with open(os.path.join(QUERY_DIR, filename), 'rb') as sql:
                    self.savedCombo.addItem(filename, sql.read())
            except IOError:
                ## this should never happen
                failedToLoad.append(filename)

        if failedToLoad:
            ## this should never happen
            QMessageBox.critical(self, "No such file or directory.", "Failed to load %d queries:

%s" % (len(failedToLoad), '
'.join(failedToLoad)))

    def readQuery(self, index):

        ''' Sets the query box to the item data of the combo box index '''

        self.queryBox.setPlainText(self.savedCombo.itemData(index))

    def queryCreator(self):

        self.queryDialog = QDialog(self)
        self.queryDialog.setWindowTitle("Query Creator")
        self.queryDialog.setFixedSize(500, 250)

        layout = QVBoxLayout()
        gridLayout = QGridLayout()
        self.queryDialog.setLayout(layout)

        self.savedCombo = QComboBox()
        self.queryBox = QTextEdit()

        clearButton = QPushButton("Clear")
        clearButton.clicked.connect(self.clearQuery)

        saveButton = QPushButton("Save")
        saveButton.clicked.connect(self.saveQuery)

        saveAsButton = QPushButton("Save As")
        saveAsButton.clicked.connect(self.saveQueryAs)

        deleteButton = QPushButton("Delete")
        deleteButton.clicked.connect(self.deleteQuery)

        executeButton = QPushButton("Execute")
        executeButton.clicked.connect(self.runQuery)

        gridLayout.addWidget(clearButton, 0, 0, 1, 4)
        gridLayout.addWidget(saveButton, 1, 0, 1, 4)
        gridLayout.addWidget(saveAsButton, 2, 0, 1, 4)
        gridLayout.addWidget(deleteButton, 3, 0, 1, 4)
        gridLayout.addWidget(executeButton, 4, 0, 1, 4)
        gridLayout.addWidget(self.savedCombo, 0, 5, 1, 15)
        gridLayout.addWidget(self.queryBox, 1, 5, 10, 15)

        layout.addLayout(gridLayout)

        self.loadQueries()

        self.savedCombo.activated.connect(self.readQuery)

        self.queryDialog.setAttribute(Qt.WA_DeleteOnClose)

        self.queryDialog.exec_()

    def clearQuery(self):

        ''' Reloads current table and index. Also clears out the queryBox if necessary. '''

        index = self.tableList.currentIndex()
        self.loadTable(index)

        if self.savedCombo:
            self.queryBox.clear()

    def saveQuery(self):

        ''' Saves changes to the currently selected query. '''

        index = self.savedCombo.currentIndex()
        saveFile = self.savedCombo.itemText(index)
        data = self.queryBox.toPlainText()

        self.savedCombo.setItemData(index, data)

        with open(os.path.join(QUERY_DIR, saveFile), 'wb') as fh:

            fh.write(self.queryBox.toPlainText())

    def saveQueryAs(self):

        ''' Saves text in TextEdit as a .sql file, reloads all the .sql files
        in QUERY_DIR, then sets the current index to the file name that was just
        saved. Have to use findText because there's no telling where it ends up. '''

        saveFile = QFileDialog.getSaveFileName(self, 'Save File', QUERY_DIR, '.sql')

        if not saveFile:
            self.queryDialog.raise_()
            return

        index = self.savedCombo.currentIndex()

        with open(saveFile, 'wb') as fh:

            fh.write(self.queryBox.toPlainText())

        self.loadQueries()

        self.queryDialog.raise_()

        self.savedCombo.setCurrentIndex(self.savedCombo.findText(os.path.basename(saveFile)))

    def deleteQuery(self):

        ''' Makes sure you're not attempting to delete the empty list item,
        removes the file from the disk, removes the item from the ComboBox,
        and removes the text from the TextEdit '''

        currentQuery = self.savedCombo.currentIndex()
        if currentQuery != 0:
            try:
                os.unlink(os.path.join(QUERY_DIR, self.savedCombo.currentText()))
            except WindowsError:
                pass
                ## file has already been deleted, silently ignore.

            self.loadQueries()

            self.queryBox.clear()

    def runQuery(self):

        if not self.currentDb:
            self.statusBar().showMessage("Must connect to a database first.")
            return

        model = QSqlQueryModel(self)
        model.setQuery(self.queryBox.toPlainText(), self.currentDb)

        self.dbTable.setModel(model)
        self.dbTable.resizeColumnsToContents()
        self.dbTable.horizontalHeader().setStretchLastSection(True)
        self.dbTable.show()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    app.setWindowIcon(QIcon(':/images/logo.png'))
    mw = MainWindow()
    mw.show()
    sys.exit(app.exec_())


"""
В это пакете находятся вызываемые страницы.
Страницы создаются один раз при загрузке игры.
Страницы обновляются по мере необходимости.
"""
from __future__ import annotations

from ui.pages.StartPage import StartPage
from ui.pages.LevelSelect import LevelSelect
from ui.pages.GameMenuPage import GameMenuPage
from ui.pages.character_page.CharacterPage import CharacterPage
from ui.pages.inventory_page.InventoryPage import InventoryPage
from ui.pages.BackGorundPage import BackGorundPage
from ui.pages.ReadmePage import ReadmePage
from ui.pages.SettingsPage import SettingsPage
from ui.pages.WidgetFactory import WidgetFactory

from typing import TYPE_CHECKING
if TYPE_CHECKING:
    from ui.core.GameRootNode import GameRootNode
    from ui.pages import AbstractPage

class GamePages(object):
    """docstring for pages."""
    def __init__(self):
        super(GamePages, self).__init__()
        self.gameRoot:GameRootNode = None
        # this is current page
        self.focusState = False
        self.focus = False
        # init pages
        self.pages = {}
        self.page: AbstractPage = None
        self.gameMenu: GameMenuPage = None
        self.background: BackGorundPage = None
        self.startPage: StartPage = None
        self.levelSelect: LevelSelect = None
        self.readme: ReadmePage = None
        self.characterPage: CharacterPage = None
        self.settigsPage: SettingsPage = None
        self.visiblePage = True

    def setGameRoot(self, gameRoot):
        self.gameRoot = gameRoot
        self.gameRoot.gamePages = self

    def setUpPages(self):
        self.setUpBackgrounPage()
        self.setUpGameMenu()
        self.gameRoot.scene.addItem(self.toolTip)
        self.gameRoot.scene.addItem(self.notify)
        self.setUpInventoryPage()
        self.gameRoot.scene.removeItem(self.startPage)
        self.gameRoot.scene.addItem(self.startPage)
        self.gameRoot.scene.removeItem(self.startPage.mainWidget)
        self.gameRoot.scene.addItem(self.startPage.mainWidget)

    def destroyPages(self):
        pages = list(self.pages.values())[:]
        keys = list(self.pages.keys())[:]
        i =0
        for page in pages:
            if not page.isService:
                page.destroy()
                print(keys[i], '-- destroy')
                if not page.scene() is None:
                    self.gameRoot.scene.removeItem(page)
                del self.pages[keys[i]]
            i += 1
        self.gameMenu = None
        self.gameRoot.scene.removeItem(self.toolTip)
        self.gameRoot.scene.removeItem(self.notify)

    def setUpBackgrounPage(self):
        self.background = self.buildPage('background', BackGorundPage)
        self.gameRoot.scene.addItem(self.background)

    def setUpStartPage(self, ui):
        self.startPage = self.buildPage('startPage', StartPage)
        self.page = self.startPage
        self.toolTip = WidgetFactory.getToolTip(self.gameRoot, w = 128, h = 0, opacity=1.)
        self.toolTip.gameRoot = self.gameRoot
        self.gameRoot.view.wheel_change.connect(self.toolTip.wheel_change)
        self.notify = WidgetFactory.getNotifyText(self.gameRoot)

    def setUpLevelsPage(self):
        self.levelSelect = self.buildPage('levelSelect', LevelSelect)
        self.gameRoot.controller.mousePress.connect(self.levelSelect.mousePressEvent)

    def setUpGameMenu(self):
        gameMenu = self.buildPage('gameMenu', GameMenuPage)
        self.gameRoot.scene.addItem(gameMenu)
        gameMenu.resized()
        self.gameMenu = gameMenu

    def setUpCharecterPage(self):
        characterPage = self.buildPage('characterPage', CharacterPage)
        self.gameRoot.scene.addItem(characterPage)
        characterPage.showPage()

    def setUpInventoryPage(self):
        inventoryPage = self.buildPage('inventoryPage', InventoryPage)
        self.gameRoot.scene.addItem(inventoryPage)
        inventoryPage.hidePage()
        # self.gameRoot.controller.mouseMove.connect(inventoryPage.mouseMoveEvent)

    def setUpReadmePage(self):
        self.readme = self.buildPage('readmePage', ReadmePage)

    def setUpSettingsPage(self):
        self.settigsPage = self.buildPage('settigsPage', SettingsPage)

    def buildPage(self, pageName, pageClass):
        print('Init page:', pageName)
        page = pageClass(self)
        self.pages[pageName] = page
        self.gameRoot.view.keyPress.connect(page.keyPressEvent)
        page.focusable.connect(self.setFocus)
        page.setUpGui()
        return page

    def mouseMoveEvent(self, e):
        # self.checkFocus(e.pos())
        self.page.mouseMoveEvent(e)
        pass

    def mousePressEvent(self, e):
        # self.page.mousePressEvent(e)
        pass

    def resized(self):
        for page in self.pages.values():
            page.resized()

    def setCharacter(self, character):
        self.characterPage.character = character

    @property
    def isGamePage(self):
        if self.startPage.state:
            return True
        if self.levelSelect.state:
            return True
        if self.settigsPage.state:
            return True
        if not self.pages.get('chaPage') is None:
            if self.pages.get('chaPage').state:
                return True
        if not self.pages.get('gameMenu') is None:
            if self.pages.get('gameMenu').isFocus():
                return True

    def setFocus(self, isFocus):
        self.focusState = isFocus

    @property
    def isFocus(self):
        return self.focusState or self.gameMenu.isFocus()


#!/usr/bin/env python
# coding=utf-8

import re
import sys
import json
config = json.load(open(sys.argv[1]))

from pyspark import SparkContext
sc = SparkContext(appName = '%s-ginf-edgelist' % config['username'])
sc.addPyFile('./ginf/helpers.py')
from helpers import safe_json_loads, safeget

# --
# Helper

def extract(x, mode='gnip'):
    if mode == 'gnip':
        return (
            (
                re.sub('tag:search.twitter.com,', '', safeget(x, 'id')),
                re.sub('id:twitter.com:', '', safeget(x, 'actor.id')),
            ),
            safeget(x, 'twitter_entities.user_mentions')
        )
    elif mode == 'twitter':
        return (
            (
                safeget(x, 'id_str'),
                safeget(x, 'user.id_str'),
            ),
            safeget(x, 'entities.user_mentions')
        )
    elif mode == 'instagram':
        return (
            (
                '-',
                safeget(x, 'user.id'),
            ),
            safeget(x, 'users_in_photo')
        )
    else:
        raise Exception('unknown extraction mode')

def pluck(x, mode='gnip'):
    if mode == 'gnip' or mode =='twitter':
        return [str(y['id']) for y in x]
    elif mode == 'instagram':
        return [str(y['user']['id']) for y in x]
    else:
        raise Exception('unknown pluck mode')

# --
# Run

''' Read and parse json '''
data = sc.textFile(config['input'])\
    .filter(lambda x: x != '')\
    .map(safe_json_loads)\
    .filter(lambda x: x != None)

'''
    Get mentions
    Remove tweets w/ no mentions
    Flatten list of mentions
    Remove self mentions
'''
edges = data.map(lambda x: extract(x, mode=config['input_format']))\
    .filter(lambda x: len(x[1]) > 0)\
    .flatMapValues(lambda x: pluck(x, mode=config['input_format']))\
    .filter(lambda x: x[0][1] != x[1])

''' Format and write '''
edges.map(lambda x: '\t'.join([x[0][0], x[0][1], x[1]]))\
    .saveAsTextFile(path=config['edgelist'])

# --
# Results

'''

To see how many mentions:
sc.textFile(outpath).count()

sc.textFile(outpath).map(lambda x: x.split('\t')[1]).distinct().count()

gnip      : 1,033,218,365
gnip-loop : 1,053,833,965
uncharted : 51,160,180

'''


#!/usr/bin/python
# -*- coding: utf-8 -*-
######################################################################################################################
#  Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.                                           #
#                                                                                #
#  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance    #
#  with the License. A copy of the License is located at                                                             #
#                                                                                #
#      http://www.apache.org/licenses/LICENSE-2.0                                                                    #
#                                                                                #
#  or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES #
#  OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions    #
#  and limitations under the License.                                                                                #
######################################################################################################################

# This file reads the AWS cloudwatch metrics for a given workspace
# This is where we will change the algorithm to determine billing preference

import boto3
import botocore
import logging
import time
from botocore.config import Config
from botocore.exceptions import ClientError

botoConfig = Config(
    max_pool_connections=100,
    retries={'max_attempts': 20}
)

log = logging.getLogger()

class MetricsHelper(object):

    def __init__(self, region):
        self.maxRetries = 20
        self.region = region
        self.client = boto3.client('cloudwatch', region_name=self.region, config=botoConfig)

        return

    '''
    returns int
    '''

    def get_billable_time(self, workspaceID, startTime, endTime):

        log.debug('getMetricStatistics')
        try:
            metrics = self.client.get_metric_statistics(
                Dimensions=[{
                    'Name': 'WorkspaceId',
                    'Value': workspaceID
                }],
                Namespace='AWS/WorkSpaces',
                MetricName='UserConnected',
                StartTime=startTime,
                EndTime=endTime,
                Period=3600,
                Statistics=['Maximum']
            )
            log.debug(metrics)
        except botocore.exceptions.ClientError as e:
            log.error(e)
            raise

        billable_time = 0
        log.info('METRICS for WS:%s:', workspaceID)
        for metric in metrics['Datapoints']:
            if metric['Maximum'] >= 1:
                billable_time += 1
                log.info('METRIC %d -> %s', billable_time, metric)

        return int(billable_time)


# -*- coding: utf-8 -*-
from __future__ import print_function
from grab.spider import Spider, Task

from decimal import Decimal

from .models import Rating, Feedback, Piano_Lesson
import time

# HEADER
headers = {
    'User-Agent':  'Mozilla/5.0 (Windows NT 6.1; rv:12.0) \
Gecko/20100101 Firefox/12.0',
    'Accept':  'text/html,application/xhtml+xml,\
application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language':  'ru-ru,ru;q=0.8,en-us;q=0.5,en;q=0.3',
    'Accept-Encoding':  'gzip, deflate',
    'Connection':  'keep-alive',
}

XPATH_LIST = u'//div[contains(@class, "Tozw4c-WsjYwc kloNzc maQuO")]'
ITEM_URL = u'//a[@data-tee-test="title"]/@href'
PAGECONTROL = '//a[@data-tee-event="pagecontrol-right"]'

SEARCH_ITEM = {
    'name': '//h1[@data-tee-test="listing-details-title"]',
    'about_helpout': '//div[@data-tee-test="listing-description-present"]',
    'about_provider': '//div[@data-tee-test="listing-qualification-present"]',
    'price': '//div[contains(@data-tee-test,"listing-price")]',
    'image_link': '//div[contains(@class, "HiaYvf-Q4BLdf")]/img/@src',
}

SEARCH_RAITING = {
    'raiting_exist': '//div[@data-tee-test="rating-histogram"]',
    'raiting_table': '//div[@data-tee-test="rating-histogram"]/table/tr',
}

SEARCH_FEEDBACK = {
    'feedback_exist': '//div[@data-tee-test="rating-feedback-block"]',
    'feedback_iter': '//div[@data-tee-test="rating-feedback-block"]',
    'feedback_name': '//*[contains(@class, "oQLbGe-gElRsf")]'
}

class GetPianoInfo(Spider):

    piano_lesson = []

    def task_initial(self, grab, task):
        """
            Get article link
        """
        for url in grab.xpath_list(ITEM_URL):
            time.sleep(1)
            yield Task('info', url=url)
        if grab.doc.select(PAGECONTROL).exists():
            yield Task('initial', grab.xpath(PAGECONTROL).get('href'))

    def task_info(self, grab, task):
        idict = {
            'url': task.url
        }
        for k, v in SEARCH_ITEM.items():
            if grab.doc.select(v).exists():
                idict[k] = grab.doc.select(v).text()

        try:
            pl = Piano_Lesson.objects.get(url=idict['url'])
        except Piano_Lesson.DoesNotExist:
            pl = Piano_Lesson.objects.create(**idict)
            if grab.doc.select(SEARCH_RAITING['raiting_exist']).exists():
                for item in grab.doc.select(SEARCH_RAITING['raiting_table']):
                    Rating.objects.create(
                        star=item.select('td[1]').text(),
                        count=item.select('td[3]').text(),
                        piano_lesson=pl
                    )
            if grab.doc.select(SEARCH_FEEDBACK['feedback_exist']).exists():
                for item in grab.doc.select(SEARCH_FEEDBACK['feedback_iter']):
                    Feedback.objects.create(
                        name=item.select(
                            SEARCH_FEEDBACK['feedback_name']).text(),
                        piano_lesson=pl
                    )


import logging

logger = logging.getLogger(__name__)

def logger_init(logfile):
    logger.setLevel(logging.INFO)

    # Log file
    if logfile is not None:
        fh = logging.FileHandler(logfile)
        fh.setLevel(logging.INFO)

    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)

    # create formatter and add it to the handlers
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if logfile is not None:
        fh.setFormatter(formatter)
        logger.addHandler(fh)

logger_init(None)


import re
import sqlite3
from os import path
from scrapy import log
from scrapy import signals
from scrapy.exceptions import DropItem
from scrapy.xlib.pydispatch import dispatcher
from scrapy.contrib.pipeline.files import FilesPipeline
from scrapy.http import Request
from scraper.items import ApkItem

# Stores the APK information in the database
class SQLiteStorePipeline(object):
    filename = '../Evolution of Android Applications.sqlite'

    def __init__(self):
        self.conn = None
        dispatcher.connect(self.initialize, signals.engine_started)
        dispatcher.connect(self.finalize, signals.engine_stopped)

    # Tries to insert the APK file's information into the database.
    # If an error occurs or the APK file is a duplicate, the APK file
    # is not downloaded and the APK file's information is not inserted
    # into the database.
    def process_item(self, item, spider):
        try:
            self.conn.execute('INSERT INTO ApkInformation (Name, Version, Rating, DatePublished, FileSize, NumberOfDownloads, URL, Genre, OSSupported, Developer) VALUES(?,?,?,?,?,?,?,?,?,?)', (item['name'], item['software_version'],item['score'], item['date_published'], item['file_size'], item['num_downloads'], item['url'], item['genre'], item['operating_systems'], item['developer']))
            return item
        except Exception as e:
            raise DropItem('%s <%s>' % (e.message, item['url']))

    def initialize(self):
        if path.exists(self.filename):
            self.conn = sqlite3.connect(self.filename)
        else:
            log.msg('File does not exist: %s' % self.filename, level=log.ERROR)

    def finalize(self):
        if self.conn is not None:
            self.conn.commit()
            self.conn.close()
            self.conn = None

# Names the downloaded file in the format APK-Name_Version.apk
class APKFilesPipeline(FilesPipeline):
    def file_name(self, item):
        name = item['name'].replace('/', '_')
        return re.sub('\s+', '-', name + '_' + item['software_version'])

    def get_media_requests(self, item, info):
        return [Request(x, meta={'file_name': self.file_name(item)}) for x in item.get(self.FILES_URLS_FIELD, [])]

    def file_path(self, request, response=None, info=None):
        media_ext = path.splitext(request.url)[1]

        # For Google Play spider, which yields something along the lines of ".apk?h=syrPj2oViqBMGpbX5XEB7g&t=1396043020"
        if len(media_ext) > 4 and media_ext.startswith('.apk'):
            media_ext = media_ext[:4]

        # Ignore any files that are not APK files
        elif media_ext != '.apk':
            raise DropItem('File is not an APK file: %s' % request.url)

        return 'full/%s%s' % (request.meta['file_name'], media_ext)


""" functions for tracking the command-line argument vector
"""
import os
import sys
import logging
from argparse import ArgumentParser
from argparse import ArgumentDefaultsHelpFormatter
from itertools import chain
from .argspec import specifier_key
from .argspec import specifier_value_mapping
from .argspec import interpret_specifier
from .argspec import specifier_from_kernel
from .argspec import set_specifier_keyword_value
from .argspec_kernels import SUBCMD as SUBCMD_K
from .argspec_kernels import PREFIX as PREFIX_K
from .argspec_kernels import LOG_NAME as LOG_NAME_K
from .argspec_kernels import LOG_LEVEL as LOG_LEVEL_K
from .argspec_kernels import PRINT_OUT as PRINT_OUT_K

PREFIX = specifier_from_kernel(PREFIX_K, opt_char='P')
LOG_NAME = specifier_from_kernel(LOG_NAME_K, opt_char='L', out=True)
LOG_LEVEL = specifier_from_kernel(LOG_LEVEL_K, opt_char='V')
PRINT_OUT = specifier_from_kernel(PRINT_OUT_K, opt_char='p')

def tracker(argv, pos):
    """ argument vector tracker
    """
    return (argv, pos)

def make_tracker(argv):
    """ make an argument vector tracker from sys.argv
    """
    argv[0] = os.path.basename(argv[0])
    return tracker(argv=argv, pos=0)

def increment_tracker(argt):
    """ advance the position of the argument vector tracker
    """
    argv, pos = argt
    pos += 1
    return tracker(argv=argv, pos=pos)

def call_name(argt):
    """ current call name
    """
    argv, pos = argt
    cmd_str = ' '.join(argv[:pos])
    return cmd_str

def has_arguments_left(argt):
    """ is this tracker at the end?
    """
    argv, pos = argt
    return pos < len(argv)

def current_position_argument(argt):
    """ current argument in the tracker
    """
    argv, pos = argt
    return argv[pos]

def last_subcommand(argt):
    """ get the last subcommand in the argument tracker
    """
    argv, pos = argt
    return argv[pos-1]

def log_file_name(argt):
    """ get a file name, based on the last subcommand of the argument tracker
    """
    base = last_subcommand(argt)
    fname = '{:s}.{:s}'.format(base, 'log')
    return fname

def current_argument_vector(argt):
    """ current command arguments
    """
    argv, pos = argt
    return argv[pos:]

def parse_current_argument(argt, spec):
    """ parse the argument at the current tracker position
    """
    cmd = call_name(argt)
    specs = [spec]
    par = _parser(cmd, specs)

    if (not has_arguments_left(argt) or
            current_position_argument(argt) in ('-h', '--help')):
        _quit_with_help_message(par)

    argv = [current_position_argument(argt)]
    val, = _parse_values(par, specs, argv)
    return val

def call_subcommand(argt, subcmds):
    """ parse argument to call appropriate subcommand function
    """
    argt = increment_tracker(argt)
    subcmd_keys, _ = zip(*subcmds)
    subcmd_key_sp = specifier_from_kernel(SUBCMD_K, allowed_values=subcmd_keys)
    subcmd_key_val = parse_current_argument(argt, subcmd_key_sp)
    subcmd_fnc = dict(subcmds)[subcmd_key_val]
    subcmd_fnc(argt)

def parse_arguments(argt, specs):
    """ parse the argument tracker according to specifications
    """
    cmd = call_name(argt)
    par = _parser(cmd, specs)

    argv = current_argument_vector(argt)

    if '-h' in argv or '--help' in argv:
        _quit_with_help_message(par)

    val_maps = tuple(map(specifier_value_mapping, specs))
    vals = _parse_values(par, specs, argv)
    vals = tuple(val_map(val) for val_map, val in zip(val_maps, vals))
    return vals

def call_routine(argt, routine, specs):
    """ parse remaining arguments to invoke a routine
    """
    argt = increment_tracker(argt)
    fname = log_file_name(argt)
    log_name_spec = set_specifier_keyword_value(LOG_NAME, 'default', fname)
    specs = tuple(chain(specs, (PREFIX, log_name_spec, LOG_LEVEL, PRINT_OUT)))
    vals = parse_arguments(argt, specs)
    routine_args, context_vals = vals[:-4], vals[-4:]

    prefix, log_name, log_level, print_out = context_vals

    calling_dir = os.getcwd()
    routine_dir = os.path.abspath(prefix)
    if not os.path.exists(routine_dir):
        os.mkdir(routine_dir)

    os.chdir(routine_dir)
    routine(*routine_args, logger=_logger(log_name, log_level, print_out))
    os.chdir(calling_dir)

def _quit_with_help_message(par):
    par.print_help()
    par.exit()

def _parse_values(par, specs, argv):
    par_dct = vars(par.parse_args(argv))
    keys = tuple(map(specifier_key, specs))
    vals = tuple(map(par_dct.__getitem__, keys))
    return vals

def _parser(cmd, specs):
    par = ArgumentParser(
        prog=cmd,
        formatter_class=ArgumentDefaultsHelpFormatter,
        add_help=False
    )
    for args, kwargs in map(interpret_specifier, specs):
        par.add_argument(*args, **kwargs)
    return par

def _logger(log_name, log_level, print_out):
    logger = logging.getLogger()
    logger.setLevel(log_level)

    fhandler = logging.FileHandler(log_name, mode='w')
    fhandler.setLevel(log_level)

    formatter = logging.Formatter('%(message)s')
    fhandler.setFormatter(formatter)

    logger.addHandler(fhandler)

    if print_out:
        shandler = logging.StreamHandler(sys.stdout)
        shandler.setLevel(log_level)
        shandler.setFormatter(formatter)
        logger.addHandler(shandler)

    return logger


from PySide2.QtWidgets import QApplication, QPushButton, QGridLayout, QWidget, QOpenGLWidget, QMainWindow
from OpenGL.GL import *

import sys

class GLWidget(QOpenGLWidget):

    def initializeGL(self):
        self.f = self.context().currentContext().functions()
        self.shader = self.context().currentContext().functions().Shaders

        self.f.glViewport(self.x(), self.y(), self.width(), self.height())
        self.f.glClear(GL_COLOR_BUFFER_BIT)

        filestream = open('vertex_program.glsl')
        program_vertex = str(filestream.readlines())

        filestream.close()

        filestream = open('fragment_program.glsl')
        program_fragment = str(filestream.readlines())

        self.shader.glCreateShader(program_vertex)
        glShaderSource(program_vertex,)

    def paintGL(self):
        self.f.glClear(GL_COLOR_BUFFER_BIT)

class MainScreen(QMainWindow):
    def switch_flag(self):
        self.flag = not self.flag

    def __init__(self, qt_app: QApplication):
        super(MainScreen, self).__init__()

        self.setGeometry(
            qt_app.primaryScreen().geometry().width() / 2 - qt_app.primaryScreen().geometry().width() / 100 * 40 / 2,
            qt_app.primaryScreen().geometry().height() / 2 - qt_app.primaryScreen().geometry().height() / 100 * 40 / 2,
            qt_app.primaryScreen().geometry().width() / 100 * 40,
            qt_app.primaryScreen().geometry().height() / 100 * 40
                         )

        self.gl_widget = GLWidget()

        self.flag = True
        self.stop_go_button = QPushButton("Stop")

        self.stop_go_button.clicked.connect(self.switch_flag)

        self.the_layout = QGridLayout()
        self.the_layout.addWidget(self.gl_widget, 1, 1)
        self.the_layout.addWidget(self.stop_go_button, 2, 1)

        self.center_screen = QWidget()
        self.center_screen.setLayout(self.the_layout)

        self.setCentralWidget(self.center_screen)

if __name__ == "__main__":
    app = QApplication(sys.argv)

    window = MainScreen(app)
    window.show()

    app.exec_()


# HTACG HTML Tidy Conan package
# Dmitriy Vetutnev, Odant 2020

from conans import ConanFile, CMake, tools

class ConanPackage(ConanFile):
    name = "tidy"
    version = "5.7.47+1"
    license = "https://raw.githubusercontent.com/htacg/tidy-html5/5.7.28/README/LICENSE.md"
    description = "libtidy is the library version of HTML Tidy. "
    url = "https://github.com/odant/conan-tidy"
    settings = {
        "os": ["Windows", "Linux"],
        "compiler": ["Visual Studio", "gcc"],
        "build_type": ["Debug", "Release"],
        "arch": ["x86", "x86_64", "mips", "armv7"]
    }
    options = {
        "fPIC": [True, False],
        "ninja": [True, False]
    }
    default_options = {
        "fPIC": True,
        "ninja": True
    }
    generators = "cmake"
    exports_sources = "src/*", "CMakeLists.txt", "rm_pkgconfig.patch", "rm_install_pdb.patch", "FindTidy.cmake"
    no_copy_source = True
    build_policy = "missing"

    def configure(self):
        if self.settings.os == "Windows":
            del self.options.fPIC

    def build_requirements(self):
        if self.options.ninja:
            self.build_requires("ninja/[>=1.9.0]")

    def source(self):
        tools.patch(patch_file="rm_pkgconfig.patch")
        tools.patch(patch_file="rm_install_pdb.patch")

    def build(self):
        cmake = CMake(self, msbuild_verbosity='normal')
        cmake.verbose = True
        cmake.definitions["BUILD_SHARED_LIB"] = "OFF"
        cmake.definitions["SUPPORT_CONSOLE_APP"] = "OFF"
        if self.settings.get_safe("compiler.runtime") in ("MT", "MTd"):
            cmake.definitions["USE_STATIC_RUNTIME"] = "ON"
        cmake.configure()
        cmake.build()
        cmake.install()

    def package(self):
        self.copy("FindTidy.cmake", dst=".", keep_path=False)
        self.copy("*/tidy-static.pdb", dst="bin", keep_path=False)

    def package_id(self):
        self.info.options.ninja = "any"

    def package_info(self):
        # Libraries
        self.cpp_info.libs = tools.collect_libs(self)
        # Defines
        if self.settings.os == "Windows":
            self.cpp_info.defines.append("TIDY_STATIC")


# -*- coding: utf-8 -*-
"""
Created on Tue Feb  7 11:32:22 2017
@author: lbrein
 ---  与tick 配套对比计算的 kline
    > 1m的 K线 模拟tick预测
    > 定期执行，用于筛选交易对
"""

from com.base.public import public, logger
from com.object.obj_entity import future_baseInfo
import talib as ta
import pandas as pd
import numpy as np
import time
from multiprocessing import Pool, Manager
from com.train.train_future_singleExpect import train_future_singleExpect
from com.base.stat_fun import fisher
import math

# 回归方法
class train_future_macd(train_future_singleExpect):
    """

    """
    iniAmount = 250000  # 单边50万
    csvList = [
        "15_2.0_30_3_2.0_1.5",
        "15_2.0_15_3_2.0_2.5",
        "30_2.0_60_3_2.0_1.5",
    ]

    def __init__(self):
        # 费率和滑点
        super().__init__()

        self.saveDetail = True  # 是否保存明细
        self.isSimTickUse = False  # 是否使用1分钟模拟tick测试，否则直接使用kline回测
        self.topUse = False
        self.isEmptyUse = False
        self.baseInfo = {}

        # 可变参数
        self.periodList = [15, 30, 60]  # 窗体参数
        self.scaleList = [2.0]
        self.klineTypeList = ['15m', '30m', '60m']

        self.widthTimesPeriodList = [3]

        self.superlineList = [2.0]
        self.powline = 0.1
        self.turnlineList = [1.5, 2.0, 2.5]
        self.turnline = 1.5
        self.paramList = ['superline', 'turnline']

        self.shiftScale = 0.527  # 滑点模拟系数
        self.scaleDiff2 = 0.8
        self.scaleDiff = -0.1
        self.processCount = 6

        self.testDays = 90
        # 起始时间
        self.startDate = public.getDate(diff=-self.testDays)  # 60天数据回测
        self.endDate = public.getDate(diff=0)

        self.total_tablename = 'train_total_1'
        self.detail_tablename = 'train_future_1'

        self.method = 'simTick' if self.isSimTickUse else 'quick'
        self.stage = 'dema6'
        self.uidKey = "%s_%s_%s_%s_%s_" + self.method + "_" + self.stage

        # 切换全部计算
        self.isAll = 0

    def iterCond(self):
        # 多重组合参数输出
        keys = self.paramList
        for s0 in self.__getattribute__(keys[0] + 'List'):
            self.__setattr__(keys[0], s0)

            for s1 in self.__getattribute__(keys[1] + 'List'):
                self.__setattr__(keys[1], s1)

                # for s2 in self.__getattribute__(keys[2] + 'List'):
                #    self.__setattr__(keys[2], s2)
                yield '%s_%s' % (str(s0), str(s1))

    def Pool(self):
        time0 = time.time()

        pool = Pool(processes=self.processCount)
        shareDict = Manager().list([])

        Base = future_baseInfo()
        # 交易量大的，按价格排序, 类型:turple,第二位为夜盘收盘时间
        lists = Base.all(vol=100)
        tops = self.tops()
        # 清空数据库
        self.switch()

        for rs in lists:
            # 检查时间匹配
            codes = [rs[0]]
            if self.topUse and codes not in tops: continue
            print(rs)
            if self.isAll == 0 and codes[0] not in ['RB','CU','ZC','FG','SC','MA']:  continue

            for kt in self.klineTypeList:
                #self.start(codes, time0, kt, shareDict)
                try:
                    pool.apply_async(self.start, (codes, time0, kt, shareDict))
                    pass
                except Exception as e:
                    print(e)
                    continue
        pool.close()
        pool.join()

    pointColumns = ['pow', 'powm', 'trend', 'isout']

    def point(self, row, ktype=15):
        wd1, s1, s2, high, low, open, close, top, lower, p_h, p_l, atr, atr1 = (row[key] for key in
                                                                                "wd1,slope,slope60,high,low,open,close,top,lower,p_h,p_l,atr,atr1".split(
                                                                                ","))
        TL, PL= self.turnline,  self.powline
        # trend.

        if close > open:
            trend = 1 if high == open else ((close - open) / (high - open) + 0.5 )
        else:
            trend = -1 if low == open else ((close - open) / (open - low) - 0.5)

        tr = (4 - atr * atr1) if atr > 2 else atr * atr1
        if tr < 0: tr = 0
        # pow
        pow = (wd1 + abs(s1 + s2 +  trend * tr)) / 4

        sign = np.sign(s1) if s1 != 0 else np.sign(trend) if trend != 0 else np.sign(s2)

        isout = -1 if p_h > top else 1 if p_l < lower else 0
        # powm
        powm = -3 * np.sign(trend) if (atr > 3.2) and (np.sign(trend) * isout < 0)  else 2 * sign if (pow > TL) else sign if pow > PL else 0

        columns = self.pointColumns
        return pd.Series([pow, powm, trend, isout], index=columns)

    def sd(self, x):
        s = round(x * 1.5, 0) / 10
        return np.sign(s) * 0.8 - 0.1 if abs(s) > 0.8 else s - 0.1

    def total(self, dfs, dfs2=None, period=60):
        # 计算参数
        df0 = dfs[self.mCodes[0]]
        df0["rel_price"] = close = df0["close"]
        df0["datetime"] = df0.index

        s0 = self.shift[0]
        p_l = df0["p_l"] = (df0["close"] + s0)
        p_h = df0["p_h"] = (df0["close"] - s0)

        # bull
        df0["ma"] = ma = ta.MA(close, timeperiod=period)
        df0["std"] = std = ta.STDDEV(close, timeperiod=period, nbdev=1)
        # 上下柜
        df0["width"] = width = (4 * std / ma * 100).fillna(0)

        # 短期价格波动参数值
        df0["wd1"] = self.stand(ta.LINEARREG_SLOPE(width, timeperiod=2))
        df0["wdstd"] = self.stand(ta.STDDEV(width, timeperiod=10, nbdev=1))

        df0["slope"] = self.stand(ta.LINEARREG_SLOPE(ta.MA(close, timeperiod=3), timeperiod=2))
        #df0["slope30"] = self.stand(ta.LINEARREG_SLOPE(ta.MA(close, timeperiod=period), timeperiod=3))
        df0["slope60"] = self.stand(ta.LINEARREG_SLOPE(ta.MA(close, timeperiod=60), timeperiod=5))
        df0['sd'] = df0['slope60'].apply(lambda x: self.sd(x))

        df0.fillna(0, inplace=True)

        # kdj顶点
        kdjK, kdjD = ta.STOCH(df0["high"], df0["low"], close,
                              fastk_period=5, slowk_period=3, slowk_matype=1, slowd_period=3,
                              slowd_matype=1)

        df0["kdj_d2"] = kdj_d2 = kdjK - kdjD
        df0["kdjm"] = kdj_d2 * kdj_d2.shift(1)
        df0["kdjm"] = df0.apply(lambda row: self.turn(row['kdjm'], row['kdj_d2'], 1), axis=1)

        # SAR
        sar = ta.SAR(df0['high'], df0['low'], acceleration=0.05, maximum=0.2)
        df0['sard'] = sard = close - sar
        df0['sarm'] = sard * sard.shift(1)
        df0['sarm'] = df0.apply(lambda row: self.turn(row['sarm'], row['sard'], 1), axis=1)

        # ATR
        df0['atr'] = ta.ATR(df0['high'], df0['low'], close, timeperiod=1) / ta.ATR(df0['high'], df0['low'], close,
                                                                                timeperiod=period)

        df0['atr1'] = self.stand(ta.ATR(df0['high'], df0['low'], close, timeperiod=period))
        df0['atrb'] = ta.ATR(df0['high'], df0['low'], close, timeperiod=period)

        # 循环 scale
        docs = []
        for scale in self.scaleList:
            for conds in self.iterCond():
                ktype = self.klineType[:-1]

                if self.isAll == 1:
                    b0 = self.baseInfo[self.mCodes[0]]
                    # 修正品种
                    bulltype = b0['bulltype']
                    #if bulltype == 0 and self.turnline > 2.0: continue
                    #if bulltype == 2 and (self.turnline < 2.0 or ktype == '5'): continue

                uid = self.uidKey % (
                    '_'.join(self.codes), str(period), str(scale), ktype,
                    str(self.widthTimesPeriod) + '_' + conds)

                df0["top"], df0["lower"] = df0['ma'] + (scale + df0['sd']) * df0['std'], df0['ma'] - (
                        scale - df0['sd']) * df0['std']

                # 计算关键转折点
                df1 = df0.apply(lambda row: self.point(row, ktype), axis=1)
                for key in self.pointColumns: df0[key] = df1[key]
                df0['isout'] = ta.SUM(df0['isout'], timeperiod=3)

                isCvs = False
                for key in self.csvList:
                    if uid.find(key)>-1:
                         isCvs = True
                         break

                if isCvs:
                    cs = []
                    bans = 'ma,open,close,high,low,p_l,p_h,top,lower,std,delta,volume,sard,rel_price,width,volm,'.split(
                        ',')
                    for c in df0.columns:
                        if c not in bans:
                            cs.append(c)

                    file = self.Rice.basePath + '%s_pre.csv' % (uid)
                    print(uid, '---------------------------- to_cvs', file)
                    df0.to_csv(file, index=0, columns=cs)
                    # self.share.append(self.codes)

                # df0.fillna(0, inplace=True)
                df1 = None
                tot = None
                tot = self.detect(df0, df1, period=period, uid=uid)
                if tot is not None and tot['amount'] != 0:
                    tot.update(
                        {
                            "scale": scale,
                            "code": self.codes[0],
                            "period": period,
                            "uid": uid,
                            "shift": (p_l - p_h).mean(),
                            "std": df0['width'].mean(),
                            "createdate": public.getDatetime()
                        }
                    )
                    docs.append(tot)
        return docs

    # 核心策略部分
    def stageApply(self, df0, df1, period=15, uid=''):
        self.records = []
        """
            布林带策略：
        """
        status, isOpen = 0, 0
        pd2 = self.turnline - 0.20
        b0 = self.baseInfo[self.mCodes[0]]
        # 修正品种
        bulltype = b0['bulltype']

        for i in range(period, len(df0)):
            isRun, isstop = False, 0

            ma, close, p_l, p_h, top, lower, std, kdjm, sarm, pow, powm, sd, isout, atr, atr1, atrb = (
                df0.ix[i, key] for key in
                "ma,close,p_l,p_h,top,lower,std,kdjm,sarm,pow,powm,sd,isout,atr,atr1,atrb".split(","))

            sd2 = (self.scaleDiff2 - sd) * std / 2  # std修正

            if status == 0 and powm in [2, -2] and isOpen not in [6, -6]:
                """
                   突变点处理
                   将布林带策略切换为策略
                """
                status = np.sign(powm)
                if isOpen != 0 and isOpen * status < 0:
                    if self.order(df0.iloc[i], None, 0, uid, df0, isstop=4):
                        isOpen, isRun = 0,  False
                        if bulltype == 0:
                            isOpen, isRun = int(5 * status), True

                elif isOpen == 0 and bulltype ==0 :
                    isOpen, isRun = int(4 * status), True

            elif (sarm * status < 0) or (abs(isOpen) in [4, 5] and isOpen * sarm < 0) or abs(powm)==3:
                """
                    交叉点: sarm结束点
                    1、结束突变状态，变为布林带处理
                    2、根据节点状况开平新仓，状态为6
                """
                # 结束macd状态
                status = 0
                if abs(powm) == 3:
                    # 触底反弹
                    if isOpen != 0 and isOpen * powm < 0:
                       if self.order(df0.iloc[i], None, 0, uid, df0, isstop=6):
                          isOpen, isRun = int(2 * powm), True

                    elif isOpen == 0 :
                        isOpen, isRun = int(2 * powm), True

                elif isOpen != 0 and isOpen * sarm < 0:
                    isOpen, isRun, isstop = 0, True, 5

            elif status == 0:
                """
                    布林带策略处理
                """
                cond1 = (powm == 0 and (atr * atr1) < 1.0)
                if isOpen == 0:
                    # 突变状态开始
                    # 大于上线轨迹
                    if p_h >= top and powm == 0:
                        isOpen, isRun = -1, True

                    elif p_l <= lower and powm == 0:
                        isOpen, isRun = 1, True

                    elif ((p_h + sd2) >= top or (isout < 0 and (p_h - sd2) >= ma)) and pow < (pd2 - sd) and kdjm < 0:
                        isOpen, isRun = -2, True

                    elif ((p_l - sd2) <= lower or (isout > 0 and (p_l + sd2) <= ma)) and pow < (pd2 + sd) and kdjm > 0:
                        isOpen, isRun = 2, True

                # 平仓
                else:
                    sign = np.sign(isOpen)
                    cond3 = (sign * ((p_h if isOpen > 0 else p_l) - ma))
                    #
                    if (cond3 >= - sd2 or sign * isout < 0) and sign * kdjm < 0:
                        if ((p_h + sd2) >= top or (p_l - sd2) <= lower) and pow < pd2:
                            if self.order(df0.iloc[i], None, 0, uid, df0, isstop=1):
                                # 再开盘
                                isOpen, isRun = -2 if (p_h + sd2) >= top else 2, True

                        else:
                            isOpen, isstop = 0, 1
                            isRun = True

                    elif cond3 >= 0 and pow < (self.powline) and cond1:
                        isOpen, isstop = 0, 0
                        isRun = True

                    # 超过2倍 atr，止损
                    else:
                        # 即时收益超出
                        if self.preNode is not None:
                            Pd = self.preNode[0]['price']
                            if (sign * (Pd-close) > 2.0 * atrb) and sign * kdjm < 0:
                                isOpen,  isRun, isstop = 0,True, 7

                    # 周末尾盘

            if isRun:
                self.order(df0.iloc[i], None, isOpen, uid, df0, isstop=isstop)
        return self.records

def main():
    action = {
        "kline": 1,
    }

    if action["kline"] == 1:
        obj = train_future_macd()
        obj.Pool()

if __name__ == '__main__':
    main()


#
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
# http://nexb.com and https://github.com/nexB/vulnerablecode/
# The VulnerableCode software is licensed under the Apache License version 2.0.
# Data generated with VulnerableCode require an acknowledgment.
#
# You may not use this software except in compliance with the License.
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
# derivative work, you must accompany this data with the following acknowledgment:
#
#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
#  for any legal advice.
#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
#  Visit https://github.com/nexB/vulnerablecode/ for support and download.

import json
import os

from django.test import TestCase

from vulnerabilities.data_dump import archlinux_dump
from vulnerabilities.data_dump import debian_dump
from vulnerabilities.data_dump import ubuntu_dump
from vulnerabilities.models import ImpactedPackage
from vulnerabilities.models import Package
from vulnerabilities.models import PackageReference
from vulnerabilities.models import ResolvedPackage
from vulnerabilities.models import Vulnerability
from vulnerabilities.models import VulnerabilityReference
from vulnerabilities.scraper import archlinux
from vulnerabilities.scraper import debian
from vulnerabilities.scraper import ubuntu

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DATA = os.path.join(BASE_DIR, 'test_data/')

class TestDebianDataDump(TestCase):
    @classmethod
    def setUpTestData(cls):
        with open(os.path.join(TEST_DATA, 'debian.json')) as f:
            test_data = json.load(f)

        extract_data = debian.extract_vulnerabilities(test_data)
        debian_dump(extract_data)

    def test_Vulnerability(self):
        """
        Check that all vulnerabilities from the test data are stored in the database
        """
        self.assertEqual(3, Vulnerability.objects.count())

        self.assertTrue(Vulnerability.objects.filter(cve_id='CVE-2009-1382'))
        self.assertTrue(Vulnerability.objects.filter(cve_id='CVE-2009-2459'))
        self.assertTrue(Vulnerability.objects.filter(cve_id='CVE-2014-8242'))

    def test_VulnerabilityReference(self):
        """
        Check that no vulnerability references were found in the test data
        """
        self.assertEqual(0, VulnerabilityReference.objects.count())

    def test_Package(self):
        """
        Check that all packages from the test data are stored in the database
        """
        # There are five rows in Package because currently the models allow duplicates
        # (see issue #28).
        self.assertEqual(5, Package.objects.count())

        self.assertTrue(Package.objects.filter(name='mimetex'))

        pkg = Package.objects.get(name='librsync')
        self.assertEqual('0.9.7-10', pkg.version)
        self.assertEqual('deb', pkg.type)
        self.assertEqual('debian', pkg.namespace)
        self.assertIn('distro=jessie', pkg.qualifiers)

    def test_ImpactedPackage(self):
        """
        Check that all impacted packages from the test data are stored in the database
        """
        impacted_pkgs = ImpactedPackage.objects.all()

        self.assertEqual(1, impacted_pkgs.count())

        ip = impacted_pkgs[0]
        self.assertEqual('librsync', ip.package.name)
        self.assertEqual('0.9.7-10', ip.package.version)

    def test_ResolvedPackage(self):
        """
        Check that all resolved packages from the test data are stored in the database
        """
        resolved_pkgs = ResolvedPackage.objects.all()
        versions = [rp.package.version for rp in resolved_pkgs]

        self.assertEqual(4, resolved_pkgs.count())
        self.assertEqual('mimetex', resolved_pkgs[0].package.name)
        self.assertIn('1.50-1.1', versions)
        self.assertIn('1.74-1', versions)

class TestUbuntuDataDump(TestCase):
    @classmethod
    def setUpTestData(cls):
        with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f:
            test_data = f.read()

        data = ubuntu.extract_cves(test_data)
        ubuntu_dump(data)

    def test_data_dump(self):
        """
        Check basic data import
        """
        self.assertTrue(Vulnerability.objects.filter(cve_id='CVE-2002-2439'))
        pkgs = Package.objects.filter(name='gcc-4.6')
        self.assertTrue(pkgs)

        pkg = pkgs[0]
        self.assertEqual('deb', pkg.type)
        self.assertEqual('ubuntu', pkg.namespace)

class TestArchLinuxDataDump(TestCase):

    CVE_IDS = ('CVE-2018-11362', 'CVE-2018-11361', 'CVE-2018-11360',
               'CVE-2018-11359', 'CVE-2018-11358', 'CVE-2018-11357',
               'CVE-2018-11356', 'CVE-2018-11355', 'CVE-2018-11354')

    @classmethod
    def setUpTestData(cls):
        with open(os.path.join(TEST_DATA, 'archlinux.json')) as f:
            test_data = json.load(f)

        archlinux_dump(test_data)

    def test_Vulnerability(self):
        """
        Check that all vulnerabilities from the test data are stored in the database
        """
        self.assertEqual(len(self.CVE_IDS), Vulnerability.objects.count())

        for cve_id in self.CVE_IDS:
            self.assertTrue(Vulnerability.objects.filter(cve_id=cve_id))

    def test_VulnerabilityReference(self):
        """
        Check that all vulnerability references from the test data are stored in the database
        """
        for ref in ('ASA-201805-22', 'ASA-201805-23', 'ASA-201805-24', 'ASA-201805-25', 'AVG-708'):
            self.assertEqual(
                    len(self.CVE_IDS),
                    VulnerabilityReference.objects.filter(reference_id=ref).count()
            )

        for ref in self.CVE_IDS:
            url = f'https://security.archlinux.org/{ref}'
            self.assertEqual(1, VulnerabilityReference.objects.filter(url=url).count())

    def test_Package(self):
        """
        Check that all packages from the test data are stored in the database
        """
        self.assertEqual(8, Package.objects.count())

        for pkg in ('wireshark-common', 'wireshark-gtk', 'wireshark-cli', 'wireshark-qt'):
            for ver in ('2.6.0-1', '2.6.1-1'):
                self.assertTrue(Package.objects.filter(name=pkg, version=ver))

        for pkg in Package.objects.filter(name='wireshark-cli'):
            self.assertEqual('pacman', pkg.type)
            self.assertEqual('archlinux', pkg.namespace)

    def test_PackageReference(self):
        """
        Check that no package references were found in the test data
        """
        self.assertEqual(0, PackageReference.objects.count())

    def test_ImpactedPackage(self):
        """
        Check there is one ImpactedPackage for the number of packages
        with the affected version number, times the number of vulnerabilities
        """
        packages = Package.objects.filter(version='2.6.0-1')
        vulnerabilities = Vulnerability.objects.all()

        impacted_pkgs_count = ImpactedPackage.objects.count()
        expected_count = packages.count() * vulnerabilities.count()

        self.assertEqual(expected_count, impacted_pkgs_count)

        for pkg in packages:
            for vuln in vulnerabilities:
                self.assertTrue(ImpactedPackage.objects.filter(
                    package=pkg,
                    vulnerability=vuln,
                ))

    def test_ResolvedPackage(self):
        """
        Check there is one ResolvedPackage for the number of packages
        with the fixed version number, times the number of vulnerabilities
        """
        packages = Package.objects.filter(version='2.6.1-1')
        vulnerabilities = Vulnerability.objects.all()

        resolved_pkgs_count = ResolvedPackage.objects.count()
        expected_count = packages.count() * vulnerabilities.count()

        self.assertEqual(expected_count, resolved_pkgs_count)

        for pkg in packages:
            for vuln in vulnerabilities:
                self.assertTrue(ResolvedPackage.objects.filter(
                    package=pkg,
                    vulnerability=vuln,
                ))


import os
import sys
sys.path.append("../")

import pickle
import numpy as np
from tensorflow.keras.utils import Sequence
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from official.nlp import optimization
from sklearn.utils import shuffle

from neurallog.models import NeuralLog
from neurallog import data_loader

log_file = "../data/raw/BGL.log"
emb_dir = "../data/embeddings/BGL"
embed_dim = 768  # Embedding size for each token
max_len = 75

class BatchGenerator(Sequence):

    def __init__(self, X, Y, batch_size):
        self.X, self.Y = X, Y
        self.batch_size = batch_size

    def __len__(self):
        return int(np.ceil(len(self.X) / float(self.batch_size)))

    def __getitem__(self, idx):
        # print(self.batch_size)
        dummy = np.zeros(shape=(embed_dim,))
        x = self.X[idx * self.batch_size:min((idx + 1) * self.batch_size, len(self.X))]
        X = np.zeros((len(x), max_len, embed_dim))
        Y = np.zeros((len(x), 2))
        item_count = 0
        for i in range(idx * self.batch_size, min((idx + 1) * self.batch_size, len(self.X))):
            x = self.X[i]
            if len(x) > max_len:
                x = x[-max_len:]
            x = np.pad(np.array(x), pad_width=((max_len - len(x), 0), (0, 0)), mode='constant',
                       constant_values=0)
            X[item_count] = np.reshape(x, [max_len, embed_dim])
            Y[item_count] = self.Y[i]
            item_count += 1
        return X[:], Y[:, 0]

def train_generator(training_generator, validate_generator, num_train_samples, num_val_samples, batch_size,
                    epoch_num, model_name=None):

    epochs = epoch_num
    steps_per_epoch = num_train_samples
    num_train_steps = steps_per_epoch * epochs
    num_warmup_steps = int(0.1 * num_train_steps)

    init_lr = 3e-4
    optimizer = optimization.create_optimizer(init_lr=init_lr,
                                              num_train_steps=num_train_steps,
                                              num_warmup_steps=num_warmup_steps,
                                              optimizer_type='adamw')

    loss_object = SparseCategoricalCrossentropy()

    model = NeuralLog(768, loss_object, optimizer)

    # model.load_weights("hdfs_transformer.hdf5")

    print(model.summary())

    # checkpoint
    filepath = model_name
    checkpoint = ModelCheckpoint(filepath,
                                 monitor='val_accuracy',
                                 verbose=1,
                                 save_best_only=True,
                                 mode='max',
                                 save_weights_only=True)
    early_stop = EarlyStopping(
        monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto',
        baseline=None, restore_best_weights=True
    )
    callbacks_list = [checkpoint, early_stop]

    model.fit_generator(generator=training_generator,
                        steps_per_epoch=int(num_train_samples / batch_size),
                        epochs=epoch_num,
                        verbose=1,
                        validation_data=validate_generator,
                        validation_steps=int(num_val_samples / batch_size),
                        workers=16,
                        max_queue_size=32,
                        callbacks=callbacks_list,
                        shuffle=True
                        )
    return model

def train(X, Y, epoch_num, batch_size, tx, ty, model_file=None):
    X, Y = shuffle(X, Y)
    n_samples = len(X)
    train_x, train_y = X[:int(n_samples * 90 / 100)], Y[:int(n_samples * 90 / 100)]
    val_x, val_y = X[int(n_samples * 90 / 100):], Y[int(n_samples * 90 / 100):]

    training_generator, num_train_samples = BatchGenerator(train_x, train_y, batch_size), len(train_x)
    validate_generator, num_val_samples = BatchGenerator(val_x, val_y, batch_size), len(val_x)

    print("Number of training samples: {0} - Number of validating samples: {1}".format(num_train_samples,
                                                                                num_val_samples))

    model = train_generator(training_generator, validate_generator, num_train_samples, num_val_samples, batch_size,
                            epoch_num, model_name=model_file)

    return model

if __name__ == '__main__':
    # (x_tr, y_tr), (x_te, y_te) = data_loader.load_Supercomputers(
    #     log_file, train_ratio=0.8, windows_size=20,
    #     step_size=5, e_type='bert', e_name=None, mode='balance')

    with open(os.path.join(emb_dir, "bert-train.pkl"), mode="rb") as f:
        (x_tr, y_tr) = pickle.load(f)

    with open(os.path.join(emb_dir, "bert-test.pkl"), mode="rb") as f:
        (x_te, y_te) = pickle.load(f)

    model = train(x_tr, y_tr, 20, 64, x_te, y_te, "bgl_transformer.hdf5")