from bs4 import BeautifulSoup
from urllib import request
import re
import json

class Crawler(object):

    def openLatestSeason(self, base):
        basePage = request.urlopen(base).read()
        soup = BeautifulSoup(basePage)
        aTags = soup.find_all("a")
        lastAnchor = aTags[-1].get('href')
        link = base + lastAnchor[1:]
        return link

    def openLatestCompetition(self, season_page_url):
        season_page = request.urlopen(season_page_url).read()
        soup = BeautifulSoup(season_page)
        aTags = soup.find_all("a")
        lastAnchor = aTags[-2].get('href')
        link = season_page_url + lastAnchor[1:]
        return link

    def getCompetitionLinks(self, competition_page_url):
        competition_page = request.urlopen(competition_page_url).read()
        eventRE = re.compile('^FTE')
        soup = BeautifulSoup(competition_page)
        aTags = soup.find_all("a")
        links = []
        for anchor in aTags:
            link = anchor.get('href')
            if eventRE.match(link):
                links.append(competition_page_url + '/' + link)
        links = set(links)
        return links

class Parser(object):

    soup = ''
    tournamentURL = ''
    eventURL = ''

    def __init__(self, tournamentURL):
        self.tournamentURL = tournamentURL
        self.links = self.setup(False if tournamentURL is '' else True)

    def setup(self, haveURL=''):
        crawler = Crawler()
        if haveURL is not '':
            links = crawler.getCompetitionLinks(self.tournamentURL)
        else:
            links = crawler.getCompetitionLinks(
                crawler.openLatestCompetition(
                    crawler.openLatestSeason(
                        'http://results.usfencingresults.org')
                )
            )
        return links

    def main(self):
        tournamentJSON = {}
        for event in self.links:
            if 'CHR' not in event and 'TM' not in event:
                self.eventURL = event
                page = request.urlopen(event).read()
                self.soup = BeautifulSoup(page)
                print(
                    '\n\n---------------' + str(event) + '-----------------\n\n')
                tournamentJSON[self.getEventName()] = self.parseCompetition()

        i = {
            'name': self.getTournamentName(),
            'URL': self.tournamentURL,
            'finished': self.getFinished(tournamentJSON)
        }
        tournamentJSON['tournament'] = i
        with open('test.json', 'w+') as file:
            file.write(json.dumps(tournamentJSON))
        return tournamentJSON

    def getFinished(self, JSON):
        for key in JSON:
            if type(JSON[key]) is type({}) and \
                    JSON[key]['event']['finished'] is 0:
                return 0
        return 1

    def parseCompetition(self):
        tabs = self.soup.find_all("a", class_='jumpListItem1')
        prelimSeed = []
        checkIn = []
        round1Scores = []
        round1Seeding = []
        finalResults = []
        round2Seeding = []
        for x in tabs:
            if 'Preliminary Seeding' in x.get_text():
                prelimSeed = self.parseSeeding('prelimSeed')
            elif 'Check-In Status' in x.get_text():
                checkIn = self.parseCheckIn()
            elif 'Round #1 - Pool' in x.get_text():
                round1Seeding = self.parseSeeding('Round1Seeding')
                round1Scores = self.parsePoolRound()
            elif 'Round #2 - DE' in x.get_text():
                round2Seeding = self.parseSeeding('Round2Seeding')
                # self.calculateDirectElimination(resultsJSON, round1JSON)
            elif 'Final Results' in x.get_text():
                finalResults = self.parseFinalResults()
        return self.exportJSON(
            prelimSeed,
            checkIn,
            round1Seeding,
            round1Scores,
            round2Seeding,
            finalResults
        )

    def exportJSON(self,
                   prelimSeed,
                   checkIn,
                   round1Seeding,
                   round1Scores,
                   round2Seeding,
                   finalResults):
        bigjson = {
            'prelimSeed': prelimSeed,
            'checkIn': checkIn,
            'round1Seeding': round1Seeding,
            'round1Scores': round1Scores,
            'round2Seeding': round2Seeding,
            'finalResults': finalResults,
            'event': {
                'URL': self.eventURL,
                'name': self.getEventName(),
                'finished': 1 if finalResults is not {} else 0
            }
        }
        return bigjson

    def getTournamentName(self):
        span = self.soup.find('span', class_='tournName')
        return span.get_text()

    def getEventName(self):
        span = self.soup.find('span', class_='tournDetails')
        return span.contents[0].strip()

    def calculateDirectElimination(self, resultsJSON, roundJSON):
        pass

    def parseFinalResults(self):
        table = self.soup.find(id='finalResults')
        table = table.find_all('tr', class_='dataTableGoldResultRow') +  \
            table.find_all('tr', class_='dataTableSilverResultRow') + \
            table.find_all('tr', class_='dataTableBronzeResultRow') + \
            table.find_all('tr', class_='dataTableOddRow') + \
            table.find_all('tr', class_='dataTableEvenRow')
        resultsJSON = []
        for row in table:
            fencer = {}
            children = [value for value in row.contents if value != '\n']
            count = 0
            for data in children:
                text = data.get_text()
                text = text.encode('utf_8')
                if count is 0:
                    fencer['place'] = self.replaceOddEncodings(text)
                elif count is 1:
                    fencer['lastName'] = self.replaceOddEncodings(
                        text.split(b',')[0])
                    fencer['firstName'] = self.replaceOddEncodings(
                        text.split(b',')[1])
                elif count is 2:
                    fencer['club'] = self.replaceOddEncodings(text)
                elif count is 3:
                    fencer['division'] = self.replaceOddEncodings(text)
                elif count is 4:
                    text = text.replace(b'\n', b'')
                    fencer['country'] = self.replaceOddEncodings(text)
                elif count is 5:
                    fencer['classification'] = self.replaceOddEncodings(text)
                elif count is 6:
                    fencer['earned'] = self.replaceOddEncodings(text)
                count += 1
            resultsJSON.append(fencer)
        return resultsJSON

    def parsePoolRound(self):
        table = self.soup.find(id='Round1Scores')
        poolNum = table.select('div.poolNum')
        stripNum = self.soup.select('span.poolStripTime')
        pools = table.select('table.pool')
        poolRoundJSON = {}
        counter = 0
        for pool in pools:
            poolJSON = {}
            poolJSON['pool number'] = self.replaceOddEncodings(
                poolNum[counter].get_text().encode('utf_8'))
            try:
                poolJSON['strip assignment'] = self.replaceOddEncodings(
                    stripNum[counter].get_text()
                    .replace('On strip', '')
                    .encode('utf_8'))
            except IndexError:
                poolJSON['strip assignment'] = ''
            children = [value for value in pool.contents if value != '\n']
            pool = {}
            rowIndicator = 0
            competitors = []
            for column in children:
                row = column.select('td.poolNameCol') + \
                    column.select('span.poolAffil') + \
                    column.select('td.poolScoreCol') + \
                    column.select('td.poolResultCol')
                finalizedRow = []
                count = 0
                for data in row:
                    if count is 0:
                        finalizedRow.append(data.contents[0].strip())
                        name = data.contents[0].strip().split(',')
                        # secondString = data.contents[2].get_text().split('/')
                        competitor = {}
                        competitor['lastName'] = name[0]
                        competitor['firstName'] = name[1]
                        # competitor['clubAbbr'] = secondString[0]
                        # if len(secondString) == 3:
                        #   competitor['country'] = secondString[2]
                        #   competitor['division'] = secondString[1]
                        # elif(secondString) == 2:
                        #   competitor['country'] = secondString[1]
                        competitors.append(competitor)
                    else:
                        finalizedRow.append(
                            self.replaceOddEncodings(data.get_text().encode(
                                'utf_8')))
                    count += 1
                pool[str(rowIndicator)] = finalizedRow
                rowIndicator += 1
            poolJSON['competitors'] = competitors
            poolJSON['pool'] = pool
            counter += 1
            poolRoundJSON[poolJSON['pool number']] = poolJSON
        return poolRoundJSON

    def parseCheckIn(self):
        table = self.soup.find(id='checkIn')
        table = table.find_all('tr', class_='dataTableOddRow') + \
            table.find_all('tr', class_='dataTableEvenRow')
        checkinJSON = []
        for row in table:
            fencer = {}
            children = [value for value in row.contents if value != '\n']
            count = 0
            for data in children:
                text = data.get_text()
                text = text.encode('utf_8')
                if count is 0:
                    img = row.find('img', attrs={'src': 'CheckIn.png'})
                    if img is not None:
                        fencer['Check In'] = True
                    else:
                        fencer['Check In'] = False
                elif count is 1:
                    fencer['lastName'] = self.replaceOddEncodings(
                        text.split(b',')[0])
                    fencer['firstName'] = self.replaceOddEncodings(
                        text.split(b',')[1])
                elif count is 2:
                    fencer['club'] = self.replaceOddEncodings(text)
                elif count is 3:
                    fencer['division'] = self.replaceOddEncodings(text)
                elif count is 4:
                    fencer['country'] = self.replaceOddEncodings(text)
                count += 1
            checkinJSON.append(fencer)
        return checkinJSON

    def parseSeeding(self, tableId):
        table = self.soup.find(id=tableId)
        table = table.find_all(
            'tr', class_='dataTableOddRow') + table.find_all(
            'tr', class_='dataTableEvenRow')
        seedingJSON = []
        for row in table:
            fencer = {}
            if tableId is 'Round1Seeding':
                fencer['status'] = ''
            pools = {}
            children = [value for value in row.contents if value != '\n']
            count = 0
            for data in children:
                text = data.get_text()
                text = text.encode('utf_8')
                if count is 0:
                    fencer['seeding'] = self.replaceOddEncodings(text)
                elif count is 1:
                    fencer['lastName'] = self.replaceOddEncodings(
                        text.split(b',')[0])
                    fencer['firstName'] = self.replaceOddEncodings(
                        text.split(b',')[1])
                elif count is 2:
                    text = text.split(b'/')
                    fencer['classification'] = self.replaceOddEncodings(
                        text[0])
                    if len(text) is 2:
                        fencer['rank'] = self.replaceOddEncodings(text[1])
                    else:
                        fencer['rank'] = None
                elif count is 3:
                    fencer['club'] = self.replaceOddEncodings(text)
                elif count is 4:
                    fencer['division'] = self.replaceOddEncodings(text)
                elif count is 5:
                    text = text.replace(b'\n', b'')
                    fencer['country'] = self.replaceOddEncodings(text)
                elif count is 6:
                    pools['v'] = self.replaceOddEncodings(text)
                elif count is 7:
                    pools['vr'] = self.replaceOddEncodings(text)
                elif count is 8:
                    pools['ts'] = self.replaceOddEncodings(text)
                elif count is 9:
                    pools['tr'] = self.replaceOddEncodings(text)
                elif count is 10:
                    pools['i'] = self.replaceOddEncodings(text)
                elif count is 11:
                    fencer['status'] = self.replaceOddEncodings(text)
                    fencer['pool'] = pools
                count += 1
            seedingJSON.append(fencer)
        return seedingJSON

    def replaceOddEncodings(self, text):
        return text.decode('utf_8').replace('\xa0', '')

if __name__ == '__main__':
    x = Parser(
        'http://results.usfencingresults.org/2014-2015/2015.06-JUN-SN')
    x.main()

    # dbm = DBManager(json)
    # dbm.main()


import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet50
from PVT.pvt import pvt_tiny

class FPN(nn.Module):
    def __init__(self, num_channels=256):
        super(FPN, self).__init__()

        self.lateral_layer_5 = nn.Conv2d(512, num_channels, kernel_size=1, stride=1, padding=0)
        self.lateral_layer_4 = nn.Conv2d(320, num_channels, kernel_size=1, stride=1, padding=0)
        self.lateral_layer_3 = nn.Conv2d(128, num_channels, kernel_size=1, stride=1, padding=0)
        self.lateral_layer_2 = nn.Conv2d(64, num_channels, kernel_size=1, stride=1, padding=0)

        self.output_layer_5 = nn.Conv2d(num_channels, num_channels, kernel_size=3, stride=1, padding=1)
        self.output_layer_4 = nn.Conv2d(num_channels, num_channels, kernel_size=3, stride=1, padding=1)
        self.output_layer_3 = nn.Conv2d(num_channels, num_channels, kernel_size=3, stride=1, padding=1)
        self.output_layer_2 = nn.Conv2d(num_channels, num_channels, kernel_size=3, stride=1, padding=1)

    def _upsample_like(self, source, target):
        _, _, height, width = target.shape
        return F.interpolate(source, size=(height, width), mode='nearest')

    def forward(self, C2, C3, C4, C5):
        P5 = nn.ReLU(inplace=True)(self.lateral_layer_5(C5))
        P5_upsampled = self._upsample_like(P5, C4)
        P5 = self.output_layer_5(P5)

        P4 = nn.ReLU(inplace=True)(self.lateral_layer_4(C4))
        P4_upsampled = self._upsample_like(P4, C3)
        P3 = self.output_layer_3(P4 + P5_upsampled)

        P3= nn.ReLU(inplace=True)(self.lateral_layer_3(C3))
        P3_upsampled = self._upsample_like(P3, C2)
        P3 = self.output_layer_3(P3 + P4_upsampled)

        P2 = nn.ReLU(inplace=True)(self.lateral_layer_2(C2))
        P2 = self.output_layer_2(P2 + P3_upsampled)

        return P2, P3, P4, P5

class Map_Branch(nn.Module):
    def __init__(self, num_channels_output):
        super(Map_Branch, self).__init__()

        self.convs_on_P5 = self._two_conv()
        self.convs_on_P4 = self._two_conv()
        self.convs_on_P3 = self._two_conv()
        self.convs_on_P2 = self._two_conv()

        self.convs_on_feature_maps = nn.Sequential(
            nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, num_channels_output, kernel_size=1, stride=1, padding=0)
        )

    def _two_conv(self):
        return nn.Sequential(
            nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True)
        )

    def forward(self, P2, P3, P4, P5):
        D5 = F.interpolate(self.convs_on_P5(P5), scale_factor=8, mode='bilinear')
        D4 = F.interpolate(self.convs_on_P4(P4), scale_factor=4, mode='bilinear')
        D3 = F.interpolate(self.convs_on_P3(P3), scale_factor=2, mode='bilinear')
        D2 = self.convs_on_P2(P2)

        feature_maps = torch.cat((D2, D3, D4, D5), dim=1)

        return self.convs_on_feature_maps(feature_maps)

# Using 2 interpolations
class Segmentation_Branch(nn.Module):
    def __init__(self):
        super(Segmentation_Branch, self).__init__()

        self.convs_on_P5 = self._two_conv()
        self.convs_on_P4 = self._two_conv()
        self.convs_on_P3 = self._two_conv()
        self.convs_on_P2 = self._two_conv()

        self.conv_on_feature_maps = nn.Sequential(
            nn.Conv2d(512, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
        )

        self.final_conv1 = nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=1)
        self.final_conv2 = nn.Conv2d(32, 1, kernel_size=3, stride=1, padding=1)

    def _two_conv(self):
        return nn.Sequential(
            nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True)
        )

    def forward(self, P2, P3, P4, P5):
        D5 = F.interpolate(self.convs_on_P5(P5), scale_factor=8, mode='bilinear')
        D4 = F.interpolate(self.convs_on_P4(P4), scale_factor=4, mode='bilinear')
        D3 = F.interpolate(self.convs_on_P3(P3), scale_factor=2, mode='bilinear')
        D2 = self.convs_on_P2(P2)

        feature_maps = torch.cat((D2, D3, D4, D5), dim=1)

        segmentation = F.interpolate(self.conv_on_feature_maps(feature_maps), scale_factor=2, mode='bilinear')
        segmentation = self.final_conv1(segmentation)
        segmentation = F.interpolate(segmentation, scale_factor=2, mode='bilinear')
        segmentation = self.final_conv2(segmentation)
        segmentation = (F.tanh(segmentation) + 1)/2

        return segmentation

class NADS_Net(torch.nn.Module):
    def __init__(self, include_seatbelt_branch, using_Aisin_output_format, include_background_output):
        super(NADS_Net, self).__init__()

        self.include_seatbelt_branch = include_seatbelt_branch

        if using_Aisin_output_format:
            if include_background_output:
                num_keypoints_output_layers = 10
            else:
                num_keypoints_output_layers = 9

            num_PAF_output_layers = 16
        else:
            num_keypoints_output_layers = 19
            num_PAF_output_layers = 38

        self.FPN = FPN()
        self.PVT = pvt_tiny()
        self.keypoint_heatmap_branch = Map_Branch(num_keypoints_output_layers)
        self.PAF_branch = Map_Branch(num_PAF_output_layers)

        if include_seatbelt_branch:
            self.seatbelt_segmentation_branch = Segmentation_Branch()

        def init_weights(m):
            if type(m) == nn.Conv2d:
                torch.nn.init.normal_(m.weight, mean=0, std=0.01)
                m.bias.data.fill_(0.0)

        self.FPN.apply(init_weights)
        self.keypoint_heatmap_branch.apply(init_weights)
        self.PAF_branch.apply(init_weights)

        if include_seatbelt_branch:
            self.seatbelt_segmentation_branch.apply(init_weights)

    def forward(self, x):
        C2, C3, C4, C5 = self.PVT(x)
        P2, P3, P4, P5 = self.FPN(C2, C3, C4, C5)

        keypoint_heatmaps = self.keypoint_heatmap_branch(P2, P3, P4, P5)
        PAFs = self.PAF_branch(P2, P3, P4, P5)

        if self.include_seatbelt_branch:
            seatbelt_segmentation = self.seatbelt_segmentation_branch(P2, P3, P4, P5)
            return keypoint_heatmaps, PAFs, seatbelt_segmentation
        else:
            return keypoint_heatmaps, PAFs


from .synthesizer_base import SynthesizerBase, run
import numpy as np
from sklearn.mixture import GaussianMixture
from synthetic_data_benchmark.utils import CONTINUOUS, ORDINAL, CATEGORICAL
rng = np.random

class IndependentSynthesizer(SynthesizerBase):
    """docstring for IdentitySynthesizer."""

    def __init__(self, gmm_n):
        self.gmm_n = gmm_n

    def train(self, train_data):
        self.dtype = train_data.dtype

        self.models = []
        for id_, info in enumerate(self.meta):
            if info['type'] == CONTINUOUS:
                model = GaussianMixture(self.gmm_n)
                model.fit(train_data[:, [id_]])
                self.models.append(model)
            else:
                nomial = np.bincount(train_data[:, id_].astype('int'), minlength=info['size'])
                nomial = nomial / np.sum(nomial)
                self.models.append(nomial)

    def generate(self, n):
        data = np.zeros([n, len(self.meta)], self.dtype)

        for i, info in enumerate(self.meta):
            if info['type'] == 'continuous':
                x, _ = self.models[i].sample(n)
                rng.shuffle(x)
                data[:, i] = x.reshape([n])
                data[:, i] = data[:, i].clip(info['min'], info['max'])
            else:
                data[:, i] = np.random.choice(np.arange(info['size']), n, p=self.models[i])

        return [(0, data)]

    def init(self, meta, working_dir):
        self.meta = meta

if __name__ == "__main__":
    run(IndependentSynthesizer(5))


from __future__ import absolute_import

import datetime
import json
import os
import time

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import transaction as django_transaction
from django.http import Http404, HttpResponseRedirect, HttpResponse, JsonResponse
from django.shortcuts import render
from django.utils.decorators import method_decorator
from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
from ipware.ip import get_ip
from raven.contrib.django.raven_compat.models import client
from redis import StrictRedis
from rratelimit import Limiter

from donation.forms import PledgeForm, PledgeComponentFormSet, PinTransactionForm, PledgeFormOld
from donation.models import PaymentMethod, Receipt, PartnerCharity, PinTransaction, RecurringFrequency
from donation.tasks import send_bank_transfer_instructions_task

r = StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT)
rate_limiter = Limiter(r,
                       action='test_credit_card',
                       limit=settings.CREDIT_CARD_RATE_LIMIT_MAX_TRANSACTIONS,
                       period=settings.CREDIT_CARD_RATE_LIMIT_PERIOD)

def download_receipt(request, pk, secret):
    if request.method != 'GET':
        raise Http404
    try:
        receipt = Receipt.objects.get(pk=pk, secret=secret)
    except Receipt.DoesNotExist:
        return HttpResponseRedirect('/pledge')
    response = HttpResponse(open(receipt.pdf_receipt_location).read(),
                            content_type='application/pdf')
    response['Content-Disposition'] = 'attachment; filename="EAA_Receipt_{0}.pdf"'.format(receipt.pk)
    return response

class PledgeJS(View):
    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(PledgeJS, self).dispatch(request, *args, **kwargs)

    @xframe_options_exempt
    def get(self, request):
        with open(os.path.join(settings.BASE_DIR, 'react/build/webpack-stats.json')) as f:
            data = json.load(f)

        time_last_compiled = datetime.datetime.utcfromtimestamp(data['endTime'] / 1000)
        time_last_compiled.timetuple()

        url = data['chunks']['donation_form'][0]['publicPath']

        response = HttpResponseRedirect(url, content_type="application/x-javascript")
        response['Last-Modified'] = time.strftime('%a, %d %b %Y %H:%M:%S GMT', time_last_compiled.timetuple())
        return response

class PledgeView(View):
    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(PledgeView, self).dispatch(request, *args, **kwargs)

    @xframe_options_exempt
    def get(self, request):
        charity = request.GET.get('charity')
        pin_environment = settings.PIN_DEFAULT_ENVIRONMENT
        pin_key = settings.PIN_ENVIRONMENTS[pin_environment].get('key')
        return render(request, 'donation_form.html', {'charity': charity, 'pin_key': pin_key,
                                                      'pin_environment': pin_environment})

    @xframe_options_exempt
    def post(self, request):
        body = json.loads(request.body.decode('utf-8'))
        pledge_form = PledgeForm(body)
        component_formset = PledgeComponentFormSet(body)

        if not (pledge_form.is_valid() and component_formset.is_valid()):
            client.captureMessage(str(pledge_form.errors) + str(component_formset.errors))
            return JsonResponse({
                'error_message': "There was a problem submitting your donation. Please contact info@eaa.org.au if problems persist."
            }, status=400)

        total = sum([component.instance.amount for component in component_formset.forms])
        if total < 2:
            return JsonResponse({
                'error_message': "The minimum donation possible is $2.",
            }, status=400)

        with django_transaction.atomic():
            pledge = pledge_form.save()
            for component in component_formset.forms:
                component.instance.pledge = pledge
            component_formset.save()

        response_data = {}

        if pledge.payment_method == PaymentMethod.BANK:
            response_data['bank_reference'] = pledge.generate_reference()
            send_bank_transfer_instructions_task.delay(pledge.id)
            return JsonResponse(response_data)

        elif pledge.payment_method == PaymentMethod.CREDIT_CARD:
            ip = get_ip(request)
            if not rate_limiter.checked_insert(ip) and not settings.DEBUG and settings.CREDIT_CARD_RATE_LIMIT_ENABLED:
                client.captureMessage('Hit rate limiter for ip: %s' % ip)
                return JsonResponse({
                    'error_message': "Our apologies: credit card donations are currently unavailable. "
                                     "Please try again tomorrow or make a payment by bank transfer.",
                }, status=400)

            if pledge.amount > 6000:
                client.captureMessage('User attempted credit card donation over $6K')
                return JsonResponse({
                    'error_message': "Our apologies: we can only accept credit card donations of up to $6000 AUD.  "
                                     "Please use bank transfer or make multiple smaller donations.",
                }, status=400)

            pin_data = body.get('pin_response')
            pin_data['amount'] = pledge.amount
            pin_data['pledge'] = pledge.id
            pin_form = PinTransactionForm(pin_data)
            if not pin_form.is_valid():
                client.captureMessage(str(pin_form.errors))
                return JsonResponse({
                    'error_message': "There was a problem submitting your donation. Please contact info@eaa.org.au if problems persist."
                }, status=400)

            transaction = pin_form.save()
            transaction.process_transaction()
            if transaction.succeeded:
                Receipt.objects.create_from_pin_transaction(transaction)
                response_data['succeeded'] = True
                receipt = transaction.receipt_set.first()
                response_data['receipt_url'] = reverse('download-receipt',
                                                       kwargs={'pk': receipt.pk, 'secret': receipt.secret})
                return JsonResponse(response_data)
            else:
                pin_repsonse_dict = json.loads(transaction.pin_response_text)
                client.captureMessage(pin_repsonse_dict['error_description'])
                return JsonResponse({
                    'error_message': "There was a problem submitting your donation. Please contact info@eaa.org.au if problems persist.",
                }, status=400)

        else:
            raise StandardError('We currently only support new donations via credit card or bank transfer.')

class PledgeViewOld(View):
    @xframe_options_exempt
    def post(self, request):
        form = PledgeFormOld(request.POST)
        amount = request.POST.get('amount')
        partner_id = request.POST.get('recipient_org')
        partner_slug = PartnerCharity.objects.get(id=partner_id).slug_id
        components_data = {'form-TOTAL_FORMS': 1, 'form-INITIAL_FORMS': 1, 'form-0-id': None,
                           'form-0-amount': amount, 'form-0-partner_charity': partner_slug}
        component_formset = PledgeComponentFormSet(components_data)

        if form.is_valid() and component_formset.is_valid():
            pledge = form.save()
            for component in component_formset.forms:
                component.instance.pledge = pledge
            component_formset.save()
        else:
            return JsonResponse({
                'error': 'form-error',
                'form_errors': form.errors
            }, status=400)

        pledge = form.instance
        if pledge.recurring:
            pledge.recurring_frequency = RecurringFrequency.MONTHLY
            pledge.save()
        payment_method = request.POST.get('payment_method')
        response_data = {'payment_method': payment_method}

        if int(payment_method) == 1:
            # bank transaction
            response_data['bank_reference'] = pledge.generate_reference()
            send_bank_transfer_instructions_task.delay(pledge.id)
            return JsonResponse(response_data)
        elif int(payment_method) == 3:
            # Rate limiting
            ip = get_ip(request)
            if not rate_limiter.checked_insert(ip):
                # Pretend it's a PIN error to save us from handling it separately in the javascript.
                return JsonResponse({
                    'error': 'pin-error',
                    'pin_response': "Our apologies: credit card donations are currently unavailable. " +
                                    "Please try again tomorrow or make a payment by bank transfer.",
                    'pin_response_text': '',
                }, status=400)

            transaction = PinTransaction()
            transaction.card_token = request.POST.get('card_token')
            transaction.ip_address = request.POST.get('ip_address')
            transaction.amount = pledge.amount  # Amount in dollars. Define with your own business logic.
            transaction.currency = 'AUD'  # Pin supports AUD and USD. Fees apply for currency conversion.
            transaction.description = 'Donation to Effective Altruism Australia'  # Define with your own business logic
            transaction.email_address = pledge.email
            transaction.pledge = pledge
            transaction.save()
            transaction.process_transaction()  # Typically "Success" or an error message
            if transaction.succeeded:
                Receipt.objects.create_from_pin_transaction(transaction)
                response_data['succeeded'] = True
                receipt = transaction.receipt_set.first()
                response_data['receipt_url'] = reverse('download-receipt',
                                                       kwargs={'pk': receipt.pk, 'secret': receipt.secret})
                return JsonResponse(response_data)
            else:
                return JsonResponse({
                    'error': 'pin-error',
                    'pin_response': transaction.pin_response,
                    'pin_response_text': transaction.pin_response_text,
                }, status=400)

    @xframe_options_exempt
    def get(self, request):

        form = PledgeFormOld()

        return render(request, 'pledge.html', {
            'form': form,
            'charity_database_ids': PartnerCharity.get_cached_database_ids(),
        })


import os
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5 import uic
from PIL import Image
import numpy as np

import sqlite3,cv2, time
detector = cv2.CascadeClassifier('haarcascade_frontalface_alt2.xml')
QTDesignerClass = uic.loadUiType("untitled.ui")[0]
class Ui_Form(QtWidgets.QWidget,QTDesignerClass):
    def __init__(self):
        super().__init__()

        self.setupUi(self)
        self.pushButton.clicked.connect(self.into)

        self.frame=QtWidgets.QLabel(self)
        self.frame.resize(400, 300)
        self.frame.setScaledContents(True)
        self.frame.move(5,5)
        self.save=0

        self.timer= QtCore.QTimer()
        self.timer.timeout.connect(self.nextFrameSlot)
        self.timer.start(1000/5)
        self.sample=41
        self.cpt = cv2.VideoCapture(0)

    def closeEvent(self, event):
        self.timer.stop()
        self.cpt.release()

    def nextFrameSlot(self):

        _,self.cam=self.cpt.read()
        gray=cv2.cvtColor(self.cam, cv2.COLOR_BGR2GRAY)
        self.faces=detector.detectMultiScale(gray, 1.3,5)
        for(x,y,w,h) in self.faces:
            cv2.rectangle(self.cam,(x,y),(x+w,y+h),(0,0,255),2)
            if(self.sample<20):
                cv2.imwrite("dataSet/User."+str(self.profile)+"."+str(self.sample)+".jpg",gray[y:y+h,x:x+w])
                self.sample=self.sample+1
                if(self.sample==20):
                    self.save=1
            cv2.waitKey(100)

        if(self.save==1):
            recognizer= cv2.face.LBPHFaceRecognizer_create()
            path='dataSet'

            faces=[]
            IDs=[]

            #read images only matching last number of id
            for f in os.listdir(path):
                imagePath=os.path.join(path,f)

                ID=int(os.path.split(imagePath) [1].split('.')[1])
                if(ID==self.profile):
                    faceImg=Image.open(imagePath).convert('L')
                    faceNp=np.array(faceImg,'uint8')
                    faces.append(faceNp)
                    IDs.append(ID)
                    print("Export comlete!")

            recognizer.train(faces, np.array(IDs))
            recognizer.save('recognizer/trainingData.%d.yml' %(self.profile))
            print("save comlete!")
            self.save=0

        self.cam=cv2.cvtColor(self.cam, cv2.COLOR_BGR2RGB)
        img=QtGui.QImage(self.cam, self.cam.shape[1], self.cam.shape[0], QtGui.QImage.Format_RGB888)
        self.pix=QtGui.QPixmap.fromImage(img)
        self.frame.setPixmap(self.pix)

    def into(self):
        self.sample=0

        conn = sqlite3.connect("text.db")
        arr = [(self.lineEdit.text(),self.lineEdit_2.text(),self.lineEdit_3.text(),self.lineEdit_4.text())]

        with conn:
            cur = conn.cursor()
            sql = "insert into Employee(name, gender, age, department) values(?, ?, ?, ?)"
            cur.executemany(sql, arr)
            sql = "select * from employee"
            cur.execute(sql)
            rows = cur.fetchall()
            count ="SELECT COUNT(*) FROM Employee"
            cur.execute(count)
            count_R=cur.fetchone() # select last row
            print(count_R)
            count=count_R[0] # first index of last row
            #                 row     index
            self.profile=rows[count-1][0]

            conn.commit()
        self.lineEdit.clear()
        self.lineEdit_2.clear()
        self.lineEdit_3.clear()
        self.lineEdit_4.clear()


# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and

import mock

from st2tests.base import BaseActionTestCase
from openpyxl.worksheet.worksheet import Worksheet
from openpyxl.cell.cell import Cell

class ExcelBaseActionTestCase(BaseActionTestCase):
    """Base class for testing the Excel pack. Includes MockWorkbook that
    mimics an Excel workbook as an array
       of cells. Only those methods needed for the tests are mocked - it is
       not a full mock of the openpyxl
       classes.
    """
    __test__ = False

    class MockCell(Cell):
        """MockCell class just overrides repr"""

        def __init__(self, worksheet, row=None, column=None, value=None,
                     style_array=None):
            super(ExcelBaseActionTestCase.MockCell, self).__init__(worksheet,
                                                                   row, column,
                                                                   value,
                                                                   style_array)

        def __repr__(self):
            return "<Cell {}>".format(self._value)

    class MockSheet(Worksheet):
        def __init__(self, sheetname, rows, **kwargs):
            ''' Mocks worksheet as an array of rows, each row is an array
                of mock cells taken from value in rows array
                :param  sheetname: Name of sheet
                :param  rows: Array of rows, each row is array of strings for
                cell values
            '''
            super(ExcelBaseActionTestCase.MockSheet, self).__init__(**kwargs)
            self.title = sheetname
            self.mockrows = []
            for i in range(len(rows)):
                newrow = []
                for j in range(len(rows[i])):
                    newrow.append(self._createCell(
                        row=i, column=j,
                        value=rows[i][j]))
                self.mockrows.append(newrow)

        def _createCell(self, row, column, value=None):
            cell = ExcelBaseActionTestCase.MockCell(self, row=row,
                                                    column=column, value=value)
            return cell

        def delete_rows(self, idx, amount=1):
            for i in range(idx - 1, idx + amount - 1):
                self.mockrows.pop(i)

        def cell(self, row, column, value=None):
            # Returns cell, or creates one if doesn't exist
            rowfound = None
            cellfound = None
            if row <= len(self.mockrows):
                rowfound = self.mockrows[row - 1]
            if not rowfound:
                for i in range(len(self.mockrows), row):
                    self.mockrows.append([])
                rowfound = self.mockrows[row - 1]
            if column <= len(rowfound):
                cellfound = rowfound[column - 1]
            if not cellfound:
                cellfound = self._createCell(
                    row=row, column=column, value=value)
                # Put empty cells before our column
                if column > (len(rowfound) + 1):
                    for i in range(column - len(rowfound) - 1):
                        rowfound.append(
                            mock.Mock(Cell(self, row, i, value=None)))
                rowfound.append(cellfound)
            return cellfound

    class MockWorkbook(object):
        def __init__(self, worksheets, new_sheet, **kwargs):
            """ Represents workbook as a dictionary of sheets -> MockWorksheet
                :param worksheets: Dictionary of sheetname to array of array
                of strings
                :param new_sheet: """
            # Convert dictionary of sheetname -> array of values to
            # dictionary of sheetname -> MockSheet
            self._mocksheets = {}
            self.encoding = "utf-8"
            for sheetname in worksheets.keys():
                self._mocksheets[
                    sheetname] = ExcelBaseActionTestCase.MockSheet(
                    sheetname, worksheets[sheetname],
                    parent=self)

        @property
        def sheetnames(self):
            return self._mocksheets.keys()

        def get_sheet_by_name(self, sheet_name):
            return self._mocksheets[sheet_name]

        def create_sheet(self, sheet_name):
            """ Creates new empty sheet for sheet_name"""
            self._mocksheets[sheet_name] = ExcelBaseActionTestCase.MockSheet(
                sheet_name, [], parent=self)
            return self._mocksheets[sheet_name]

    def setUp(self):
        super(ExcelBaseActionTestCase, self).setUp()

    @staticmethod
    def mock_file_exists(filename):
        """ Mocks os_file_exists such that returns True to file exists,
        but False for existence of lock file"""
        if "lock" in filename:
            return False
        else:
            return True

    @staticmethod
    def _util_get_column(rows, column):
        """ Returns array of values for column specified from specified array
        of array of values
            :param rows: Array of array of strings, representing the
            rows/columns of spreadsheet
            :param column: Column to return from array (starting at 0) """
        values = []
        for i in range(len(rows)):
            values.append(rows[i][column].value)
        return values


from setuptools import setup, find_packages
from pip.req import parse_requirements
from os.path import join

install_reqs = parse_requirements('requirements.txt', session=False)
reqs = [str(ir.req) for ir in install_reqs]

NAME = "blocklister"
DESCRIPTION = "Create Mikrotik Firewall address-lists and offer them via http"
AUTHOR = "Frank Lazzarini"
AUTHOR_EMAIL = "flazzarini@gmail.com"
VERSION = open(join(NAME, 'version.txt')).read().strip()
LONG_DESCRIPTION = open("README.rst").read()

setup(
    name=NAME,
    version=VERSION,
    description=DESCRIPTION,
    long_description=LONG_DESCRIPTION,
    author=AUTHOR,
    author_email=AUTHOR_EMAIL,
    license="Private",
    include_package_data=True,
    install_requires=reqs,
    entry_points={
        'console_scripts': [
            'blocklister-updater=blocklister.updater:run'
        ]
    },
    dependency_links=[],
    packages=find_packages(exclude=["tests.*", "tests"]),
    zip_safe=False,
)


import os, sys, math, time
from collections import deque
from multiprocessing import Lock
import numpy, scipy, scipy.spatial.distance, scipy.misc, scipy.cluster.hierarchy
import pandas
import argparse

# Main callback when new social relations and person tracks are available
def detectGroups(trackedPersons, socialRelations, timestamp):
    # Collect track positions in matrix (rows = tracks, cols = x and y coordinates)
    #trackCount = len(trackedPersons.pedID.unique())
    trackCount = len(trackedPersons)

    # Create a lookup from arbitrary track IDs to zero-based track indices in distances matrix
    trackIndex = 0
    trackIdToIndex = dict()
    for _, trackedPerson in trackedPersons.iterrows():
        trackIdToIndex[trackedPerson.pedID] = trackIndex
        trackIndex += 1

    # Build full (symmetric) distances matrix from provided person relations. Diagonal is initialized with zeroes.
    socialRelationsMatrix = numpy.ones( (trackCount, trackCount) )
    for _, socialRelation in socialRelations.iterrows():
        try:
            index1 = trackIdToIndex[socialRelation.trackID1]
            index2 = trackIdToIndex[socialRelation.trackID2]
        except KeyError:
            print("Key error while looking up tracks %d and %d!" % (socialRelation.trackID1, socialRelation.trackID2) )
            continue

        # Using min() here in case there are multiple types of relations per track pair -- in that case, use strongest type of relation
        socialRelationsMatrix[index1, index2] = min(socialRelationsMatrix[index1, index2], 1.0 - socialRelation.strength) # strong relation --> small distance
        socialRelationsMatrix[index2, index1] = min(socialRelationsMatrix[index2, index1], 1.0 - socialRelation.strength)

    for i in range(0, trackCount):
        socialRelationsMatrix[i,i] = 0.0 # diagonal elements have to be zero

    # Convert into condensed form
    trackDistances = scipy.spatial.distance.squareform(socialRelationsMatrix, force='tovector')

    # Cluster
    relationThreshold = 0.75 # relation strength above which we consider tracks to be in a group
    groupIndices = cluster(trackCount, trackDistances, relationThreshold) # outputs one group index per track

    #import pdb; pdb.set_trace()
    # Create groups, by assembling one set of track IDs per group
    groups = createGroups(groupIndices, trackedPersons)

    # Associate newly created groups with groups from previous cycle for ID consistency, and generate TrackedGroup instances
    trackedGroups = trackGroups(groups, trackedPersons, timestamp)
    return trackedGroups

### Creates group indices by single-linkage clustering
def cluster(trackCount, trackDistances, threshold):
    # Safety check if there are less than 2 tracks, otherwise clustering fails
    if trackCount == 0:
        return []
    elif trackCount == 1:
        return [0]

    # Perform clustering
    linkage = scipy.cluster.hierarchy.linkage(trackDistances, method='single')
    groupIndices = scipy.cluster.hierarchy.fcluster(linkage, threshold, 'distance')
    return groupIndices

# Generates a set of track IDs per group
def createGroups(groupIndices, tracks):
    groups = dict()
    trackIndex = 0
    for groupIndex in groupIndices:
        if not groupIndex in groups:
            groups[groupIndex] = []

        trackId = tracks.iloc[trackIndex].pedID
        groups[groupIndex].append(trackId)
        trackIndex += 1
    return groups

# For remembering recent group ID assignments for a given set of track IDs
class GroupIdAssignment(object):
    def __init__(self, trackIds, groupId, createdAt):
        self.trackIds = set(trackIds)
        self.groupId = groupId
        self.createdAt = createdAt

    def __str__(self):
        return "%s = %d" % (str(list(self.trackIds)), self.groupId)

# For publishing consecutive group IDs, even if internally they are not consecutive
class GroupIdRemapping(object):
    def __init__(self, groupId, publishedGroupId):
        self.originalGroupId = groupId
        self.publishedGroupId = publishedGroupId

def remapGroupId(groupId):
    publishedGroupId = None
    for groupIdRemapping in trackGroups.groupIdRemapping:
        if groupIdRemapping.originalGroupId == groupId:
            publishedGroupId = groupIdRemapping.publishedGroupId
            break
    if publishedGroupId is None:
        trackGroups.largestPublishedGroupId += 1
        publishedGroupId = trackGroups.largestPublishedGroupId

    trackGroups.groupIdRemapping.append( GroupIdRemapping(groupId, publishedGroupId) )
    return publishedGroupId

class BoundingBox():
    def __init__(self, x, y, height, width):
        self.x = x
        self.y = y
        self.height = height
        self.width = width

class TrackedGroup():
    def __init__(self, groupID, pedIDs, x, y):
        self.groupID = groupID
        self.pedIDs = pedIDs
        self.x = x
        self.y = y
        self.bbox = None

    def __str__(self):
        out = [str(self.groupID), str(self.x), str(self.y)]
        return ",".join(out)

class GroupCentroid():
    def __init__(self, pos=0, groupId=0, size=0):
        self.pos = pos
        self.groupId = groupId
        self.size = size

### Calculates group centroids for the current groups
def calculateGroupCentroids(groups, trackedPersons):
    # Get track positions
    trackPositionsById = dict()
    for track in trackedPersons.tracks:
        pos = track.pose.pose.position
        trackPositionsById[track.track_id] = [ pos.x, pos.y ]

    # Determine centroids
    centroids = dict()
    for groupId, track_ids in groups.iteritems():
        positions = numpy.zeros( (len(track_ids), 2) )
        trackIndex = 0
        for track_id in track_ids:
            positions[trackIndex, :] = trackPositionsById[track_id]
            trackIndex += 1

        currentCentroid = GroupCentroid()
        currentCentroid.pos = numpy.mean(positions, 0)
        currentCentroid.groupId = groupId
        currentCentroid.size = len(track_ids)
        centroids[groupId] = currentCentroid
    return centroids

# Associates current groups with previously tracked groups via track IDs
def trackGroups(groups, trackedPersons, timestamp):
    # Initialize variables
    publishSinglePersonGroups = False   # add to parameters of this script
    trackedGroups = []
    assignedGroupIds = []

    # Sort groups by smallest track ID per group to ensure reproducible group ID assignments
    #sortedGroups = sorted(groups.iteritems(), key=lambda (clusterId, track_ids) : sorted(track_ids)[0])
    #import pdb; pdb.set_trace()
    sortedGroups = sorted(groups.items(), key=lambda group : sorted(group[1])[0])

    # Used to calculate group centroids
    trackPositionsById = dict()
    for _, trackedPerson in trackedPersons.iterrows():
        trackPositionsById[trackedPerson.pedID] = numpy.array([ trackedPerson.x, trackedPerson.y ])

    # Create a TrackedGroup for each group, and assign a unique ID
    for clusterId, track_ids in sortedGroups:
        # Check if we encountered this combination of track IDs, or a superset thereof, before
        bestGroupIdAssignment = None

        trackIdSet = set(track_ids)
        for groupIdAssignment in trackGroups.groupIdAssignmentMemory:
            if groupIdAssignment.trackIds.issuperset(trackIdSet) or groupIdAssignment.trackIds.issubset(trackIdSet):
                trackCount = len(groupIdAssignment.trackIds)
                bestTrackCount = None if bestGroupIdAssignment is None else len(bestGroupIdAssignment.trackIds)

                if bestGroupIdAssignment is None or trackCount > bestTrackCount or (trackCount == bestTrackCount and groupIdAssignment.createdAt < bestGroupIdAssignment.createdAt):
                    if groupIdAssignment.groupId not in assignedGroupIds:
                        bestGroupIdAssignment = groupIdAssignment

        groupId = None
        if bestGroupIdAssignment is not None:
            groupId = bestGroupIdAssignment.groupId

        if groupId == None or groupId in assignedGroupIds:
            groupId = trackGroups.largestGroupId + 1 # just generate a new ID

        # Remember that this group ID has been used in this cycle
        assignedGroupIds.append(groupId)

        # Remember which group ID we assigned to this combination of track IDs
        groupIdAssignmentsToRemove = []
        groupExistsSince = timestamp
        for groupIdAssignment in trackGroups.groupIdAssignmentMemory:
            if set(track_ids) == groupIdAssignment.trackIds:
                groupExistsSince = min(groupIdAssignment.createdAt, groupExistsSince)
                groupIdAssignmentsToRemove.append(groupIdAssignment)  # remove any old entries with same track IDs

        for groupIdAssignment in groupIdAssignmentsToRemove:
            trackGroups.groupIdAssignmentMemory.remove(groupIdAssignment)

        trackGroups.groupIdAssignmentMemory.append( GroupIdAssignment(track_ids, groupId, groupExistsSince) )

        # Remember largest group ID used so far
        if(groupId > trackGroups.largestGroupId):
            trackGroups.largestGroupId = groupId

        # Do not publish single-person groups if not desired
        if publishSinglePersonGroups or len(track_ids) > 1:
            accumulatedPosition = numpy.array([0.0, 0.0])
            activeTrackCount = 0
            for track_id in track_ids:
                if track_id in trackPositionsById:
                    accumulatedPosition += trackPositionsById[track_id]
                    activeTrackCount += 1

            trackedGroup = TrackedGroup(groupID=remapGroupId(groupId),
                                        pedIDs=track_ids,
                                        x=accumulatedPosition[0] / float(activeTrackCount),
                                        y=accumulatedPosition[1] / float(activeTrackCount))
            trackedGroups.append(trackedGroup)

    # Return currently tracked groups
    return trackedGroups

def calcBoundingBoxes(trackedGroups, trackedPersons):
    #import pdb; pdb.set_trace()
    for group in trackedGroups:
        xs = []
        ys = []
        for ped in group.pedIDs:
            currPed = trackedPersons.loc[trackedPersons.pedID == ped]
            x = currPed.iloc[0].x
            y = currPed.iloc[0].y
            w = currPed.iloc[0].w
            h = currPed.iloc[0].h

            xs.extend([x, x+w])
            ys.extend([y, y+h])

        X = min(xs)
        Y = min(ys)
        height = max(ys) - min(ys)
        width = max(xs) - min(xs)
        group.bbox = BoundingBox(round(X,3), round(Y,3), round(height,3), round(width,3))

def parseArguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input_file")
    parser.add_argument("--social_relations")
    parser.add_argument("--output_file")

    return parser.parse_args()

### Main method
def main():
    args = parseArguments()

    df_social_relations = pandas.read_csv(args.social_relations)
    df_tracking = pandas.read_csv(args.input_file)

    #df_tracking = df_tracking.loc[((df_tracking.dataset == 'test') & (df_tracking.segment_num == 1))]
    #df_tracking = df_tracking.loc[(df_tracking.dataset == 'test')]

    header = "frameID,groupID,x,y,w,h"
    with open(args.output_file, "w") as file:
        file.write(header + "\n")

    header1 = "frameID, groupID, pedIDs"
    ped_outfile = "data/detected_groups_test2_pedIDs.csv"
    with open(ped_outfile, "w") as file:
        file.write(header1 + "\n")

    trackGroups.largestGroupId = -1
    trackGroups.groupIdAssignmentMemory = deque(maxlen=300)
    trackGroups.groupIdRemapping = deque(maxlen=50)
    trackGroups.largestPublishedGroupId = -1

    # detect groups frame by frame
    frames = df_tracking.frameID.unique()
    timestamp = 0
    '''
    n = 4
    for f in frames[::n]:
        f_end = f + n - 1
        trackedPersons = df_tracking.loc[((df_tracking.frameID >= f) & (df_tracking.frameID <= f_end))]
        socialRelations = df_social_relations.loc[((df_social_relations.frameID >= f) & (df_social_relations.frameID <= f_end))]
    '''
    for f in frames:
        trackedPersons = df_tracking.loc[(df_tracking.frameID == f)]
        socialRelations = df_social_relations.loc[(df_social_relations.frameID == f)]
        # get groups in frame
        groups = detectGroups(trackedPersons, socialRelations, timestamp)
        calcBoundingBoxes(groups, trackedPersons)

        # write groups to outfile
        with open(args.output_file, "a+") as file:
            for g in groups:
                out = ','.join([str(f),
                                str(g.groupID),
                                str(g.bbox.x),
                                str(g.bbox.y),
                                str(g.bbox.width),
                                str(g.bbox.height)])
                file.write(out + "\n")
        with open(ped_outfile, "a+") as file:
            for g in groups:
                pedIDs = ','.join([str(x) for x in g.pedIDs])
                out = ','.join([str(f),
                                str(g.groupID),
                                pedIDs])
                file.write(out + "\n")

        if f % 1000 == 0:
            from datetime import datetime
            now = datetime.now().time() # time object
            print(f,now)
        timestamp += 1

### Entry point
if __name__ == '__main__':
    main()


import cv2
import numpy as np
import matplotlib.pyplot as plt

def crop(frame,left,top,w,h):
    return frame[top:top+h,left:left+w]

def main():

    w = 800
    h = 600

    left=0
    top=300
    height=250
    width=800
    font = cv2.FONT_HERSHEY_SIMPLEX

    cap = cv2.VideoCapture(0)

    cap.set(3, w)
    cap.set(4, h)

#    print(cap.get(3))
#    print(cap.get(4))

    if cap.isOpened():
        ret, frame = cap.read()
    else:
        ret = False

    ret, frame1 = cap.read()
    ret, frame2 = cap.read()
    cars=0.0

    while ret:
        cars+=0.01
        txt_title1="Cars counter : "+str(int(cars))
        d = cv2.absdiff(frame1, frame2)
        grey = cv2.cvtColor(d, cv2.COLOR_BGR2GRAY)
        blur = cv2.GaussianBlur(grey, (5, 5), 0)
        ret, th = cv2.threshold( blur, 20, 255, cv2.THRESH_BINARY)
        dilated = cv2.dilate(th, np.ones((3, 3), np.uint8), iterations=1 )
        #eroded = cv2.erode(blur, np.ones((3, 3), np.uint8), iterations=1 )
        eroded = cv2.erode(dilated, np.ones((3, 3), np.uint8), iterations=1 )
        c, h = cv2.findContours(eroded, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        cv2.drawContours(frame1, c, -1, (0, 25, 230), 2)

        eroded=crop(eroded,left,top,width,height)
        eroded=cv2.GaussianBlur(eroded, (25, 25), 0)
        frame1 = cv2.rectangle(frame1,(0,0),(800,10),(100,10,120),100)

        cv2.putText(frame1,txt_title1,(10,50), font, 1,(255,255,255),2,cv2.LINE_AA)

        cv2.imshow("Work",eroded)
        cv2.imshow("input", frame1)
        #plt.imshow(frame1,cmap='gray')
        if cv2.waitKey(1) == 27: # exit on ESC
            break

        frame1 = frame2
        ret, frame2 = cap.read()

    cv2.destroyAllWindows()
    cap.release()

if __name__ == "__main__":
    main()


import hashlib
import json

import requests
from bs4 import BeautifulSoup

def get_playlist(url):
    # richiedo la playlist sul sito
    playlist_request = requests.get(url)
    document = BeautifulSoup(playlist_request.content)

    # prendo gli id delle canzoni (nell'attributo rel)
    ids = [int(id['rel'][0]) for id in document.select('li.item li.play a')]
    title = document.select('div.media h1')[0].next_element

    return {'ids': ids, 'title': title}

def get_song(id, folder, number):
    # richiedo info sulle singole tracce
    track_info_request = requests.post('http://www.rockit.it/web/include/ajax.play.php',
                                       data={'id': id, '0k': 'ok'})
    track_info = json.loads(track_info_request.text)

    # elaboro rockitID
    _m = hashlib.md5()
    _m.update(track_info['url'] + '-secondomeno')  # si ottiene decompilando l'swf
    hashed = _m.hexdigest()

    # richiedo l'mp3
    mp3_request = requests.post(track_info['url'], data={'rockitID': hashed},
                                headers={'Referer': 'http://www.rockit.it/web/js/playswf/new0716299.swf'})

    # salvo in cartella/00 nomecanzone.mp3
    with open(u'%s/%s %s.mp3' % (folder, str(number).zfill(2), track_info['title']), 'wb') as f:
        f.write(mp3_request.content)


import random
import datetime
import json
airline = ['国航', '东方航空', '福州航空', '厦门航空','南方航空', '深圳航空', '山东航空', '春秋', '海南', '祥鹏航空','成都航空','长龙航空', '南方']

citys = ['深圳', '天津','南昌', "宜昌",'西安' ,'上海虹桥', '贵阳',  '北京大兴' , '黄山' ,'三亚' ,'郑州','成都' ,'温州', '青岛', '重庆' ,'昆明', '常州', '海口', '遵义' ,'杭州']

flights = []
for i in range(300):
    al = airline[random.randint(0,12)]
    begin_place = citys[random.randint(0,19)]
    end_place = begin_place
    while begin_place == end_place:
        end_place = citys[random.randint(0,19)]

    begin_time = datetime.datetime.now().timestamp() + random.randint(0,21168000)
    end_time = random.randint(3600, 36000) + begin_time
    id = i + 1
    tickets = random.randint(30,200)
    remain = tickets - random.randint(5, tickets - 15)
    flight = {
        'id': id,
        'airline': al,
        'flight_number': str(random.randint(100000, 999999)),
        'begin_place': begin_place,
        'end_place': end_place,
        'begin_time': begin_time,
        'end_time': end_time,
        'fly_by': '[]',
        'tickets': tickets,
        'remain': remain,
        'created_time': datetime.datetime.now().timestamp(),
        'user_list': '[]',
        'valid': 1,
        'appointment': '[]',
        'price': random.randint(200,2000),
    }
    flights.append(flight)

with open('ticket.db', 'w') as f:
    f.write(json.dumps(flights))


import numpy as np
import scipy
import pandas as pd
from matplotlib import pyplot as plt
import wavio

from datetime import datetime, time, date, timedelta, MINYEAR
from signal_lib import *

import sys

def main_():
    # read wav file and get sampling rate and data
    fs, data = wavfile.read('data/piano_c5.wav')
    duration = len(data)/fs
    print(duration)
    # number of frames
    frames = 30*duration
    print(frames)
    # reshape the data per frame
    data_per_frame = np.array_split(data, frames)
    # return fft
    def generate(data, fs):
        # fft
        fft = np.absolute( np.fft.rfft(data,  n = len(data)))
        # get frequencies
        freq = np.fft.fftfreq(len(fft), d = 1.0/fs)
        # return fft and freq
        return fft, freq

    fft, freq = generate(data_per_frame[0], fs)
    print(fft)
    print(freq)
    print(fft.shape, freq.shape)
    plt.plot(freq, fft)
    plt.show()

    # create figure
    fig, ax = plt.subplots()
    line_plot, = ax.plot(freq, fft)
    ax.set_xlabel("Frequencies")
    ax.set_ylabel("Magnitude")

    # function to initialize frame
    def init():
        data, _ = generate(data_per_frame[0], fs)
        line_plot.set_ydata( data )
        return line_plot,

    def animate(i):
        # change the data for the spectrum at time i, and change title to time i
        data, _ = generate( data_per_frame[ i % len(data_per_frame) ], fs )
        line_plot.set_ydata( data )
        return line_plot,
    # create animation
    ani = animation.FuncAnimation( fig, animate, init_func=init, interval=100/6, blit=True, save_count=50, repeat_delay = 1000)
    #ani.save('movie.mp4')
    plt.show()

# create datafrme from wavio object
def to_signal(wavio_object, microsecond=0):
    # get data
    data = wavio_object.data.copy()
    # get index
    # base start
    base = datetime.today()
    #base = base.replace(hour = 0, second=0, minute=0, microsecond=microsecond)
    # increase timedelta
    index = [base, ]
    # for each value
    for i in range(1, wavio_object.data.shape[0]):
        # add to index the last element + timedelta
        index.append( index[-1] + timedelta(milliseconds=1000.0/wavio_object.rate) )
    return [ Signal(pd.Series(data[:, i])) for i in range(data.shape[1]) ]

def main(path = "../../data/8bit-C4.wav"):
    wav_control = wavio.read(path)
    print(wav_control)
    # creat signal
    signals = to_signal(wav_control)
    signal = signals[0]
    signal.plot()
    #signal.fft().abs().plot()

if __name__ == "__main__":
    # get args
    args = sys.argv
    # get path
    file = "8bit-C4.wav" if len(args) == 1 else args[1]
    main("../../data/" + file)


#!/usr/bin/env python3

import pytest
import freezerbox
import stepwise

class MockReagent(freezerbox.Reagent):
    tag_prefix = 'x'

class MockMolecule(freezerbox.Molecule):
    tag_prefix = 'x'

    def _calc_mw(self):
        raise freezerbox.QueryError

class MockNucleicAcid(freezerbox.NucleicAcid):
    tag_prefix = 'x'

class MockSoloMaker:

    @classmethod
    def make(cls, db, products):
        yield from (cls(x) for x in products)

    def __init__(self, product):
        args = product.maker_args

        self.products = [product]

        if 'protocol' in args:
            self.protocol = stepwise.Protocol(steps=args['protocol'])

        if 'deps' in args:
            self.dependencies = args['deps']
        else:
            self.dependencies = []

        if 'seq' in args:
            self.product_seqs = [args['seq']]

        if 'molecule' in args:
            self.product_molecule = args['molecule']

        if 'conc' in args:
            self.product_conc = stepwise.Quantity.from_string(args['conc'])

        if 'circular' in args:
            self.is_product_circular = freezerbox.parse_bool(args['circular'])

class MockComboMaker:

    @classmethod
    def make(cls, db, products):
        yield cls(list(products))

    def __init__(self, products):
        from more_itertools import flatten

        steps = list(flatten(
            x.maker_args.get('protocol', [])
            for x in products
        ))
        deps = list(flatten(
            x.maker_args.get('deps', [])
            for x in products
        ))

        self.products = products
        self.protocol = stepwise.Protocol(steps=steps)
        self.dependencies = deps
class MockEntryPoint:

    def __init__(self, plugin):
        self.plugin = plugin

    def load(self):
        return self.plugin

MockMaker = MockSoloMaker

@pytest.fixture
def mock_plugins(monkeypatch):
    from string import ascii_lowercase
    monkeypatch.setattr(freezerbox.model, 'MAKER_PLUGINS', {
        **freezerbox.model.MAKER_PLUGINS,
        'mock': MockEntryPoint(MockSoloMaker),
        'merge': MockEntryPoint(MockComboMaker),
        **{k: MockEntryPoint(MockSoloMaker) for k in ascii_lowercase},
    })


import numpy as np
import sys
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.model_selection import train_test_split
from tqdm import tqdm
import time
import logreg
import rf
import xgb

models = [logreg, rf, xgb]

def main():
    data_dir = sys.argv[1]
    train_file = sys.argv[2]
    num_features = int(sys.argv[3])

    #Open training file
    train_file = os.path.join(data_dir, train_file)
    X_raw_train=[]
    y_train=[]
    print("Reading from train_file", train_file)
    fp=open(train_file, 'r')
    for line in fp:
        (label, sentence) = line.strip().split('\t')
        X_raw_train.append(sentence)
        y_train.append(int(label))
    fp.close()
    print(len(X_raw_train), len(y_train))
    train = None
    test = None
    X_raw_train = X_raw_train[:train]
    y_train = y_train[:train]
    X_all = X_raw_train
    y_all = y_train
    X_raw_train, X_raw_test, y_train, y_test = train_test_split(X_all, y_all, test_size=0.20, random_state=42)

    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5)
    X_train = vectorizer.fit_transform(X_raw_train)

    ch2 = SelectKBest(chi2, k=num_features)
    X_train = ch2.fit_transform(X_train, y_train)
    feature_names =vectorizer.get_feature_names()
    feature_names = [feature_names[i] for i in ch2.get_support(indices=True)]
    for fname in feature_names:
        print(fname, occurance(fname, X_raw_train), occurance(fname, X_raw_test))
        pass

    X_train =feature_transform(X_raw_train, feature_names)
    X_test =feature_transform(X_raw_test, feature_names)
    X_train = np.array(X_train)
    X_test = np.array(X_test)
    print("Shape of train=", X_train.shape, "Shape of test=", X_test.shape)
    for model in models:
        model.fit_predict(X_train, y_train, X_test, y_test)

def occurance(word, sentences):
    return sum([s.lower().count(word) for s in sentences])

def feature_transform(X_train, feature_names):
    zero_count = 0
    X_transformed = []
    for X in X_train:
        features=[0 for i in feature_names]
        features = [ X.count(f) for f in feature_names]
        X_transformed.append(features)
        if sum(features)==0:
            zero_count += 1
    print("Zero count instances=", zero_count)

    return X_transformed

if __name__ == "__main__":
    main()


import os
import sys
import pytest
from unittest.mock import MagicMock, call
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + "/../python")
import python.infotiv_launcher
from python.infotiv_launcher import RotationCMD

@pytest.fixture()
def launcher():
    print('\n*********Start*********')
    launcher = python.infotiv_launcher.Launcher()
    launcher.rc = MagicMock()
    yield launcher
    print('\n**********End**********')

# ---------------------------------------------------------------------------------
# -------------------------set_rotation_position-----------------------------------
# ---------------------------------------------------------------------------------
def test_encoders_ready_check_encoders_not_ready(launcher):
    # GIVEN
    launcher.encoders_ready = 0

    # WHEN
    return_value = launcher.encoder_ready_check()

    # THEN
    assert return_value == 0

def test_set_rotation_position_encoders_not_ready(launcher):
    # GIVEN
    with pytest.raises(Exception) as err:
        launcher.encoders_ready = 0
    # WHEN
        launcher.set_rotation_position(1)
    # THEN
    assert err.match('Encoder Not Ready')

@pytest.mark.parametrize("invalid_data", [(-1), 131])
def test_set_rotation_position_invalid_data_type_error(launcher, invalid_data):
    # GIVEN
    with pytest.raises(Exception, match='out of bounds') as err:
        launcher.encoders_ready = 1
    # WHEN
        launcher.set_rotation_position(invalid_data)
    # THEN
    assert err.type is ValueError

def test_set_rotation_position_zero_increment_negative(launcher):
    # GIVEN
    launcher.encoders_ready = 1
    launcher.rc.ReadEncM2.return_value = (1, 2, 2)

    # WHEN
    launcher.set_rotation_position(0)

    # THEN
    calls = [call(launcher.address, -launcher.rotation_speed_pulses, 2, 1), call(launcher.address, 0, 0, 0)]
    launcher.rc.SpeedDistanceM2.assert_has_calls(calls)
    assert launcher.rc.SpeedDistanceM2.call_count == 2

def test_set_rotation_position_higher_than_zero_increment_positive(launcher):
    # GIVEN
    launcher.encoders_ready = 1
    launcher.rc.ReadEncM2.return_value = (1, 4, 2)

    # WHEN
    launcher.set_rotation_position(1)

    # THEN
    calls = [call(launcher.address, launcher.rotation_speed_pulses, 2634, 1), call(launcher.address, 0, 0, 0)]
    launcher.rc.SpeedDistanceM2.assert_has_calls(calls)
    assert launcher.rc.SpeedDistanceM2.call_count == 2

def test_set_rotation_position_max_increment_positive(launcher):
    # GIVEN
    launcher.encoders_ready = 1
    launcher.rc.ReadEncM2.return_value = (1, -1.5, 2)

    # WHEN
    launcher.set_rotation_position(130)

    # THEN
    calls = [call(launcher.address, launcher.rotation_speed_pulses, 343056.5, 1), call(launcher.address, 0, 0, 0)]
    launcher.rc.SpeedDistanceM2.assert_has_calls(calls)
    assert launcher.rc.SpeedDistanceM2.call_count == 2

# ---------------------------------------------------------------------------------
# -------------------------rotation_control----------------------------------------
# ---------------------------------------------------------------------------------
def test_rotation_control_right(launcher):
    # GIVEN
    launcher.rc = MagicMock()
    # WHEN
    launcher.rotation_control(RotationCMD(1))
    # THEN
    launcher.rc.ForwardM1.assert_called_with(launcher.address_2, launcher.rotation_speed_manual)
    launcher.rc.ForwardM2.assert_called_with(launcher.address_2, launcher.rotation_speed_manual)
    launcher.rc.BackwardM1.assert_not_called()

def test_rotation_control_left(launcher):
    # GIVEN
    launcher.rc = MagicMock()
    # WHEN
    launcher.rotation_control(RotationCMD(2))
    # THEN
    launcher.rc.BackwardM1.assert_called_with(launcher.address_2, launcher.rotation_speed_manual)
    launcher.rc.BackwardM2.assert_called_with(launcher.address_2, launcher.rotation_speed_manual)
    launcher.rc.ForwardM1.assert_not_called()

def test_rotation_control_stop(launcher):
    # GIVEN
    launcher.rc = MagicMock()
    # WHEN
    launcher.rotation_control(RotationCMD(3))
    # THEN
    launcher.rc.ForwardM1.assert_called_with(launcher.address_2, 0)
    launcher.rc.ForwardM2.assert_called_with(launcher.address_2, 0)
    launcher.rc.BackwardM1.assert_not_called()


from src.runners.run_parameters import RunParameters
from collections import OrderedDict

class SearcherRunParamsProvider:
    def __init__(self, param_ranges: OrderedDict, default_parameter_factory):
        self.param_factory = default_parameter_factory
        self.param_ranges = param_ranges
        self.counters = [0 for _ in self.param_ranges]
        self.complete = False

    def __iter__(self):
        self.counters = [0 for _ in self.param_ranges]
        self.complete = False
        return self

    def __next__(self):
        if self.complete:
            raise StopIteration
        params = self.__create_run_params()
        last = self.__increment_counter(0)
        if last:
            self.complete = True
        print(f"Returning params: {params.all_params}")
        return params

    def __len__(self):
        acc = 1
        for x in self.param_ranges.values():
            acc = acc * len(x)
        return acc

    def __create_run_params(self):
        params: RunParameters = self.param_factory()
        search_overrides = {}
        for i, key in enumerate(self.param_ranges.keys()):
            param_value = self.param_ranges[key][self.counters[i]]
            search_overrides[key] = param_value

        params.apply_overrides(search_overrides)
        return params

    def __increment_counter(self, index):
        if len(self.counters) == index:
            return True
        param_ranges_at_index = self.param_ranges[list(self.param_ranges.keys())[index]]

        if self.counters[index] >= (len(param_ranges_at_index) - 1):
            # overflow in the counter
            self.counters[index] = 0
            return self.__increment_counter(index + 1)
        else:
            self.counters[index] = self.counters[index] + 1
            return False

class SearcherRunParamsProviderBuilder:

    def __init__(self, default_parameter_factory):
        self.param_factory = default_parameter_factory
        self.param_ranges = OrderedDict()

    def add_param(self, name, param_range):
        self.param_ranges[name] = param_range
        return self

    def build(self) -> SearcherRunParamsProvider:
        return SearcherRunParamsProvider(self.param_ranges, self.param_factory)


from django.views.generic import FormView, View
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm, AuthenticationForm
from django.contrib.auth import login, logout
from django.shortcuts import render, redirect
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse_lazy
from django.http import HttpResponse

from .models import File, UserFileRelationship
from .tools import get_hash, check_quota

import os

class RegisterFormView(FormView):
    form_class = UserCreationForm
    success_url = reverse_lazy('index')
    template_name = "register.html"

    def form_valid(self, form):
        form.save()

        return super(RegisterFormView, self).form_valid(form)

class LoginFormView(FormView):
    form_class = AuthenticationForm
    success_url = reverse_lazy('lobby')
    template_name = "index.html"

    def form_valid(self, form):
        self.user = form.get_user()

        login(self.request, self.user)
        return super(LoginFormView, self).form_valid(form)

class LogoutView(View):
    @staticmethod
    def get(request):
        logout(request)

        return redirect('index')

@login_required
def lobby(request):
    files = UserFileRelationship.objects.filter(user=request.user)
    return render(request, 'lobby.html', {'files': files})

@login_required
def upload_file(request):
    """
    Trying to upload file. Getting md5 sum of it, if file with the same hash exists,
    putting link in the service table on it. Otherwise creating new File object.
    File upload quota = 100 files for each user
    """
    if request.method == 'POST':
        if check_quota(request):
            md5sum = get_hash(request.FILES.get('datafile',))
            try:
                file_obj = File.objects.get(md5sum=md5sum)
            except ObjectDoesNotExist:
                file_obj = File.objects.create(file_info=request.FILES['datafile'],
                                               md5sum=md5sum)
            UserFileRelationship.objects.create(user=request.user,
                                                file_info=file_obj,
                                                file_name=request.FILES['datafile'].name)
        return redirect('lobby')
    return render(request, 'upload_file.html')

@login_required
def delete_file(request, file_id):
    """
    Delete file method. If file had more than 1 link in the service table deleting only this link.
    Otherwise we remove link on it in service table, record from File table and remove file from cloud storage.
    """
    query = UserFileRelationship.objects.get(pk=file_id)
    if UserFileRelationship.objects.filter(file_info=query.file_info).count() > 1:
        query.delete()
    else:
        file_obj = File.objects.get(id=query.file_info_id)
        path = str(file_obj.file_info)
        os.remove(path)
        file_obj.delete()
    return redirect('lobby')

@login_required
def get_download_link(request, file_id):
    query = UserFileRelationship.objects.get(pk=file_id)
    file_obj = File.objects.get(id=query.file_info_id)
    md5sum = file_obj.md5sum

    return render(request, 'download_file.html',  {'md5sum': md5sum, 'file': file_obj})

def download(request, md5sum):
    file_obj = File.objects.get(md5sum=md5sum)

    filename = file_obj.file_info.name.split('/')[-1]
    response = HttpResponse(file_obj.file_info, content_type='text/plain')
    response['Content-Disposition'] = 'attachment; filename=%s' % filename

    return response


from django.shortcuts import render
import subprocess
import pandas as pd
import json
from django.http import JsonResponse
import random

# Create your views here.

csv_path = "/home/micron/perf/demo.csv"
csv_path_2 = "/home/micron/perf/hammerdb.txt"

last_data = 0
def demo(request):
    context = {}
    return (render(request, 'demo/linely_demo.html', context))

def get_data_demo(request):
    # Get first value from file (header).
    cmd = "head -1 "+ csv_path
    first_data = subprocess.check_output('{}'.format(cmd), shell=True)
    first_data = first_data.decode("utf-8").replace("\n", "0").replace('"', "")
    first_data = first_data.split(",")

    # Get last value from file (most recent data).
    cmd = "tail -1 "+ csv_path
    last_data = subprocess.check_output('{}'.format(cmd), shell=True)
    last_data = last_data.decode("utf-8").replace("\n", "0")
    last_data = last_data.split(",")
    last_data = [int(x) for x in last_data]

    # Put into dataframe and rename columns.
    df = pd.DataFrame(data=last_data)
    df = df.transpose()
    df.columns = first_data

    # Node (processor socket).
    N_val = "0"
    # HomeAgent or Group of Memory Channels.
    H_val = "1"
    # Channel Number within Home Agent.
    C_val = "1"

    # Calculating values needed for plotting.
    # read_bw = (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[1]-Read"] * 64)
    # write_bw = (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[2]-Write"] * 64)
    read_bw = (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[1]-Read"] * 4) / (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[Fixed]-Clks"])
    write_bw = (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[2]-Write"] * 4) / (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[Fixed]-Clks"])
    # read_bw = (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[1]-Read"])
    # write_bw = (df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[2]-Write"])

    errors = float(df["iMC:N" + N_val + "_H" + H_val + "_C" + C_val + "[4]-Errs"].tolist()[0])

    # # Calculating SET per second.
    # cmd = "tail -1 "+ csv_path_2+"redis-perf-set.csv"
    # db_set = subprocess.check_output('{}'.format(cmd), shell=True)
    # db_set = db_set.decode("utf-8")
    # db_set = db_set.split(",")[1].strip()
    # db_set = float(db_set.replace('\"',""))
    #
    #
    # # Calculating SET per second.
    # cmd = "tail -1 "+ csv_path_2+"redis-perf-get.csv"
    # db_get = subprocess.check_output('{}'.format(cmd), shell=True)
    # db_get = db_get.decode("utf-8")
    # db_get = db_get.split(",")[1].strip()
    # db_get = float(db_get.replace('\"',""))

    # try:
    #     cmd = "redis-benchmark -q -c 50 -d 2 -n 5000 -t SET --csv"
    #     db_set = subprocess.check_output('{}'.format(cmd), shell=True)
    #     db_set = db_set.decode("utf-8")
    #     db_set = db_set.split(",")[1].strip()
    #     db_set = float(db_set.replace('\"', ""))
    #     last_data = db_set
    # except:
    #     db_set = last_data

    # cmd = "redis-benchmark -q -c 50 -d 2 -n 10000 -t GET --csv"
    # db_get = subprocess.check_output('{}'.format(cmd), shell=True)
    # db_get = db_get.decode("utf-8")
    # db_get = db_get.split(",")[1].strip()
    # db_get = float(db_get.replace('\"', ""))

    # Get db value from file (tail).
    cmd = "tail -1 "+ csv_path_2
    db_set = subprocess.check_output('{}'.format(cmd), shell=True)
    db_set = int(db_set.strip())

    context = {
        'read_bw': 100*float(read_bw),
        'write_bw': 100*float(write_bw),
        'tps': db_set/1000,
        'errors': int(errors)
    }
    return JsonResponse(json.loads(json.dumps(context)))


import datetime
import json_tricks
import os
from os.path import (abspath, basename, dirname, exists, expanduser, isdir,
                     join, realpath, splitext)
import re
import shutil
import sys

from traits.api import (Any, Dict, Enum, HasTraits, Instance, List, Long,
                        Str)
from whoosh import fields, qparser, query

from .media import Media
from .directory import Directory
from . import processor

if sys.version_info[0] > 2:
    unicode = str
INT = fields.NUMERIC(numtype=int)
FLOAT = fields.NUMERIC(numtype=float)

def get_project_dir():
    d = expanduser(join('~', '.vixen'))
    if not isdir(d):
        os.makedirs(d)
    return d

def get_file_saved_time(path):
    dt = datetime.datetime.fromtimestamp(os.stat(path).st_ctime)
    return dt.ctime()

class TagInfo(HasTraits):
    name = Str
    type = Enum("string", "int", "float", "bool")
    default = Any

    def __repr__(self):
        return 'TagInfo(%r, %r)' % (self.name, self.type)

    def _default_default(self):
        map = {"string": "", "int": 0, "float": 0.0, "bool": False}
        return map[self.type]

def open_file(fname_or_file, mode='r'):
    if hasattr(fname_or_file, 'read'):
        return fname_or_file
    else:
        return open(fname_or_file, mode)

def sanitize_name(name):
    name = name.lower()
    name = re.sub(r'\s+', '_', name)
    return re.sub(r'\W+', '', name)

def get_non_existing_filename(fname):
    if exists(fname):
        base, ext = splitext(basename(fname))
        return join(dirname(fname), base + '_a' + ext)
    else:
        return fname

COMMON_TAGS = dict(
    type='string', file_name='string', path='string',
    ctime='string', mtime='string', size='int'
)

def _cleanup_query(q, tag_types):
    type_map = dict(float=FLOAT.from_bytes, int=INT.from_bytes)
    for term in q.leaves():
        if isinstance(term, query.Term):
            if isinstance(term.text, (str, unicode)):
                fieldtype = tag_types[term.fieldname]
                if fieldtype in type_map:
                    term.text = type_map[fieldtype](term.text)
                else:
                    term.text = term.text.lower()
        elif isinstance(term, query.Phrase):
            term.words = [x.lower() for x in term.words]

def _check_value(value, expr):
    if isinstance(expr, (str, unicode)):
        return expr in value.lower()
    else:
        return expr == value

def _check_range(x, term):
    result = True
    if term.start is not None:
        if term.startexcl:
            result &= x > term.start
        else:
            result &= x >= term.start
    if term.end is not None and result:
        if term.endexcl:
            result &= x < term.end
        else:
            result &= x <= term.end
    return result

def _check_date_range(x, term):
    result = True
    if term.startdate is not None:
        result &= x >= term.startdate
    if term.enddate is not None and result:
        result &= x <= term.enddate
    return result

def _get_tag(media, attr):
    if attr in COMMON_TAGS:
        return getattr(media, attr)
    else:
        return media.tags.get(attr)

def _search_media(expr, media):
    if expr.is_leaf():
        if isinstance(expr, query.Term):
            attr = expr.fieldname
            return _check_value(_get_tag(media, attr), expr.text)
        elif isinstance(expr, query.Phrase):
            attr = expr.fieldname
            text = " ".join(expr.words)
            return _check_value(_get_tag(media, attr), text)
        elif isinstance(expr, query.DateRange):
            value = media._ctime if expr.fieldname == 'ctime' else media._mtime
            return _check_date_range(value, expr)
        elif isinstance(expr, query.NumericRange):
            attr = expr.fieldname
            return _check_range(_get_tag(media, attr), expr)
        else:
            print("Unsupported term: %r" % expr)
            return False
    else:
        if isinstance(expr, query.And):
            result = True
            for child in expr.children():
                result &= _search_media(child, media)
                if not result:
                    break
            return result
        elif isinstance(expr, query.Or):
            result = False
            for child in expr.children():
                result |= _search_media(child, media)
                if result:
                    break
            return result
        elif isinstance(expr, query.Not):
            subquery = list(expr.children())[0]
            return not _search_media(subquery, media)
        else:
            print("Unsupported term: %r" % expr)
            return False

class Project(HasTraits):
    name = Str
    description = Str
    path = Str
    root = Instance(Directory)
    tags = List(TagInfo)

    media = Dict(Str, Media)

    extensions = List(Str)

    processors = List(processor.FactoryBase)

    number_of_files = Long

    # Path where the project data is saved.
    save_file = Str

    last_save_time = Str

    _query_parser = Instance(qparser.QueryParser)

    def add_tags(self, tags):
        tags = list(self.tags) + tags
        self.update_tags(tags)

    def update_tags(self, new_tags):
        old_tags = self.tags
        new_tag_names = set(tag.name for tag in new_tags)
        tag_info = dict((tag.name, tag.type) for tag in old_tags)
        removed = []
        added = []
        for tag in new_tags:
            if tag.name not in tag_info:
                added.append(tag)
            elif tag_info[tag.name] != tag.type:
                removed.append(tag)
                added.append(tag)
        for tag in old_tags:
            if tag.name not in new_tag_names:
                removed.append(tag)
        self.tags = new_tags

        for m in self.media.values():
            for tag in removed:
                del m.tags[tag.name]
            for tag in added:
                m.tags[tag.name] = tag.default

        self._query_parser = self._make_query_parser()

    def export_csv(self, fp, cols=None):
        """Export metadata to a csv file.  If `cols` are not specified,
        it writes out all the useful metadata.

        Parameters
        -----------

        path: str: a path to the csv file to dump.
        cols: sequence: a sequence of columns to write.
        """
        lines = []
        data = []
        all_keys = set()
        for key in sorted(self.media.keys()):
            item = self.media[key]
            d = item.flatten()
            all_keys.update(d.keys())
            data.append(d)

        all_keys -= set(('_ctime', '_mtime'))
        if cols is None:
            cols = all_keys
            cols = list(sorted(cols))
        # Write the header.
        lines.append(','.join(cols))
        # Assemble the lines.
        for d in data:
            line = []
            for key in cols:
                elem = d[key]
                if isinstance(elem, str):
                    elem = '"%s"' % elem
                else:
                    elem = str(elem) if elem is not None else ""
                line.append(elem)
            lines.append(','.join(line))

        # Write it out.
        of = open_file(fp, 'w')
        for line in lines:
            of.write(line + '\n')
        of.close()

    def load(self, fp=None):
        """Load media info from opened file object.
        """
        if fp is None:
            if not exists(self.save_file):
                return
            fp = open_file(self.save_file)
        else:
            fp = open_file(fp)

        data = json_tricks.load(
            fp, preserve_order=False, ignore_comments=False
        )
        fp.close()
        self.name = data.get('name', '')
        self.description = data.get('description', '')
        self.path = data.get('path')
        self.tags = [TagInfo(name=x[0], type=x[1]) for x in data['tags']]
        self.processors = [processor.load(x)
                           for x in data.get('processors', [])]
        media = dict((key, Media(**kw)) for key, kw in data['media'])
        # Don't send object change notifications when this large data changes.
        self.trait_setq(media=media)
        root = Directory()
        root.__setstate__(data.get('root'))
        self.extensions = root.extensions
        self.root = root
        self.number_of_files = len(self.media)
        # This is needed as this is what makes the association from the media
        # to the file.
        self.scan()

    def save(self):
        """Save current media info to a file object
        """
        if len(self.save_file) > 0:
            self.save_as(self.save_file)
            self._update_last_save_time()
        else:
            raise IOError("No valid save file set.")

    def save_as(self, fp):
        """Save copy to specified path.
        """
        fp = open_file(fp, 'w')
        media = [(key, m.to_dict()) for key, m in self.media.items()]
        tags = [(t.name, t.type) for t in self.tags]
        root = self.root.__getstate__()
        processors = [processor.dump(x) for x in self.processors]
        data = dict(
            version=1, path=self.path, name=self.name,
            description=self.description, tags=tags, media=media,
            root=root, processors=processors
        )
        json_tricks.dump(data, fp)
        fp.close()

    def scan(self, refresh=False):
        """Find all the media recursively inside the root directory.
        This will not clobber existing records but will add any new ones.
        """
        media = self.media
        new_media = {}
        self._setup_root()
        default_tags = dict((ti.name, ti.default) for ti in self.tags)

        def _scan(dir):
            for f in dir.files:
                m = media.get(f.relpath)
                if m is None:
                    m = self._create_media(f, default_tags)
                    new_media[f.relpath] = m
                f.media = m
                if refresh:
                    m.update()
            for d in dir.directories:
                if refresh:
                    d.refresh()
                _scan(d)

        if refresh:
            self.root.refresh()
        _scan(self.root)

        if len(new_media) > 0:
            # This is done because if media is changed, a trait change notify
            # will be sent to listeners with a very large amount of data
            # potentially.  The jigna webserver will marshal all the keys
            # and not be able to send the information.
            media_copy = dict(media)
            media_copy.update(new_media)
            self.trait_setq(media=media_copy)
            self.number_of_files = len(self.media)

    def search(self, q):
        try:
            parsed_q = self._query_parser.parse(q)
        except Exception:
            print("Invalid search expression: %s" % q)
            return
        tag_types = self._get_tag_types()
        _cleanup_query(parsed_q, tag_types)
        for relpath, m in self.media.items():
            if _search_media(parsed_q, m):
                yield m

    def refresh(self):
        self.scan(refresh=True)

    # #### Private protocol ################################################

    def _create_media(self, f, default_tags):
        m = Media.from_path(f.path)
        m.tags = dict(default_tags)
        return m

    def _setup_root(self):
        path = abspath(expanduser(self.path))
        root = self.root
        if root is None or realpath(root.path) != realpath(path):
            self.root = Directory(path=path, extensions=self.extensions)

    def _tags_default(self):
        return [TagInfo(name='completed', type='bool')]

    def _save_file_default(self):
        if len(self.name) > 0:
            fname = sanitize_name(self.name) + '.vxn'
            d = get_project_dir()
            return get_non_existing_filename(join(d, fname))
        else:
            return ''

    def _update_last_save_time(self):
        self.last_save_time = get_file_saved_time(self.save_file)

    def _last_save_time_default(self):
        if exists(self.save_file):
            return get_file_saved_time(self.save_file)

    def _name_changed(self, name):
        if len(name) > 0:
            old_save_file = self.save_file
            old_dir = dirname(old_save_file)
            new_save_file = join(old_dir, sanitize_name(name) + '.vxn')
            if new_save_file != old_save_file:
                self.save_file = new_save_file
                if exists(old_save_file):
                    shutil.move(old_save_file, self.save_file)

    def _extensions_changed(self, ext):
        if self.root is not None:
            self.root.extensions = ext

    def _extensions_items_changed(self):
        if self.root is not None:
            self.root.extensions = self.extensions

    def _get_tag_types(self):
        result = dict(COMMON_TAGS)
        result.update(dict((t.name, t.type) for t in self.tags))
        return result

    def _make_schema(self):
        from whoosh.fields import BOOLEAN, DATETIME, TEXT, Schema
        kw = dict(
            type=TEXT, file_name=TEXT, path=TEXT,
            mtime=DATETIME, ctime=DATETIME, size=INT
        )
        type_to_field = dict(
            string=TEXT, int=INT, float=FLOAT, bool=BOOLEAN
        )
        for tag in self.tags:
            kw[tag.name] = type_to_field[tag.type]

        return Schema(**kw)

    def _make_query_parser(self):
        schema = self._make_schema()
        qp = qparser.QueryParser('path', schema=schema)
        qp.add_plugin(qparser.GtLtPlugin())
        from whoosh.qparser.dateparse import DateParserPlugin
        qp.add_plugin(DateParserPlugin())
        return qp

    def __query_parser_default(self):
        return self._make_query_parser()