import mysql.connector
import requests
import json

# Ready config file
config = json.load(open('Connection/config.json'))
db = config['connections']['local']

# Connect
con = mysql.connector.connect(**db)

select_stm = con.cursor()
update_stm = con.cursor()
select_stm.execute(
    "SELECT id,lat,lng,address,cep FROM supermarketsv2 where id > 510 and bairro is null")
supermarkets = select_stm.fetchall()

# Ready Keys
keys = json.load(open('files/api_keys.json'))
geoKeys = keys['connections']['freeKeys']

cont = 0
key = 0
zip_code = None
street = None
neighborhood = None
city = None
state = None
formatted_address = None
lat = None
lng = None

for supermarket in supermarkets:
    response = requests.get(
        'https://maps.googleapis.com/maps/api/geocode/json?address=%s,%s&region=br&language=pt-BR&key=%s' %
        (supermarket[1], supermarket[2], geoKeys[key]))
    responseJson = json.loads(response.text)

    if responseJson['status'] == 'OVER_QUERY_LIMIT':
        key += 1
        print('You have exceeded your daily request quota for this API.')
        print('Refazendo consulta!')
        response = requests.get(
            'https://maps.googleapis.com/maps/api/geocode/json?address=%s&language=pt-BR&region=br&key=%s' %
            (supermarket[4], geoKeys[key]))
        responseJson = json.loads(response.text)

    if responseJson['status'] == 'OK':
        addresses = responseJson['results'][0]['address_components']
        for address in addresses:
            # Street
            if 'route' in address['types']:
                street = address['long_name']
            else:
                street = None
            # Neighborhood
            if 'administrative_area_level_4' in address['types'] or 'sublocality' in address['types']:
                neighborhood = address['long_name']
            else:
                neighborhood = None
            # City
            if 'administrative_area_level_2' in address['types']:
                city = address['long_name']
            else:
                city = None
            # State
            if 'administrative_area_level_1' in address['types']:
                state = address['short_name']
            else:
                state = None
            # Zip_code
            if 'postal' in address['types']:
                zip_code = address['short_name']
            else:
                zip_code = None

            formatted_address = responseJson['results'][0]['formatted_address']
            lat = responseJson['results'][0]['geometry']['location']['lat']
            lng = responseJson['results'][0]['geometry']['location']['lng']

            if state is not None:
                update_stm.execute("update supermarketsv2 SET estado='%s' where id=%s" %
                                   (state, supermarket[0]))
                con.commit()
    if responseJson['status'] == 'ZERO_RESULTS':
        print('Nenhum resultado.Supermercado Id:', supermarket[0])
        zip_code = None
        street = None
        neighborhood = None
        city = None
        state = None
        formatted_address = None
        lat = None
        lng = None


#!/usr/bin/env python3
#
# Script tomado de https://flask.palletsprojects.com/en/1.1.x/quickstart/
#
# Author: John Sanabria - john.sanabria@correounivalle.edu.co
# Date: 2020-03-28
#
from flask import Flask, request
from flask_restplus import Api, Resource, fields
from time import gmtime, strftime
import constant # constantes de este proyecto
import myfields # campos monitoreados
import time
import subprocess

flask_app = Flask(__name__)
app = Api(app = flask_app, version = "0.0.1", title = "COVID 19 Web Service", description = "Endpoint para consultar el avance del COVID 19 por pais" )

name_space = app.namespace('COVID19', description='Web Service COVID19')
model = app.model('Name Model',
                  {'name': fields.String(required = True,
                                         description="Name of the country",
                                         help="Name cannot be blank.")
                  })

def findCountryData(country):
	cmd =  "grep %s %s/%s | tail -n 1"%(country,constant.COVIDDIR,constant.FILENAME)
	output = subprocess.check_output(cmd, shell = True)[:-1]
	print("[%s]"%(output))
	output = output.decode().split(',')
	if len(output) < 2:
		return { "Status": "%s Not Found"%(country) }
	dictio = {}
	dictio['GMT time'] = time.strftime("%a, %d %b %Y %I:%M:%S %p %Z", time.gmtime())
	dictio['Server time'] = strftime("%a, %d %b %Y %I:%M:%S %p %Z")
	dictio[myfields.STDATETIME] = output[myfields.DATETIME]
	dictio[myfields.STCOUNTRY] = output[myfields.COUNTRY]
	dictio[myfields.STTCASES] = output[myfields.TCASES]
	dictio[myfields.STNCASES] = output[myfields.NCASES]
	dictio[myfields.STTDEATHS] = output[myfields.TDEATHS]
	dictio[myfields.STNDEATHS] = output[myfields.NDEATHS]
	dictio[myfields.STTRECOVERED] = output[myfields.TRECOVERED]
	dictio[myfields.STACASES] = output[myfields.ACASES]
	dictio[myfields.STSCRITICAL] = output[myfields.SCRITICAL]
	dictio[myfields.STCASEXMILLION] = output[myfields.CASEXMILLION]
	dictio[myfields.STDEATHSXMILLION] = output[myfields.DEATHSXMILLION]
	dictio[myfields.STFSTCASE] = output[myfields.FSTCASE]

	return dictio

@name_space.route("/<string:id>")
class MainClass(Resource):
	@app.doc(responses={200: 'OK', 400: 'Invalid Argument', 500: 'Mapping Key Error'}, params={'id': 'Country'})
	def get(self,id = 'Colombia'):
		return findCountryData(id)
	@app.expect(model)
	def post(self):
		countryName = request.json['name']
		return findCountryData(countryName)


import random
import requests

get = str(input("enter url of the img you want to download : "))

def download_image(url):
    print("downloading ...")
    name = random.randrange(1, 1000)
    full_name = str(name) + ".jpeg"
    r = requests.get(url, stream = True)
    with open (full_name, 'wb') as f:
        f.write(r.content)
    return "download completed and saved as :" + full_name

print(download_image(get))


import logging

from google.oauth2 import service_account
import tor_async_util

_logger = logging.getLogger(__name__)

class Config(tor_async_util.Config):
    """An abstract base call for all service's configuration classes."""

    """At the very least each service is expected to derive
    a class from this abstract base class and provide a
    real value for ```config_section```.
    """
    config_section = 'config_section'

    def __init__(self, config_file_name):
        tor_async_util.Config.__init__(self, config_file_name)

        self.logging_level = self.get_logging_level(
            self.config_section,
            'log_level')

        #
        # The docker image containing our spiders.
        #
        self.docker_image_name = self.get(
            self.config_section,
            'docker_image_name',
            'bindle/berry:latest')

        #
        # where to find kubectl proxy that allows services and tasks to
        # interact with the Kubernetes API Server
        #
        self.kubectl_proxy = self.get(
            self.config_section,
            'kubectl_proxy',
            'http://127.0.0.1:8001')

        #
        # connection to kubectl proxy timeout in seconds
        #
        self.kubectl_proxy_connect_timeout_in_seconds = self.get_int(
            self.config_section,
            'kubectl_proxy_connect_timeout_in_seconds',
            60)

        #
        # kubectl proxy request timeout in seconds
        #
        self.kubectl_proxy_request_timeout_in_seconds = self.get_int(
            self.config_section,
            'kubectl_proxy_request_timeout_in_seconds',
            5 * 60)

        self.max_concurrent_executing_http_requests = self.get_int(
            self.config_section,
            'max_concurrent_executing_http_requests',
            10)

        self.address = self.get(
            self.config_section,
            'address',
            '127.0.0.1')

        self.port = self.get_int(
            self.config_section,
            'port',
            80)

        #
        # The services that need to read and write crawl results to and
        # from the crawl results cache do so via the service's RESTful API.
        # The base url for this API is radically different in development
        # on a local machine vs when the service is running inside Kubernetes
        # and hence the motivation for this configuration.
        #
        # :TODO: don't like cloudfeaster namespace being here should
        # dynamically adapt to the namespace in which the service is
        # being run using the Kubernetes Downward API and an environment
        # variable.
        #
        self.base_crawl_results_cache_service_url = self.get(
            self.config_section,
            'base_crawl_results_cache_service_url',
            'http://crawl-results-cache.cloudfeaster.svc.cluster.local:8443')

        #
        # The services that need to read and write spider metadata to and
        # from the spider metadata cache do so via the service's RESTful API.
        # The base url for this API is radically different in development
        # on a local machine vs when the service is running inside Kubernetes
        # and hence the motivation for this configuration.
        #
        # :TODO: don't like cloudfeaster namespace being here should
        # dynamically adapt to the namespace in which the service is
        # being run using the Kubernetes Downward API and an environment
        # variable.
        #
        self.base_spider_metadata_cache_service_url = self.get(
            self.config_section,
            'base_spider_metadata_cache_service_url',
            'http://spider-metadata-cache.cloudfeaster.svc.cluster.local:8443')

        #
        # The services that need to read and write users to and
        # from the users service do so via the service's RESTful API.
        # The base url for this API is radically different in development
        # on a local machine vs when the service is running inside Kubernetes
        # and hence the motivation for this configuration.
        #
        # :TODO: don't like cloudfeaster namespace being here should
        # dynamically adapt to the namespace in which the service is
        # being run using the Kubernetes Downward API and an environment
        # variable.
        #
        self.base_users_service_url = self.get(
            self.config_section,
            'base_users_service_url',
            'http://users.cloudfeaster.svc.cluster.local:8443')

        #
        # The services that need to read and write crawl perf results to and
        # from the crawl perf results analysis service do so via the service's
        # RESTful API. The base url for this API is radically different in development
        # on a local machine vs when the service is running inside Kubernetes
        # and hence the motivation for this configuration.
        #
        # :TODO: don't like cloudfeaster namespace being here should
        # dynamically adapt to the namespace in which the service is
        # being run using the Kubernetes Downward API and an environment
        # variable.
        #
        self.base_crawl_perf_results_analysis_service_url = self.get(
            self.config_section,
            'base_crawl_perf_results_analysis_service_url',
            'http://crawl-perf-results-analysis.cloudfeaster.svc.cluster.local:8443')

        #
        # Per what is described in the "Make requests to the external services" section
        # of https://istio.io/docs/tasks/traffic-management/egress.html different
        # urls need to be used when using and when not using istio when making TLS
        # requests to services outside of the cluster. Service code uses the ```using_istio```
        # property to determine the appropriate URL to use/generate.
        #
        # :TODO: the above is not the best pattern but we'll use this approach as a
        # start to identify the service code that is affected by istio and then find
        # a clear approach.
        #
        self.using_istio = self.get_boolean(
            self.config_section,
            'using_istio',
            False)

    def get_google_cloud_service_account_credentials(self, section, option, value_if_not_found=None):
        """Creates and returns a Google Cloud Service Account's credentials
        which are read from the file pointed to by section & option.
        If something doesn't get found or an error occurs during credentials
        creation, then ```value_if_not_found``` is returned.
        """
        filename = self.get(section, option, None)
        if not filename:
            return value_if_not_found

        try:
            _logger.info(
                "attempting to read service account credentials from '%s'",
                filename)

            rv = service_account.Credentials.from_service_account_file(filename)

            _logger.info(
                "successfully read service account credentials from '%s'",
                filename)

            return rv
        except Exception as ex:
            _logger.error(
                "could not read service account credentials from '%s' - %s",
                filename,
                ex)

            return value_if_not_found


import os
import numpy as np
import cv2
from PIL import Image
from skimage.io import imread

import wwtool

Image.MAX_IMAGE_PIXELS = int(2048 * 2048 * 2048 // 4 // 3)

if __name__ == '__main__':
    xview_class_labels_file = 'tools/datasets/xview/xview_class_labels.txt'
    json_file = '/home/jwwangchn/Documents/100-Work/170-Codes/wwtool/data/xview/v0/xView_train.geojson'
    xview_parse = wwtool.XVIEW_PARSE(json_file, xview_class_labels_file)

    image_sets = ['train']
    image_format = '.tif'

    subimage_size = 800
    gap = 200

    for image_set in image_sets:
        image_path = './data/xview/v0/{}_images'.format(image_set)

        image_save_path = './data/xview/v1/{}/images'.format(image_set)
        wwtool.mkdir_or_exist(image_save_path)
        label_save_path = './data/xview/v1/{}/labels'.format(image_set)
        wwtool.mkdir_or_exist(label_save_path)

        for idx, image_name in enumerate(os.listdir(image_path)):
            print(idx, image_name)
            file_name = image_name.split(image_format)[0]
            image_file = os.path.join(image_path, file_name + image_format)

            img = imread(image_file)

            objects = xview_parse.xview_parse(image_name)
            bboxes = np.array([wwtool.xyxy2cxcywh(obj['bbox']) for obj in objects])
            labels = np.array([obj['label'] for obj in objects])

            subimages = wwtool.split_image(img, subsize=subimage_size, gap=gap)
            subimage_coordinates = list(subimages.keys())
            bboxes_ = bboxes.copy()
            labels_ = labels.copy()

            if bboxes_.shape[0] == 0:
                continue

            for subimage_coordinate in subimage_coordinates:
                objects = []

                bboxes_[:, 0] = bboxes[:, 0] - subimage_coordinate[0]
                bboxes_[:, 1] = bboxes[:, 1] - subimage_coordinate[1]
                cx_bool = np.logical_and(bboxes_[:, 0] >= 0, bboxes_[:, 0] < subimage_size)
                cy_bool = np.logical_and(bboxes_[:, 1] >= 0, bboxes_[:, 1] < subimage_size)
                subimage_bboxes = bboxes_[np.logical_and(cx_bool, cy_bool)]
                subimage_labels = labels_[np.logical_and(cx_bool, cy_bool)]

                if len(subimage_bboxes) == 0:
                    continue
                img = subimages[subimage_coordinate]
                if np.mean(img) == 0:
                    continue

                label_save_file = os.path.join(label_save_path, '{}__{}_{}.txt'.format(file_name, subimage_coordinate[0], subimage_coordinate[1]))
                image_save_file = os.path.join(image_save_path, '{}__{}_{}.png'.format(file_name, subimage_coordinate[0], subimage_coordinate[1]))
                cv2.imwrite(image_save_file, img)

                for subimage_bbox, subimage_label in zip(subimage_bboxes, subimage_labels):
                    subimage_objects = dict()
                    subimage_objects['bbox'] = wwtool.cxcywh2xyxy(subimage_bbox.tolist())
                    subimage_objects['label'] = subimage_label
                    objects.append(subimage_objects)
                wwtool.simpletxt_dump(objects, label_save_file)


import csv
import statistics
import numpy as np
import matplotlib.pyplot as plt

with open('activity.csv', 'r') as f:
    data = csv.reader(f)
    mydictWeekend = {}
    mydictWeekdays = {}
    weekendList = ('2012-10-06', '2012-10-07', '2012-10-13', '2012-10-14', '2012-10-20', '2012-10-21', '2012-10-27', '2012-10-28', '2012-11-03', '2012-11-04', '2012-11-10', '2012-11-11', '2012-11-17', '2012-11-18', '2012-11-24', '2012-11-25')

    for row in data:
        if row[1] in weekendList:
            if row[0] != 'NA' and row[0] != 'steps':
                if row[2] not in mydictWeekend:
                    mydictWeekend[row[2]] = [int(row[0])]
                else:
                    mydictWeekend[row[2]].append(int(row[0]))
        else:
            if row[0] != 'NA' and row[0] != 'steps':
                if row[2] not in mydictWeekdays:
                    mydictWeekdays[row[2]] = [int(row[0])]
                else:
                    mydictWeekdays[row[2]].append(int(row[0]))

pertimeWeekend = {}
mean_pertimeWeekend = {}

for key, value in mydictWeekend.items():
    pertimeWeekend[key] = sum(value)
    mean_pertimeWeekend[key] = round(statistics.mean(value),2)

plt.plot(list(mean_pertimeWeekend.keys()), list(mean_pertimeWeekend.values()))
plt.title('Weekends')
plt.xlabel('time')
plt.ylabel('steps each 5 time')
plt.show()

pertimeWeekdays = {}
mean_pertimeWeekdays = {}

for key, value in mydictWeekdays.items():
    pertimeWeekdays[key] = sum(value)
    mean_pertimeWeekdays[key] = round(statistics.mean(value),2)

plt.plot(list(mean_pertimeWeekdays.keys()), list(mean_pertimeWeekdays.values()))
plt.title('Weekdays')
plt.xlabel('time')
plt.ylabel('steps each 5 time')
plt.show()


import re
import base64
import requests
import tempfile
from datetime import timedelta
from odoo import models
from odoo.exceptions import ValidationError, UserError

class AccountMove(models.Model):
    _inherit = 'account.move'

    def validate_data_boleto(self):
        errors = []
        for invoice in self:
            if not invoice.payment_journal_id or not self.payment_journal_id.l10n_br_use_boleto_inter:
                continue

            partner = invoice.partner_id.commercial_partner_id
            if partner.is_company and not partner.l10n_br_legal_name:
                errors.append('Destinatário - Razão Social')
            if not partner.street:
                errors.append('Destinatário / Endereço - Rua')
            if not partner.l10n_br_number:
                errors.append('Destinatário / Endereço - Número')
            if not partner.zip or len(re.sub(r"\D", "", partner.zip)) != 8:
                errors.append('Destinatário / Endereço - CEP')
            if not partner.state_id:
                errors.append(u'Destinatário / Endereço - Estado')
            if not partner.city_id:
                errors.append(u'Destinatário / Endereço - Município')
            if not partner.country_id:
                errors.append(u'Destinatário / Endereço - País')
        if len(errors) > 0:
            msg = "\n".join(
                ["Por favor corrija os erros antes de prosseguir"] + errors)
            raise ValidationError(msg)

    def send_information_to_banco_inter(self):
        if not self.payment_journal_id or not self.payment_journal_id.l10n_br_use_boleto_inter:
            return

        for moveline in self.receivable_move_line_ids:
            acquirer = self.env['payment.acquirer'].search([('provider', '=', 'boleto-inter')])
            if not acquirer:
                raise UserError('Configure o modo de pagamento do Boleto Banco Inter')
            transaction = self.env['payment.transaction'].create({
                'acquirer_id': acquirer.id,
                'amount': moveline.amount_residual,
                'currency_id': moveline.move_id.currency_id.id,
                'partner_id': moveline.partner_id.id,
                'type': 'server2server',
                'date_maturity': moveline.date_maturity,
                'invoice_ids': [(6, 0, self.ids)],
            })

            url = "https://apis.bancointer.com.br/openbanking/v1/certificado/boletos"

            tipo_mora = "ISENTO"
            if self.payment_journal_id.l10n_br_valor_juros_mora:
                tipo_mora = "TAXAMENSAL"
            tipo_multa = "NAOTEMMULTA"
            if self.payment_journal_id.l10n_br_valor_multa:
                tipo_multa = "PERCENTUAL"  # Percentual

            partner_id = self.partner_id.commercial_partner_id
            vals = {
                "pagador":{
                    "cnpjCpf": re.sub("[^0-9]", "",partner_id.l10n_br_cnpj_cpf),
                    "nome": partner_id.l10n_br_legal_name or partner_id.name,
                    "email": self.partner_id.email or "",
                    "telefone":"",
                    "ddd":"",
                    "cep": re.sub("[^0-9]", "", partner_id.zip),
                    "numero": partner_id.l10n_br_number,
                    "complemento": partner_id.street2 or "",
                    "bairro": partner_id.l10n_br_district,
                    "cidade": partner_id.city_id.name,
                    "uf": partner_id.state_id.code,
                    "endereco": partner_id.street,
                    "tipoPessoa": "FISICA" if partner_id.company_type == "person" else "JURIDICA"
                },
                "dataEmissao": transaction.create_date.date().isoformat(),
                "seuNumero": transaction.reference,
                "dataVencimento": moveline.date_maturity.isoformat(),
                "mensagem": {
                    "linha1": "",
                },
                "desconto1":{
                    "codigoDesconto": "NAOTEMDESCONTO",
                    "taxa": 0,
                    "valor": 0,
                    "data": ""
                },
                "desconto2": {
                    "codigoDesconto": "NAOTEMDESCONTO",
                    "taxa": 0,
                    "valor": 0,
                    "data": ""
                },
                "desconto3":{
                    "codigoDesconto": "NAOTEMDESCONTO",
                    "taxa": 0,
                    "valor": 0,
                    "data": ""
                },
                "valorNominal": moveline.amount_residual,
                "valorAbatimento": 0,
                "multa": {
                    "codigoMulta": tipo_multa,
                    "data": (moveline.date_maturity + timedelta(days=1)).isoformat(),
                    "taxa": self.payment_journal_id.l10n_br_valor_multa,
                    "valor": 0,
                },
                "mora": {
                    "codigoMora": tipo_mora,
                    "data": (moveline.date_maturity + timedelta(days=1)).isoformat(),
                    "taxa": self.payment_journal_id.l10n_br_valor_juros_mora,
                    "valor": 0,
                },
                "cnpjCPFBeneficiario": re.sub("[^0-9]", "", self.company_id.l10n_br_cnpj_cpf),
                "numDiasAgenda": "SESSENTA",
                "dataLimite": "SESSENTA",
            }
            headers = {
                "x-inter-conta-corrente": re.sub("[^0-9]", "", self.payment_journal_id.bank_account_id.acc_number)
            }
            cert = base64.b64decode(self.payment_journal_id.l10n_br_inter_cert)
            key = base64.b64decode(self.payment_journal_id.l10n_br_inter_key)

            cert_path = tempfile.mkstemp()[1]
            key_path = tempfile.mkstemp()[1]

            arq_temp = open(cert_path, "w")
            arq_temp.write(cert.decode())
            arq_temp.close()

            arq_temp = open(key_path, "w")
            arq_temp.write(key.decode())
            arq_temp.close()

            response = requests.post(url, json=vals, headers=headers, cert=(cert_path, key_path))
            if response.status_code == 200:
                json_p = response.json()
                nosso_numero = json_p["nossoNumero"]
            elif response.status_code == 401:
                raise UserError("Erro de autorização ao consultar a API do Banco Inter")
            else:
                raise UserError('Houve um erro com a API do Banco Inter:\n%s' % response.text)

            transaction.write({
                'acquirer_reference': nosso_numero,
            })

    def generate_boleto_banco_inter_transactions(self):
        for item in self:
            item.send_information_to_banco_inter()

    def action_post(self):
        self.validate_data_boleto()
        result = super(AccountMove, self).action_post()
        self.generate_boleto_banco_inter_transactions()
        return result


import json

import requests

groupUrl = "https://gitlab.com/api/v4/groups/{group}/projects?private_token={token}"
labelsUrl = "https://gitlab.com/api/v4/projects/{project_id}/labels?private_token={token}"

def main():
    print("Enter GitLab Private Token:")
    key = input()
    key = key.strip()

    print("Enter GitLab Group Name:")
    group = input()

    if (not group):
        print("group should not be empty")
        return

    group = group.replace("/","%2F")
    url = groupUrl.replace("{group}", group)

    url = url.replace("{token}", key)

    answer = requests.get(url)

    if (answer.status_code != 200):
        if('message' in answer):
            print(answer.json()['message'] + " for group: " + group)
        else:
            print(answer.json())
        return

    projects = answer.json()

    ids = []
    for count, project in enumerate(projects):
        name = "%2d %50s: %8d" % (count + 1, project['name'], project['id'])
        ids.append(project['id'])
        print(name)

    if (len(ids) < 1):
        print("No Repository found for group: " + group)
        return

    print("Choose Repository:")
    choosedId = int(input()) - 1
    repositoryId = ids[choosedId]
    url = labelsUrl.replace('{project_id}', str(repositoryId))
    url = url.replace("{token}", key)
    answer = requests.get(url)
    if (answer.status_code == 200):
        with open('labels.json') as file:
            labels = json.load(file)
        for label in labels:
            answer = requests.post(url, {"name": label[0], "color": label[1]}).json()
            if ('name' in answer or 'color' in answer):
                print("Label %10s with color %10s created" % (answer['name'], answer['color']))
            elif ('message' in answer):
                print(answer['message'] + ": " + label[0])
            else:
                print(answer)

if __name__ == "__main__":
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import keras
from keras.layers import Dense
import pickle
import numpy as np

BASE_DIR = '/data1/ceph/yzchen'

def unpickle(file):
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

def load_data(source):
    if source == 'cifar-10':
        heigth = 32
        width = 32
        channels = 3
        raw_data = unpickle(os.path.join(BASE_DIR, 'datasets/cf10/data_batch_1'))
        x_train = np.array(raw_data[b'data'])
        y_train = np.array(raw_data[b'labels'])
        for i in range(2,6):
            raw_data = unpickle(os.path.join(BASE_DIR, 'datasets/cf10/data_batch_'+str(i)))
            x_train = np.concatenate([x_train, np.array(raw_data[b'data'])])
            y_train = np.concatenate([y_train, np.array(raw_data[b'labels'])])
        x_train = x_train.reshape(x_train.shape[0], channels, heigth, width)\
            .transpose([0,2,3,1]).astype('float32') / 255.
        y_train = keras.utils.to_categorical(y_train)

        raw_data = unpickle(os.path.join(BASE_DIR, 'datasets/cf10/test_batch'))
        x_test = np.array(raw_data[b'data'])
        y_test = np.array(raw_data[b'labels'])
        x_test = x_test.reshape(x_test.shape[0], channels, heigth, width)\
                    .transpose([0,2,3,1]).astype('float32') / 255.
        y_test = keras.utils.to_categorical(y_test)

        return x_train, y_train, x_test, y_test

x_train, y_train, x_test, y_test = load_data('cifar-10')

model = keras.models.Sequential()
model.add(Dense(100, activation='relu', input_shape=(32*32*3,)))
model.add(Dense(100, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.summary()

model.compile(loss='categorical_crossentropy',
              optimizer='sgd',
              metrics=['accuracy'])

history = model.fit(x_train.reshape(-1,32*32*3), y_train, validation_split=0.1, batch_size=32, epochs=10)
print(model.evaluate(x_test.reshape(-1,32*32*3), y_test))


# -*- coding: utf-8 -*-
# Time: 2019/04/01 01:18
# Author: caiongbo
# File: main.py
# 功能：
'''
以中国新闻网社会新闻板块为抓取对象
http://www.chinanews.com/society.shtml
当前抓取了3月份所有数据新闻数据，后期进行数据分析
可以在news_spider.py中设置任意的时间段
'''

from multiprocessing import Pool
from news_spider import news, get_res, get_all_links, item_info, url_lists

def all_links():
    '''
    先从网页爬取所有的链接数据存入数据库，
    然后从数据库里读取所有的链接
    :return: 以所有链接的列表形式
    '''
    get_all_links()
    allLinks = []
    for data in url_lists.find():
        allLinks.append(data['links'])
    return  allLinks

if __name__ == '__main__':
    pool = Pool()  # 创建多线程池
    pool.map(get_res, all_links())


from dxl.learn.test import TestCase

# from dxl.learn.zoo.incident.data import create_dataset, dataset_db

import os
path_db = os.environ['GHOME'] + '/Workspace/IncidentEstimation/data/gamma.db'

import pytest

@pytest.mark.skip(reason="not imp yet!")
class TestDataset(TestCase):
    @pytest.mark.skip("slow")
    def test_load(self):
        dataset = create_dataset(dataset_db, path_db, 5, 32)
        samples = []
        with tf.Session() as sess:
            for i in range(1000):
                samples.append(sess.run(
                    [dataset.hits.data, dataset.first_hit_index.data, dataset.padded_size.data]))
        assert samples[0][0].shape == (32, 5, 4)


from __future__ import absolute_import, division, print_function

import pytest
import tempfile
import pandas as pd

from blaze import CSV, Table
from blaze.expr import *
from blaze.expr.core import discover
from blaze.utils import raises
from datashape import dshape, var, int32, int64, Record, DataShape
from toolz import identity, first
import numpy as np

def test_dshape():
    t = TableSymbol('t', '{name: string, amount: int}')
    assert t.dshape == dshape('var * {name: string, amount: int}')

def test_eq():
    assert TableSymbol('t', '{a: string, b: int}') == \
            TableSymbol('t', '{a: string, b: int}')
    assert TableSymbol('t', '{b: string, a: int}') != \
            TableSymbol('t', '{a: string, b: int}')

def test_arithmetic():
    t = TableSymbol('t', '{x: int, y: int, z: int}')
    x, y, z = t['x'], t['y'], t['z']
    exprs = [x + 1, x + y, 1 + y,
             x - y, 1 - x, x - 1,
             x ** y, x ** 2, 2 ** x,
             x * y, x ** 2, 2 ** x,
             x / y, x / 2, 2 / x,
             x % y, x % 2, 2 % x]

def test_column():
    t = TableSymbol('t', '{name: string, amount: int}')
    assert t.columns == ['name', 'amount']

    assert eval(str(t['name'])) == t['name']
    assert str(t['name']) == "t['name']"

def test_Projection():
    t = TableSymbol('t', '{name: string, amount: int, id: int32}')
    p = projection(t, ['amount', 'name'])
    assert p.schema == dshape('{amount: int32, name: string}')
    print(t['amount'].dshape)
    print(dshape('var * int32'))
    assert t['amount'].dshape == dshape('var * {amount: int32}')

    assert eval(str(p)).isidentical(p)

def test_indexing():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')
    assert t[['amount', 'id']] == projection(t, ['amount', 'id'])
    assert t['amount'].isidentical(Column(t, 'amount'))

def test_relational():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')

    r = (t['name'] == 'Alice')

    assert r.dshape == dshape('var * bool')

def test_selection():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')

    s = selection(t, t['name'] == 'Alice')

    assert s.dshape == t.dshape

def test_selection_typecheck():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')

    assert raises(TypeError, lambda: t[t['amount'] + t['id']])
    assert raises(TypeError, lambda: t[t['name']])

def test_selection_by_indexing():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')

    result = t[t['name'] == 'Alice']

    assert t.schema == result.schema
    assert 'Alice' in str(result)

def test_selection_by_getattr():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')

    result = t[t.name == 'Alice']

    assert t.schema == result.schema
    assert 'Alice' in str(result)

def test_selection_path_check():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')
    t2 = t[t.name == 'Alice']
    t3 = t2[t2.amount > 0]
    assert t3

def test_different_schema_raises():
    with tempfile.NamedTemporaryFile(delete=False) as f:
        df = pd.DataFrame(np.random.randn(10, 2))
        df.to_csv(f.name, index=False, header=False)
        with pytest.raises(TypeError):
            Table(CSV(f.name), columns=list('ab'))

def test_getattr_doesnt_override_properties():
    t = TableSymbol('t', '{iscolumn: string, schema: string}')
    assert isinstance(t.iscolumn, bool)
    assert isinstance(t.schema, DataShape)

def test_dir_contains_columns():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')
    result = dir(t)
    columns_set = set(t.columns)
    assert set(result) & columns_set == columns_set

def test_selection_consistent_children():
    t = TableSymbol('t', '{name: string, amount: int, id: int}')

    expr = t['name'][t['amount'] < 0]

    assert list(expr.columns) == ['name']

def test_columnwise_syntax():
    t = TableSymbol('t', '{x: real, y: real, z: real}')
    x, y, z = t['x'], t['y'], t['z']
    assert (x + y).active_columns() == ['x', 'y']
    assert (z + y).active_columns() == ['y', 'z']
    assert ((z + y) * x).active_columns() == ['x', 'y', 'z']

    expr = (z % x * y + z ** 2 > 0) & (x < 0)
    assert isinstance(expr, ColumnWise)

def test_str():
    import re
    t = TableSymbol('t', '{name: string, amount: int, id: int}')
    expr = t[t['amount'] < 0]['name'] * 2
    assert '<class' not in str(expr)
    assert not re.search('0x[0-9a-f]+', str(expr))

    assert eval(str(expr)) == expr

    assert '*' in repr(expr)

def test_join():
    t = TableSymbol('t', '{name: string, amount: int}')
    s = TableSymbol('t', '{name: string, id: int}')
    j = join(t, s, 'name', 'name')

    assert j.schema == dshape('{name: string, amount: int, id: int}')

    assert join(t, s, 'name') == join(t, s, 'name')

def test_joined_column_first_in_schema():
    t = TableSymbol('t', '{x: int, y: int, z: int}')
    s = TableSymbol('s', '{w: int, y: int}')

    assert join(t, s).schema == dshape('{y: int, x: int, z: int, w: int}')

def test_outer_join():
    t = TableSymbol('t', '{name: string, amount: int}')
    s = TableSymbol('t', '{name: string, id: int}')

    jleft = join(t, s, 'name', 'name', how='left')
    jright = join(t, s, 'name', 'name', how='right')
    jinner = join(t, s, 'name', 'name', how='inner')
    jouter = join(t, s, 'name', 'name', how='outer')

    js = [jleft, jright, jinner, jouter]

    assert len(set(js)) == 4  # not equal

    assert jinner.schema == dshape('{name: string, amount: int, id: int}')
    assert jleft.schema == dshape('{name: string, amount: int, id: ?int}')
    assert jright.schema == dshape('{name: string, amount: ?int, id: int}')
    assert jouter.schema == dshape('{name: string, amount: ?int, id: ?int}')

    # Default behavior
    assert join(t, s, 'name', 'name', how='inner') == \
            join(t, s, 'name', 'name')

def test_join_default_shared_columns():
    t = TableSymbol('t', '{name: string, amount: int}')
    s = TableSymbol('t', '{name: string, id: int}')
    assert join(t, s) == join(t, s, 'name', 'name')

def test_multi_column_join():
    a = TableSymbol('a', '{x: int, y: int, z: int}')
    b = TableSymbol('b', '{w: int, x: int, y: int}')
    j = join(a, b, ['x', 'y'])

    assert set(j.columns) == set('wxyz')

    assert j.on_left == j.on_right == ['x', 'y']
    assert hash(j)

    assert j.columns == ['x', 'y', 'z', 'w']

def test_traverse():
    t = TableSymbol('t', '{name: string, amount: int}')
    assert t in list(t.traverse())

    expr = t[t['amount'] < 0]['name']
    trav = list(expr.traverse())
    assert any(t['amount'].isidentical(x) for x in trav)
    assert any((t['amount'] < 0).isidentical(x) for x in trav)

def test_unary_ops():
    t = TableSymbol('t', '{name: string, amount: int}')
    expr = cos(exp(t['amount']))
    assert 'cos' in str(expr)

    assert '~' in str(~(t.amount > 0))

def test_reduction():
    t = TableSymbol('t', '{name: string, amount: int32}')
    r = sum(t['amount'])
    print(type(r.dshape))
    print(type(dshape('int32')))
    print(r.dshape)
    assert r.dshape in (dshape('int32'),
                        dshape('{amount: int32}'),
                        dshape('{amount_sum: int32}'))

    assert 'amount' not in str(t.count().dshape)

    assert first(t.count().dshape[0].types)[0] in (int32, int64)

    assert 'int' in str(t.count().dshape)
    assert 'int' in str(t.nunique().dshape)
    assert 'string' in str(t['name'].max().dshape)
    assert 'string' in str(t['name'].min().dshape)
    assert 'string' not in str(t.count().dshape)

    t = TableSymbol('t', '{name: string, amount: real, id: int}')

    assert 'int' in str(t['id'].sum().dshape)
    assert 'int' not in str(t['amount'].sum().dshape)

def test_Distinct():
    t = TableSymbol('t', '{name: string, amount: int32}')
    r = distinct(t['name'])
    print(r.dshape)
    assert r.dshape  == dshape('var * {name: string}')

    r = t.distinct()
    assert r.dshape  == t.dshape

def test_by():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')
    r = by(t, t['name'], sum(t['amount']))

    print(r.schema)
    assert isinstance(r.schema[0], Record)
    assert str(r.schema[0]['name']) == 'string'

def test_by_columns():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')

    assert len(by(t, t['id'], t['amount'].sum()).columns) == 2
    assert len(by(t, t['id'], t['id'].count()).columns) == 2
    print(by(t, t, t.count()).columns)
    assert len(by(t, t, t.count()).columns) == 4

def test_sort():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')
    s = t.sort('amount', ascending=True)
    print(str(s))
    assert eval(str(s)).isidentical(s)

    assert s.schema == t.schema

    assert t['amount'].sort().key == 'amount'

def test_head():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')
    s = t.head(10)
    assert eval(str(s)).isidentical(s)

    assert s.schema == t.schema

def test_label():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')
    quantity = (t['amount'] + 100).label('quantity')

    assert eval(str(quantity)).isidentical(quantity)

    assert quantity.columns == ['quantity']

def test_columns():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')
    assert list(t.columns) == ['name', 'amount', 'id']
    assert list(t['name'].columns) == ['name']
    (t['amount'] + 1).columns

def test_relabel():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')

    rl = t.relabel({'name': 'NAME', 'id': 'ID'})

    assert eval(str(rl)).isidentical(rl)

    print(rl.columns)
    assert rl.columns == ['NAME', 'amount', 'ID']

def test_relabel_join():
    names = TableSymbol('names', '{first: string, last: string}')

    siblings = join(names.relabel({'last': 'left'}),
                    names.relabel({'last': 'right'}), 'first')

    assert siblings.columns == ['first', 'left', 'right']

def test_map():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')
    inc = lambda x: x + 1
    assert t['amount'].map(inc).iscolumn
    assert t['amount'].map(inc, schema='{amount: int}').iscolumn
    s = t['amount'].map(inc, schema='{amount: int}', iscolumn=False)
    assert not s.iscolumn

    assert s.dshape == dshape('var * {amount: int}')

    assert not t[['name', 'amount']].map(identity).iscolumn

def test_apply():
    t = TableSymbol('t', '{name: string, amount: int32, id: int32}')
    s = Apply(sum, t['amount'], dshape='real')

    assert s.dshape == dshape('real')

def test_columnwise():
    from blaze.expr.scalar import Add, Eq, Mult
    t = TableSymbol('t', '{x: int, y: int, z: int}')
    x = t['x']
    y = t['y']
    z = t['z']
    assert str(columnwise(Add, x, y).expr) == 'x + y'
    assert columnwise(Add, x, y).child.isidentical(t)

    c1 = columnwise(Add, x, y)
    c2 = columnwise(Mult, x, z)

    assert eval_str(columnwise(Eq, c1, c2).expr) == '(x + y) == (x * z)'
    assert columnwise(Eq, c1, c2).child.isidentical(t)

    assert str(columnwise(Add, x, 1).expr) == 'x + 1'

def test_TableSymbol_printing_is_legible():
    accounts = TableSymbol('accounts', '{name: string, balance: int, id: int}')

    expr = (exp(accounts['balance'] * 10)) + accounts['id']
    assert "exp(accounts['balance'] * 10)" in str(expr)
    assert "+ accounts['id']" in str(expr)

def test_dtype():
    accounts = TableSymbol('accounts',
                           '{name: string, balance: int32, id: int32}')

    assert accounts['name'].dtype == dshape('string')
    assert accounts['balance'].dtype == dshape('int32')
    assert (accounts['balance'] > accounts['id']).dtype == dshape('bool')

def test_merge():
    accounts = TableSymbol('accounts',
                           '{name: string, balance: int32, id: int32}')
    new_amount = (accounts['balance'] * 1.5).label('new')

    c = merge(accounts[['name', 'balance']], new_amount)
    assert c.columns == ['name', 'balance', 'new']

inc = lambda x: x + 1

def test_subterms():
    a = TableSymbol('a', '{x: int, y: int, z: int}')
    assert list(a.subterms()) == [a]
    assert set(a['x'].subterms()) == set([a, a['x']])
    assert set(a['x'].map(inc).subterms()) == set([a, a['x'], a['x'].map(inc)])
    assert a in set((a['x'] + 1).subterms())

def test_common_subexpression():
    a = TableSymbol('a', '{x: int, y: int, z: int}')

    assert common_subexpression(a).isidentical(a)
    assert common_subexpression(a, a['x']).isidentical(a)
    assert common_subexpression(a['y'] + 1, a['x']).isidentical(a)
    assert common_subexpression(a['x'].map(inc), a['x']).isidentical(a['x'])

def test_schema_of_complex_interaction():
    a = TableSymbol('a', '{x: int, y: int, z: int}')
    expr = (a['x'] + a['y']) / a['z']
    assert expr.dtype == dshape('real')

    expr = expr.label('foo')
    print(expr.dtype)
    assert expr.dtype == dshape('real')

def test_iscolumn():
    a = TableSymbol('a', '{x: int, y: int, z: int}')
    assert not a.iscolumn
    assert a['x'].iscolumn
    assert not a[['x', 'y']].iscolumn
    assert not a[['x']].iscolumn
    assert (a['x'] + a['y']).iscolumn
    assert a['x'].distinct().iscolumn
    assert not a[['x']].distinct().iscolumn
    assert not by(a, a['x'], a['y'].sum()).iscolumn
    assert a['x'][a['x'] > 1].iscolumn
    assert not a[['x', 'y']][a['x'] > 1].iscolumn
    assert a['x'].sort().iscolumn
    assert not a[['x', 'y']].sort().iscolumn
    assert a['x'].head().iscolumn
    assert not a[['x', 'y']].head().iscolumn

    assert TableSymbol('b', '{x: int}', iscolumn=True).iscolumn
    assert not TableSymbol('b', '{x: int}', iscolumn=False).iscolumn
    assert TableSymbol('b', '{x: int}', iscolumn=True) != \
            TableSymbol('b', '{x: int}', iscolumn=False)

def test_discover():
    schema = '{x: int, y: int, z: int}'
    a = TableSymbol('a', schema)

    assert discover(a) == var * schema

def test_improper_selection():
    t = TableSymbol('t', '{x: int, y: int, z: int}')

    assert raises(Exception, lambda: t[t['x'] > 0][t.sort()[t['y' > 0]]])

def test_union():
    schema = '{x: int, y: int, z: int}'
    a = TableSymbol('a', schema)
    b = TableSymbol('b', schema)
    c = TableSymbol('c', schema)

    u = union(a, b, c)
    assert u.schema == a.schema

    assert raises(Exception,
                  lambda: union(a, TableSymbol('q', '{name: string}')))

def test_serializable():
    t = TableSymbol('t', '{id: int, name: string, amount: int}')
    import pickle
    t2 = pickle.loads(pickle.dumps(t))

    assert t.isidentical(t2)

    s = TableSymbol('t', '{id: int, city: string}')
    expr = join(t[t.amount < 0], s).sort('id').city.head()
    expr2 = pickle.loads(pickle.dumps(expr))

    assert expr.isidentical(expr2)

def test_table_coercion():
    from datetime import date
    t = TableSymbol('t', '{name: string, amount: int, timestamp: ?date}')
    assert (t.amount + '10').expr.rhs == 10

    assert (t.timestamp < '2014-12-01').expr.rhs == date(2014, 12, 1)

def test_isnan():
    t = TableSymbol('t', '{name: string, amount: int, timestamp: ?date}')

    for expr in [t.amount.isnan(), ~t.amount.isnan()]:
        assert eval(str(expr)).isidentical(expr)

    assert isinstance(t.amount.isnan(), TableExpr)
    assert 'bool' in str(t.amount.isnan().dshape)


import argparse

import torch
import pytorch_lightning as pl
from torch.nn import functional as F
from torch.utils.data import DataLoader, random_split
import data_loader, loader_test
import faiss
import os,sys
from trainer import XlingualDictionary
from collections import defaultdict
from transformers import AutoTokenizer
sys.path.append("../")
from models.lstm import LSTM_model
from helper_functions import read_json_file

def preprocess_text(phrase, src_lang, target_word, dataset):
    '''

    preprocess text
    '''

    tokens = dataset.preprocessing_data(phrase, src_lang, src=True)
    input_idx = dataset.tokens2tensor(tokens)

    #target = torch.tensor(self.targets[idx])
    #target = (target_word)
    mean_embedding = dataset.get_mean_embeddings(input_idx)
    label = torch.ones(input_idx.shape[0], 1)
    return {
            "phrase": {
                        'input_ids': input_idx.squeeze(),
                    },
            #"target": target,
            "mean_ft": mean_embedding,
            "label": label
        }

def get_accuracy(test_data, model, index_dir, test_dataset, k=10, batch_size=32):

    model.eval()

    lang_map = {'HI': 'hi', 'BE': 'bn', 'GU': 'gu', 'OD': 'or', 'PU': 'pa', 'EN': 'en', 'MA': 'mr'}
    data_by_lang = defaultdict(list)

    for d in test_data:
        data_by_lang[d["Target_ID"]].append(d)

    correct = 0
    total = 0

    for lang in lang_map.keys():
        with open(os.path.join(index_dir, lang_map[lang] + ".vocab"), 'r') as f:
            vocab = [line.strip() for line in f]

        phrases = [d["Source_text"] for d in data_by_lang[lang]]
        targets = [d["Target_keyword"] for d in data_by_lang[lang]]
        b = 0

        index = faiss.read_index(os.path.join(index_dir, lang_map[lang] + ".index"))
        with torch.no_grad():

            for i in range(len(phrases)):

                phrase = phrases[i]
                src_lang = lang_map[lang]
                target_word = targets[i]

                batch = preprocess_text(phrase, src_lang, target_word, test_dataset)

                #x, label =  batch["phrase"]["input_ids"].unsqueeze(0).cuda(),  batch["label"].unsqueeze(0).cuda()
                x = batch["mean_ft"]
                y_hat = x
                #y_hat = model(x)
                #y_hat = y_hat.squeeze()
                #print(y_hat.shape)
                # embeddings = torch.mean(outputs.last_hidden_state, 1).detach().cpu().numpy()
                y_hat = y_hat.detach().cpu().numpy()
                faiss.normalize_L2(y_hat)
                D, I = index.search(y_hat, k)

                words = [[vocab[i] for i in row] for row in I]

                if target_word in words[0]:
                    correct+=1

                total += 1
                #print(correct/total)
                if total % ( batch_size) == 0:
                    print("{} samples processed {}".format(total, correct/total))

        print("{} done!".format(lang))

    return correct / total

def xlingual_reverse_dictionary(model, index_dir, k):

    model.eval()

    tokenizer = AutoTokenizer.from_pretrained("ai4bharat/indic-bert")

    #lang_map = {"EN": "en"}
    lang_map = {'HI': 'hi', 'BE': 'bn', 'GU': 'gu', 'OD': 'or', 'PU': 'pa', 'EN': 'en', 'MA': 'mr'}
    data_by_lang = defaultdict(list)
    vocab = {}

    for lang in lang_map.keys():
        with open(os.path.join(index_dir, lang_map[lang] + ".vocab"), 'r') as f:
            vocab[lang_map[lang]] = [line.strip() for line in f]

    lang = input("Enter output language from HI-BE-GU-OD-PU-EN-MA \n")

    with torch.no_grad():
        while (1):
            phrases = input("Enter source phrase:\n")
            phrases = [phrases.lower()]

            print(phrases)
            tokens = tokenizer(phrases, padding="max_length", truncation=True, max_length=128, return_tensors="pt")
            tokens.to('cuda')
            _, I = model(tokens, os.path.join(index_dir, lang_map[lang] + ".index"), k)
            print(I)
            words = [[vocab[lang_map[lang]][i] for i in row] for row in I]
            for word in words[0]:
                print(word)
            # print(words)
            #print(words)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(add_help=True)
    parser.add_argument("test_data", type=str)
    parser.add_argument("index_dir", type=str)
    parser.add_argument("emb_path", type=str)
    parser.add_argument("emb_dir", type=str)
    parser.add_argument("model_path", type=str)
    parser.add_argument("k", type=int, default=10)
    args = parser.parse_args()

    test_dataset = loader_test.XLingualTrainDataset_baseline_lstm(args.test_data, args.emb_path, args.emb_dir)
    test_dataloader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=10, drop_last = True)

    EMBEDDING_DIM = test_dataset.embeddings.vectors[0].shape[0]
    PAD_IDX = test_dataset.embeddings.stoi[test_dataset.vocabulary.pad_token]
    UNK_IDX = test_dataset.embeddings.stoi[test_dataset.vocabulary.unk_token]

    model = LSTM_model(vocab_size = len(test_dataset.embeddings), input_dim = EMBEDDING_DIM, hidden_size = (EMBEDDING_DIM), num_layers = 5, padding_idx = PAD_IDX)
    model.embedding.weight.data.copy_(test_dataset.embeddings.vectors)
    model.load_state_dict(torch.load(args.model_path))
    model.to('cuda')

    # if args.test_data == "test":
    #     xlingual_reverse_dictionary(model, index_dir=args.index_dir, k = args.k)
    # elif os.path.exists(args.test_data):
    print(get_accuracy(read_json_file(args.test_data), model, args.index_dir, test_dataset, args.k))


from flask_restful import Resource
from flask import request, jsonify

from DatabaseStatements.DBQueries import InsertNewCar, UpdateCarInfos
from utils.DBConnection import DBConnection
from flask_jwt_extended import jwt_required, get_jwt_identity

class UpdateCarInfo(Resource):
    @jwt_required
    def post(self):
        data_form = request.form

        car_id = data_form.get("id")
        car_manufacturer = data_form.get("manufacturer")
        car_model = data_form.get('model')
        car_color = data_form.get('color')
        car_size = data_form.get('size')

        try:
            identity_from_token = get_jwt_identity()
            con = DBConnection()

            if car_id != '':
                con.execute_insert_query(UpdateCarInfos, (car_manufacturer, car_model, car_color, car_size, car_id))
            else:
                con.execute_insert_query(InsertNewCar, (identity_from_token['id'], car_manufacturer, car_model, car_color, car_size))

            return jsonify(ok=1, message='Information updated')

        except:
            return jsonify(ok=0, message='Something when wrong with driver')

    def get(self):
        return '', 404


import cv2
import time
import numpy as np
import pandas as pd

"""
prev = new = ''
cap = cv2.VideoCapture(0)

def getColorName(R,G,B):
    minimum = 10000
    for i in range(len(csv)):
        d = abs(R- int(csv.loc[i,"R"])) + abs(G- int(csv.loc[i,"G"]))+ abs(B- int(csv.loc[i,"B"]))
        if(d<=minimum):
            minimum = d
            cname = csv.loc[i,"color_name"]
    return cname

while True:
    index=["color_name","hex","R","G","B"]
    csv = pd.read_csv('colorsV3.csv', names=index, header=None,encoding='latin-1')
    ret, frame = cap.read()

    cv2.rectangle(frame, (309,229), (329,249), (0,255,0), 1)       #pixel range = x ==> 0-639 and y == 0 - 479
    roi = frame[230:249, 310:329]
    avg1 = np.average(roi, axis=0)
    avg2 = np.average(avg1, axis=0)
    avg2_int = avg2.astype(int)
    avg2_int = avg2_int[::-1]           #reversed for rgb
   # avg2_int_tup = tuple(avg2_int)
    r = avg2_int[0]
    g = avg2_int[1]
    b = avg2_int[2]

    print(r," ",g," ",b )
    time.sleep(1)
    '''new = getColorName(r,g,b)

    if (new != prev):
        print(new)
        prev = new
    frame = cv2.flip(frame,1)

    cv2.imshow('CAMERA',frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    '''

cap.release()

cv2.destroyAllWindows()

"""

index=["color_name","R","G","B"]
csv = pd.read_csv('colorsT2.csv', names=index, header=None,encoding='latin-1')
def getColorName(R,G,B):
    minimum = 10000
    for i in range(len(csv)):
        d = abs(R- int(csv.loc[i,"R"])) + abs(G- int(csv.loc[i,"G"]))+ abs(B- int(csv.loc[i,"B"]))
        if(d<=minimum):
            minimum = d
            cname = csv.loc[i,"color_name"]
    return cname

def detect_color(s1,s2,e1,e2):
    #print((e1-s1)*(e2-s2))
    red = 0
    blue = 0
    white = 0
    ph = 0
    count = 0
    for x in range( s1, e1-1,10) :
        for y in range( s2 , e2-1,10) :
            count+=1
            if frameHeight <= x+10 or frameWidth <= y+10:
                break
            roi = img[x:x+10, y:y+10]

            avg1 = np.average(roi, axis=0)
            avg2 = np.average(avg1, axis=0)
            avg2_int = avg2.astype(int)
            avg2_int = avg2_int[::-1]           #reversed for rgb
           # avg2_int_tup = tuple(avg2_int)
            r = avg2_int[0]
            g = avg2_int[1]
            b = avg2_int[2]

            new = getColorName(r,g,b)
            if(new =='Red'):
                red +=1
            elif(new== 'Blue'):
                blue +=1
            elif(new == 'Ph'):
                ph +=1
            else:
                white +=1

    print(red,"   ",blue,"    ",white,"     ",ph,"    ",count)

    if(red > blue and red > 10):
        print("Red detected")
    elif(blue > red and blue> 10 ):
        print("Blue detected")
    elif(ph>30):
        print("Ph paper only")
    else:
        print("no ph paper detected")

frameWidth = 640
frameHeight = 480
cap = cv2.VideoCapture(0)
cap.set(3, frameWidth)
cap.set(4, frameHeight)

def empty(a):
    pass

cv2.namedWindow("Parameters")
cv2.resizeWindow("Parameters",640,240)
cv2.createTrackbar("Threshold1","Parameters",15,255,empty)
cv2.createTrackbar("Threshold2","Parameters",10,255,empty)
cv2.createTrackbar("Area","Parameters",1000,30000,empty)

def stackImages(scale,imgArray):
    rows = len(imgArray)
    cols = len(imgArray[0])
    rowsAvailable = isinstance(imgArray[0], list)
    width = imgArray[0][0].shape[1]
    height = imgArray[0][0].shape[0]
    if rowsAvailable:
        for x in range ( 0, rows):
            for y in range(0, cols):
                if imgArray[x][y].shape[:2] == imgArray[0][0].shape [:2]:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (0, 0), None, scale, scale)
                else:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (imgArray[0][0].shape[1], imgArray[0][0].shape[0]), None, scale, scale)
                if len(imgArray[x][y].shape) == 2: imgArray[x][y]= cv2.cvtColor( imgArray[x][y], cv2.COLOR_GRAY2BGR)
        imageBlank = np.zeros((height, width, 3), np.uint8)
        hor = [imageBlank]*rows
        hor_con = [imageBlank]*rows
        for x in range(0, rows):
            hor[x] = np.hstack(imgArray[x])
        ver = np.vstack(hor)
    else:
        for x in range(0, rows):
            if imgArray[x].shape[:2] == imgArray[0].shape[:2]:
                imgArray[x] = cv2.resize(imgArray[x], (0, 0), None, scale, scale)
            else:
                imgArray[x] = cv2.resize(imgArray[x], (imgArray[0].shape[1], imgArray[0].shape[0]), None,scale, scale)
            if len(imgArray[x].shape) == 2: imgArray[x] = cv2.cvtColor(imgArray[x], cv2.COLOR_GRAY2BGR)
        hor= np.hstack(imgArray)
        ver = hor
    return ver

def getContours(img,imgContour):
    contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    for cnt in contours:
        area = cv2.contourArea(cnt)
        areaMin = cv2.getTrackbarPos("Area", "Parameters")
        if area > areaMin:
            cv2.drawContours(imgContour, cnt, -1, (255, 0, 255), 7)
            peri = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)

            #time.sleep(0.1)

            if len(approx)<4 or len(approx)>6:
                break

            print(len(approx))

            x , y , w, h = cv2.boundingRect(approx)
            detect_color(x,y,x+w,y+h)
            x=x+10
            y=y+10
            w=w-20
            h=h-20

            cv2.rectangle(imgContour, (x , y ), (x + w , y + h ), (0, 255, 0), 5)

            cv2.putText(imgContour, "Points: " + str(len(approx)), (x + w + 20, y + 20), cv2.FONT_HERSHEY_COMPLEX, .7,
                        (0, 255, 0), 2)
            cv2.putText(imgContour, "Area: " + str(int(area)), (x + w + 20, y + 45), cv2.FONT_HERSHEY_COMPLEX, 0.7,
                        (0, 255, 0), 2)

            #time.sleep(0.2)

while True:
    success, img = cap.read()
    time.sleep(0.1)
    imgContour = img.copy()
    imgBlur = cv2.GaussianBlur(img, (7, 7), 1)
    imgGray = cv2.cvtColor(imgBlur, cv2.COLOR_BGR2GRAY)
    threshold1 = cv2.getTrackbarPos("Threshold1", "Parameters")
    threshold2 = cv2.getTrackbarPos("Threshold2", "Parameters")
    imgCanny = cv2.Canny(imgGray,threshold1,threshold2)
    kernel = np.ones((5, 5))
    imgDil = cv2.dilate(imgCanny, kernel, iterations=1)
    getContours(imgDil,imgContour)
    imgStack = stackImages(0.8,([img,imgCanny],
                                [imgDil,imgContour]))
    cv2.imshow("Result", imgStack)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

COLOR PICKER/Python test codes/v3_contour/rect_git_1.py


import sys, socket, json, urllib.request, argparse
from PyQt5.QtWidgets import QApplication, QWidget, QInputDialog, QDialog, QLabel, QMessageBox, QProgressBar
from PyQt5.QtGui import QIcon, QPixmap, QImage
from PyQt5.QtCore import Qt, QBasicTimer
from ResponseThread import ResponseThread
from HelpDialog import HelpDialog

class Gui(QWidget):

    def __init__(self):
        super().__init__()

        self.mySocket = socket.socket()
        self.params = {}
        self.responseThread = ResponseThread(self.mySocket)
        self.isPlaying = False

        self.setGeometry(300, 300, 420, 420)
        self.setWindowTitle('Kodi Remote Control')
        self.setWindowIcon(QIcon('./img/icon.png'))

        self.img = QLabel('img', self)
        self.img.setGeometry((420-256)/2,(380-256)/2,256,256)
        self.img.setPixmap(QPixmap('./img/splash.png'))

        self.status = QLabel('OK', self)
        self.status.setGeometry(0,380,400,20)

        self.pbar = QProgressBar(self)
        self.pbar.setGeometry(0,400,410,20)
        self.progress = 0

        self.timer = QBasicTimer()

        self.show()

        self.responseThread.mySignal.connect(self.handleSignals)
        self.responseThread.start()

    def handleSignals(self, signal, param):
        #print("Signal caught")
        if signal == 'Input.OnInputRequested':
            self.showInputDialog()
            print("Text sending")
        if signal == 'Player.OnStop':
            self.isPlaying = False
            self.status.setText( 'Nothing playing now' )
            self.img.setPixmap(QPixmap('./img/splash.png'))
            if self.timer.isActive():
                self.timer.stop()
                self.pbar.setValue(0)
                print('Stoping timer')
        if signal == 'Player.OnPlay':
            self.isPlaying = True
            self.status.setText( param )
            self.rpc("Player.GetItem",{"properties":["thumbnail","fanart"],"playerid":1}, True)
            if not self.timer.isActive():
                self.timer.start(1000, self)
                print('Starting timer')
        if signal == 'Playlist.OnAdd':
            gui.rpc("GUI.ShowNotification",{"title":"Added to playlist!", "message":param}, False)
        if signal == 'System.OnQuit' or signal == 'System.OnRestart':
            if self.timer.isActive():
                self.timer.stop()
                print('Stoping timer')
            self.quit(False)
        if signal == 'thumbnail':
            url = param.split('image://',1)[1]
            url = url[0:(len(url)-1)]
            url = urllib.parse.unquote(url)
            img_arr = urllib.request.urlopen(url).read()
            qImg = QImage()
            qImg.loadFromData(img_arr)
            self.img.setPixmap(QPixmap(qImg))
        if signal == 'time':
            param = json.loads(param)
            totalTime = {
                'hours' : param['totaltime']['hours'],
                'minutes' : param['totaltime']['minutes'],
                'seconds' : param['totaltime']['seconds']
            }
            time = {
                'hours' : param['time']['hours'],
                'minutes' : param['time']['minutes'],
                'seconds' : param['time']['seconds']
            }
            totTime = timeToDuration(totalTime)
            curTime = timeToDuration(time)
            if totTime>0:
                self.pbar.setValue((curTime/totTime)*100)
        #print (signal)

    def timerEvent(self, e):
            self.rpc("Player.GetProperties",{"playerid":1,"properties":["totaltime","time"]}, True)

    def showInputDialog(self):
        text, ok = QInputDialog.getText(self, 'Send Text',
            'Type here what you want to send:')

        if ok:
            self.rpc("Input.SendText",{"text":str(text)}, False)

    def showHelpDialog(self):
        h = HelpDialog()
        h.exec_()

    def keyPressEvent(self, e):
        if e.key() == Qt.Key_Up:
            self.rpc("Input.Up",[], False)
        elif e.key() == Qt.Key_Down:
            self.rpc("Input.Down",[], False)
        elif e.key() == Qt.Key_Left:
            self.rpc("Input.Left",[], False)
        elif e.key() == Qt.Key_Right:
            self.rpc("Input.Right",[], False)
        elif e.key() == Qt.Key_F9:
            self.rpc("Input.ExecuteAction",{"action":"volumedown"}, False)
        elif e.key() == Qt.Key_F10:
            self.rpc("Input.ExecuteAction",{"action":"volumeup"}, False)
        elif e.key() == Qt.Key_F11:
            self.rpc("Input.ExecuteAction",{"action":"mute"}, False)
        elif e.key() == Qt.Key_Tab:
            self.rpc("Input.ExecuteAction",{"action":"fullscreen"}, False)
        elif e.key() == Qt.Key_Space:
            self.rpc("Input.ExecuteAction",{"action":"playpause"}, False)
        elif e.key() == Qt.Key_Control:
            if self.isPlaying:
                self.rpc("Input.ShowOSD",[], False)
        elif e.key() == Qt.Key_PageUp:
            self.rpc("Input.ExecuteAction",{"action":"pageup"}, False)
        elif e.key() == Qt.Key_PageDown:
            self.rpc("Input.ExecuteAction",{"action":"pagedown"}, False)
        elif e.key() == Qt.Key_Escape:
            self.rpc("Input.ExecuteAction",{"action":"parentfolder"}, False)
        elif e.key() == Qt.Key_Home:
            self.rpc("Input.Home",[], False)
        elif e.key() == Qt.Key_Backspace:
            self.rpc("Input.Back",[], False)
        elif e.key() == Qt.Key_Return:
            self.rpc("Input.Select",[], False)
        elif e.key() == Qt.Key_I:
            if self.isPlaying:
                self.rpc("Player.GetItem",{"playerid":1}, True)
            else:
                self.rpc("Input.Info",[], False)
        elif e.key() == Qt.Key_Q:
            self.status.setText("Quiting now...")
            self.quit(True)
        elif e.key() == Qt.Key_A:
            self.rpc("Input.ExecuteAction",{"action":"queue"}, True)
        elif e.key() == Qt.Key_C:
            self.rpc("Input.ContextMenu",[], False)
        elif e.key() == Qt.Key_P:
            self.rpc("Input.ExecuteAction",{"action":"playlist"}, True)
        elif e.key() == Qt.Key_Z:
            self.rpc("Input.ExecuteAction",{"action":"aspectratio"}, False)
        elif e.key() == Qt.Key_Comma:
            self.rpc("Input.ExecuteAction",{"action":"stepback"}, False)
        elif e.key() == Qt.Key_Period:
            self.rpc("Input.ExecuteAction",{"action":"stepforward"}, False)
        elif e.key() == Qt.Key_BracketLeft:
            self.rpc("Input.ExecuteAction",{"action":"subtitledelayminus"}, False)
        elif e.key() == Qt.Key_BracketRight:
            self.rpc("Input.ExecuteAction",{"action":"subtitledelayplus"}, False)
        elif e.key() == Qt.Key_1:
            self.rpc("Addons.ExecuteAddon",{"addonid":"plugin.video.exodus"}, False)
        elif e.key() == Qt.Key_2:
            self.rpc("Addons.ExecuteAddon",{"addonid":"plugin.video.twitch"}, False)
        elif e.key() == Qt.Key_F1:
            self.showHelpDialog()
        elif e.key() == Qt.Key_Backslash:
            self.showInputDialog()

            #self.rpc("Addons.ExecuteAddon",{"addonid":"plugin.video.exodus","params":{"action":"seasons"}}, True)
            #self.rpc("GUI.ActivateWindow",{"window":"video", "parameters":["sources://video"]}, True)

    def rpc(self, method, params, should_respond):
        d = {
            "jsonrpc": "2.0",
            "method": method,
            "params": params,
            }
        if should_respond:
            d.update({"id":1})
        m = json.dumps(d)
        self.mySocket.send(m.encode())

    def quit(self, shutdownKodi):
        if self.responseThread.isRunning():
            #print("Killing thread")
            self.responseThread.terminate()

        print("Killed thread")

        if shutdownKodi:
            self.rpc("System.Shutdown",[], False)

        self.mySocket.close()
        print("Dropped tcp connection")
        sys.exit(0)

def timeToDuration(time):
    duration = time['hours']*3600 + time['minutes']*60 + time['seconds']
    return duration

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--ip", help="The ip of the machine kodi is running on")
    parser.add_argument("-p", "--port", help="The port kodi is listening to", type=int)
    args = parser.parse_args()

    app = QApplication(sys.argv)
    gui = Gui()

    if args.ip:
        gui.params['ip'] = str(args.ip)
    else:
        gui.params['ip'] = "DEFAULT_IP_HERE"

    if args.port:
        gui.params['port'] = args.port
    else:
        gui.params['port'] = DEFAULT_PORT_HERE

    print("Initiating tcp connection")
    gui.mySocket.connect((gui.params['ip'],gui.params['port']))

    gui.rpc("GUI.ShowNotification",{"title":"Daddy's home!", "message":"The only rightful owner"}, False)

    app.exec()
    gui.quit(False)


import numpy as np
import torch
import torch.utils.data as data_utils
from sklearn.preprocessing import minmax_scale
from torchvision import transforms

from src.data.Audio.CustomTensor import CustomTensorDataset
from src.data.Audio.TorchLoader import TorchLoader

def dataloader(args, log):
    audio = TorchLoader(path_source=args.source_folder, log=log, sample_rate=args.sample_rate)
    x_train, train_y, x_val, val_y, x_test, test_y = audio.load_data(
        source_path=args.source_folder).load_files().save_data(
        destination_path=args.source_folder).get_data()

    x_train = minmax_scale(x_train)
    x_val = minmax_scale(x_val)
    x_test = minmax_scale(x_test)

    x_train = np.reshape(x_train, (-1, args.in_channels, args.x, args.y))
    x_val = np.reshape(x_val, (-1, args.in_channels, args.x, args.y))
    x_test = np.reshape(x_test, (-1, args.in_channels, args.x, args.y))

    class PadNumpy(object):

        def __init__(self, type_padding):
            self._type_padding = type_padding

        def __call__(self, sample):
            real_shape = sample.shape
            if real_shape[2] == 49:
                sample = np.pad(sample, ((0, 0), (0, 0), (8, 7)), self._type_padding)
            if real_shape[2] == 40:
                sample = np.pad(sample, ((0, 0), (0, 0), (12, 12)), self._type_padding)
            if real_shape[1] == 25:
                sample = np.pad(sample, ((0, 0), (20, 19), (0, 0)), self._type_padding)
            if real_shape[2] == 99:
                sample = np.pad(sample, ((0, 0), (0, 0), (15, 14)), self._type_padding)
            if real_shape[2] == 80:
                sample = np.pad(sample, ((0, 0), (0, 0), (24, 24)), self._type_padding)
            if real_shape[1] == 50:
                sample = np.pad(sample, ((0, 0), (39, 39), (0, 0)), self._type_padding)
            return sample

    data_transform_pad = transforms.Compose([
        PadNumpy("constant"),
    ])

    train_dataset = CustomTensorDataset(torch.from_numpy(x_train), torch.from_numpy(x_train),
                                        transform=data_transform_pad)
    train_loader = data_utils.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=2)

    validation_dataset = CustomTensorDataset(torch.from_numpy(x_val), torch.from_numpy(x_val),
                                             transform=data_transform_pad)
    val_loader = data_utils.DataLoader(validation_dataset, batch_size=args.batch_size, shuffle=False, num_workers=2)

    test_dataset = CustomTensorDataset(torch.from_numpy(x_test), torch.from_numpy(x_test), transform=data_transform_pad)
    test_loader = data_utils.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=2)

    return train_loader, val_loader, test_loader


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"rest - Functions for implementing a RESTful API for the TesteWeb."

__author__ = 'Vitor Henrique L. Mesquita <vitor@vitorhenrique.me>'

import json
import logging
import os
import sys

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'persistence')))

from database import db_session, DB_PATH, DB_NAME, init_db
from flask import Flask, jsonify, request, jsonify, make_response
from flask_cors import CORS, cross_origin
from model import Product

COMPONENT = None
REST = '/rest'
app = Flask(__name__)
CORS(app)
def urljoin(*args):
    """Joins the arguments in the likes of os.path.join.
        :param args: The arguments to be joined.
    """
    return '/'.join(s for s in args if s.strip() != '')

class ApplicationComponent(object):
    def __init__(self):
        pass
    def insert_product(self, session, data):
        try:
            transformed_data = {}
            for x in data:
                transformed_data[x['name']] = x['value']
            product = Product(transformed_data['cod_mer'],
                              transformed_data['type_mer'],
                              transformed_data['name_mer'],
                              transformed_data['amt'],
                              transformed_data['price'],
                              transformed_data['type_b'])
            session.add(product)
            session.flush
            session.commit()
            return True
        except ValueError:
            return ValueError
    def get_all_product(self, session):
        return session.query(Product).all()

@app.route(urljoin(REST, "product/"), methods=['POST'])
def new_product():
    try:
        data = request.get_json()
        if COMPONENT.insert_product(db_session, data):
            return jsonify(True)
        else:
            return make_response(jsonify(False), 404)
    except:
            return make_response(jsonify(False), 404)
@app.route(urljoin(REST, "products/"), methods=['GET'])
def get_products():
    rows ={"data":[]}
    result = COMPONENT.get_all_product(db_session)
    for x in result:
        dictret = dict(x.__dict__)
        dictret.pop('_sa_instance_state', None)
        rows['data'].append(dictret)
    return jsonify(rows)

def main():
    database = os.path.join(DB_PATH, DB_NAME)
    if not os.path.exists(database):
        init_db()
    global COMPONENT
    COMPONENT = ApplicationComponent()
    app.run(host='0.0.0.0',port=8080, debug=True, use_reloader=False, threaded=True)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
# (c) Copyright IBM Corp. 2019. All Rights Reserved.
# pragma pylint: disable=unused-argument, no-self-use

import requests
from resilient_lib.components.integration_errors import IntegrationError

class RequestsCommon:
    """
    This class represents common functions around the use of the requests package for REST based APIs.
    It incorporates the app.config section "integrations" which can be used to define a common set of proxies
    for use by all functions using this library:

    [integrations]
    http_proxy=
    https_proxy=

    Similar properties may exist in the function's section which would override the [integrations] properties.
    """
    def __init__(self, opts=None, function_opts=None):
        # capture the properties for the integration as well as the global settings for all integrations for proxy urls
        self.integration_options = opts.get('integrations', None) if opts else None
        self.function_opts = function_opts

    def get_proxies(self):
        """ proxies can be specified globally for all integrations or specifically per function """
        proxies = None
        if self.integration_options and (self.integration_options.get("http_proxy") or self.integration_options.get("https_proxy")):
            proxies = {'http': self.integration_options.get("http_proxy"), 'https': self.integration_options.get("https_proxy")}

        if self.function_opts and (self.function_opts.get("http_proxy") or self.function_opts.get("https_proxy")):
            proxies = {'http': self.function_opts.get("http_proxy"), 'https': self.function_opts.get("https_proxy")}

        return proxies

    def execute_call(self, verb, url, payload={}, log=None, basicauth=None, verify_flag=True, headers=None,
                     proxies=None, timeout=None, resp_type='json', callback=None):
        """
        Function: perform the http API call. Different types of http operations are supported:
        GET, HEAD, PATCH, POST, PUT, DELETE
        Errors raise IntegrationError
        If a callback method is provided, then it's called to handle the error

        Using the 'json' parameter in the request will change the Content-Type in the header to application/json.
        If the headers you send include a specific Content-Type, your payload will be send as 'data' and not 'json'.

        :param verb: GET, HEAD, PATCH, POST, PUT, DELETE
        :param url:
        :param basicauth: used for basic authentication - (user, password)
        :param payload:
        :param log: optional log statement
        :param verify_flag: True/False - False used for debugging generally
        :param headers: dictionary of http headers
        :param proxies: http and https proxies for call
        :param timeout: timeout before call should abort
        :param resp_type: type of output to return: json, text, bytes
        :param callback: callback routine used to handle errors
        :return: json of returned data
        """

        try:
            (payload and log) and log.debug(payload)

            if verb.lower() not in ('get', 'post', 'put', 'patch', 'delete', 'head'):
                raise IntegrationError("unknown verb {}".format(verb))

            if proxies is None:
                proxies = self.get_proxies()

            if verb.lower() == 'post':
                if verify_headers(headers):
                    # If headers dict specifies Content-Type,
                    # pass the payload to the data argument.
                    # Use Content-Type specified in headers.
                    resp = requests.request(verb.upper(), url, verify=verify_flag, headers=headers, data=payload,
                                            auth=basicauth, timeout=timeout, proxies=proxies)
                else:
                    # If headers dict does not specify Content-Type,
                    # pass payload to the json argument.
                    # Content-Type will automatically set to application/json.
                    resp = requests.request(verb.upper(), url, verify=verify_flag, headers=headers, json=payload,
                                            auth=basicauth, timeout=timeout, proxies=proxies)
            else:
                resp = requests.request(verb.upper(), url, verify=verify_flag, headers=headers, params=payload,
                                        auth=basicauth, timeout=timeout, proxies=proxies)

            if resp is None:
                raise IntegrationError('no response returned')

            # custom handler for response handling?
            if callback:
                return callback(resp)

            # standard error handling
            if resp.status_code >= 300:
                # get the result
                # log resp.status_code in case resp.text isn't available
                raise IntegrationError(
                    "status_code: {}, msg: {}".format(resp.status_code, resp.text if resp.text else "N/A"))

            # check if anything returned
            log and log.debug(resp.text)

            # get the result
            if resp_type == 'json':
                r = resp.json()
            elif resp_type == 'text':
                r = resp.text
            elif resp_type == 'bytes':
                r = resp.content
            else:
                raise IntegrationError("incorrect response type: {}".format(resp_type))

            # Produce a IntegrationError with the return value
            return r      # json object needed, not a string representation
        except Exception as err:
            msg = str(err)
            log and log.error(msg)
            raise IntegrationError(msg)

def verify_headers(headers):
    """
    Verify if headers dict includes 'Content-Type' key.
    :param headers:
    :return: True or False
    """
    if headers is None:
        return False

    for key in headers:
        if key.lower() == "content-type":
            return True
    return False


# queue.py
# by James Fulford
# for Joyful Love

# implements needed functions from Buffer API

import utilities
from utilities import get_when
from utilities import dtformat
import buffpy

class Queue(utilities.SaveLoad):
    """
    Represents a single buffer.
        .name() returns a pretty string
            Queue.name(profile) returns pretty string for profile too
        .pending_updates() returns list of updates in Buffer and Reserve
        .get_buffer() returns list of updates in Buffer
        .add(update) tries to add update to Reserve
        .sync() sends reserved updates to
    """

    def __init__(self, profile, path=None):
        self._prof = profile

        part = path
        if not path:
            part = Queue.profile_name(profile)

        try:
            self.reserve = utilities.load(part)["reserve"]
        except IOError:
            self.reserve = []

    def save(self, fp=None):
        from json import dump
        part = ""
        if fp:
            part = fp + "/"
        with open(part + self.name(), "wb") as phile:
            dump(self.dictionary(), phile, indent=4, sort_keys=True)

    def dictionary(self):
        return {
            "reserve": sorted(self.reserve, key=get_when)
        }

    def name(self):
        return Queue.profile_name(self._prof)

    @staticmethod
    def profile_name(prof):
        return prof["formatted_service"] + " " + prof["formatted_username"]

    __str__ = name

    def pending_updates(self):
        """
        Returns list of posts in Buffer and in reserve
        """
        upd = self.get_buffer()
        upd.extend(self.reserve)
        return sorted(upd, key=get_when)

    def get_buffer(self):
        """
        Returns list of posts in Buffer
        """
        return self._prof.updates.pending

    def add(self, update):
        """
        Adds updates to reserve.
        """
        import buffpy
        upds = self.pending_updates()

        if get_when(update) not in map(get_when, upds):
            self.reserve.append(update)
        else:
            msg = "Already a message scheduled for " + get_when(update).strftime(dtformat())
            exp = Exception(msg)
            exp.error = {
                "update": update,
                "queue": self,
            }
            raise exp

    def sync(self):
        """
        Sends reserved updates to Buffer, stores the rest back in reserve
        Returns time of the latest post

        Order matters, so do not do major multithreading if it can be avoided
        """
        upds = self.reserve
        self.reserve = []
        upds.sort(key=get_when)
        while len(self.get_buffer()) < 10 and len(upds) > 0:
            update = upds.pop(0)
            try:
                self.post(update)
            except buffpy.exceptions.BuffpyRestException:
                print(self.name() + " did not post: " + get_when(update).strftime(dtformat()))
        self.reserve.extend(upds)

    def next_time(self):
        to_consider = []
        buff = self.get_buffer()
        if len(buff) > 0:
            to_consider.append(max(map(get_when, buff)))
        if len(self.reserve) > 0:
            to_consider.append(min(map(get_when, self.reserve)))

        if len(to_consider) > 0:
            return min(to_consider)
        return None

    def post(self, update):
        """
        Adds update to profile's buffer.
        TODO: Allow posting by sending email to secret buffer address
        """
        content = ""
        if "content" in update.keys():
            content = update["content"]
        if "text" in update.keys():
            content = update["text"]

        if "timezone" not in update.keys():
            update["timezone"] = self._prof["timezone"]

        def post_no_media(update):
            if "post_at" in update.keys():
                return self._prof.updates.new(content, when=str(get_when(update)))
            else:
                return self._prof.updates.new(content, now=True)

        def post_with_media(update):
            media = update["media"]
            if "post_at" in update.keys():
                when = str(get_when(update))
                return self._prof.updates.new(content, media=media, when=when)
            else:
                return self._prof.updates.new(content, media=media, now=True)

        def post_link(update):
            # assert url_exists(media["link"])
            # remove picture or thumbnail if url is bad
            if "picture" in media.keys():
                # if not url_exists(media["picture"]):
                del media["picture"]
            if "thumbnail" in media.keys():
                # if not url_exists(media["thumbnail"]):
                del media["thumbnail"]
            post_with_media(update)

        def url_exists(url):
            import httplib
            conn = httplib.HTTPConnection(url)
            conn.request("HEAD", "/index.html")
            res = conn.getresponse()
            return res.status < 400

        # now to post
        try:
            assert "media" in update.keys()
            # if there is not media, posts without media.

            media = update["media"]
            # if media is a photo
            if "photo" in media.keys():
                import os
                try:

                    # if photo url fails, post link
                    # assert url_exists(media["photo"])

                    # post photo
                    post_with_media(update)
                except AssertionError:
                    del media["photo"]

                    # if post_link fails, posts without media
                    post_link(update)
            else:
                # if there isn't a photo,
                # post link instead.
                # if post_link fails, posts without media
                post_link(update)
        except AssertionError:
            # posts without media
            post_no_media(update)


#    Copyright 2016 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.test import TestCase
import mock

from devops.driver.libvirt.libvirt_driver import LibvirtDriver
from devops.driver.libvirt.libvirt_driver import LibvirtManager

CAPS_XML = """
<capabilities>

  <host>
    <cpu>
      <arch>i686</arch>
      <features>
        <pae/>
        <nonpae/>
      </features>
    </cpu>
    <power_management/>
    <secmodel>
      <model>testSecurity</model>
      <doi></doi>
    </secmodel>
  </host>

  <guest>
    <os_type>hvm</os_type>
    <arch name='i686'>
      <wordsize>32</wordsize>
      <emulator>/usr/bin/qemu-system-i386</emulator>
      <domain type='qemu'>
      </domain>
      <domain type='test'>
        <emulator>/usr/bin/test-emulator</emulator>
      </domain>
    </arch>
    <features>
      <cpuselection/>
      <deviceboot/>
      <acpi default='on' toggle='yes'/>
      <apic default='on' toggle='no'/>
      <pae/>
      <nonpae/>
    </features>
  </guest>

  <guest>
    <os_type>hvm</os_type>
    <arch name='x86_64'>
      <wordsize>64</wordsize>
      <emulator>/usr/bin/qemu-system-x86_64</emulator>
      <domain type='test'>
        <emulator>/usr/bin/test-emulator</emulator>
      </domain>
    </arch>
    <features>
      <cpuselection/>
      <deviceboot/>
      <acpi default='on' toggle='yes'/>
      <apic default='on' toggle='no'/>
    </features>
  </guest>

</capabilities>
"""

class LibvirtTestCase(TestCase):

    def patch(self, *args, **kwargs):
        patcher = mock.patch(*args, **kwargs)
        m = patcher.start()
        self.addCleanup(patcher.stop)
        return m

    def setUp(self):
        # reset device names
        LibvirtDriver._device_name_generators = {}

        self.libvirt_vol_up_mock = self.patch('libvirt.virStorageVol.upload')
        self.libvirt_vol_resize_mock = self.patch(
            'libvirt.virStorageVol.resize')
        self.libvirt_stream_snd_mock = self.patch('libvirt.virStream.sendAll')
        self.libvirt_stream_fin_mock = self.patch('libvirt.virStream.finish')

        self.libvirt_nwfilter_define_mock = self.patch(
            'libvirt.virConnect.nwfilterDefineXML')
        self.libvirt_nwfilter_lookup_mock = self.patch(
            'libvirt.virConnect.nwfilterLookupByName')
        self.libvirt_list_all_devs_mock = self.patch(
            'libvirt.virConnect.listAllDevices')

        self._libvirt_clear_all()
        conn = LibvirtManager.get_connection('test:///default')

        self.caps_patcher = mock.patch.object(conn, 'getCapabilities')
        self.caps_mock = self.caps_patcher.start()
        self.addCleanup(self.caps_patcher.stop)
        self.caps_mock.return_value = CAPS_XML

    def tearDown(self):
        self._libvirt_clear_all()

    @staticmethod
    def _libvirt_clear_all():
        conn = LibvirtManager.get_connection('test:///default')
        pool = conn.storagePoolLookupByName('default-pool')

        for domain in conn.listAllDomains():
            for snapshot in domain.listAllSnapshots(0):
                snapshot.delete()
            if domain.isActive():
                domain.destroy()
            domain.undefine()
        for network in conn.listAllNetworks():
            network.destroy()
            network.undefine()
        for vol in pool.listAllVolumes():
            vol.delete()


import socket
import json
import sys
import random
import time
from goodmovesonly import good_moves

def receiveJSON(socket):
    fullreceive = False
    message = ''
    while not fullreceive:
        message += socket.recv(512).decode('utf8')
        try:
            data = json.loads(message)
            fullreceive = True
        except json.JSONDecodeError:
            pass
    return data

def sendJSON(socket, data):
    data = json.dumps(data).encode('utf8')
    totalsend = 0
    while totalsend < len(data):
        sent = socket.send(data[totalsend:])
        totalsend += sent

def subscribe(port,name):
    port = int(port)
    message = {"request": "subscribe","port": port,"name": name,"matricules": ["19022", "18164"]}
    s = socket.socket()
    s.connect(('localhost',3000))
    sendJSON(s, message)
    s.close()
    #boucle de réception de requêtes
    listening = True
    while listening == True:
        s = socket.socket()
        s.bind(('0.0.0.0', port))
        s.listen()
        client, address = s.accept()
        serverrequest = receiveJSON(client)
        if serverrequest['request'] == 'ping':
            sendJSON(client, {'response': 'pong'})
            listening = False

def SENDmove(server_request):
    # 'current': 0 -> black, 'current': 1 -> white
    playercolors = ['B','W']
    playerindice = server_request['state']['current']
    moves = good_moves(server_request['state']['board'],playercolors[playerindice])
    move = random.choice(moves)
    marbles = []
    for i in move[1]:
        if type(i) is tuple:
            marbles.append(list(i))
        else:
            direction = i
    moverequest = {"marbles": marbles,"direction": direction}
    return {"response": "move","move": moverequest}

defaultport = '5000'
defaultname = 'Random AI'
if len(sys.argv) == 2:
    try:
        isinteger = int(sys.argv[1])
        port = sys.argv[1]
        name = defaultname
    except ValueError:
        port = defaultport
        name = sys.argv[1]
elif len(sys.argv) == 3:
    port = sys.argv[1]
    name = sys.argv[2]
elif len(sys.argv) > 3:
    print('Invalid input')
    sys.exit()
else:
    port = defaultport
    name = defaultname
subscribe(port, name)
listening = True
while listening == True:
        s = socket.socket()
        s.bind(('0.0.0.0', int(port)))
        s.listen()
        client, address = s.accept()
        serverrequest = receiveJSON(client)
        print(serverrequest['request'])
        if serverrequest['request'] == 'ping':
            sendJSON(client, {'response': 'pong'})
        if serverrequest['request'] == 'play':
            start = time.time()
            sendJSON(client, SENDmove(serverrequest))
            print('Temps de réponse : {} s'.format(time.time()-start))


from django.http import HttpResponseRedirect
from base.models import PermUser, PermUserObj, PermGroup, UserPermStrategy

def rights_required(perm_code, obj_name=None, redirect_url='/base/login/'):
    """
    Decorator for views that checks whether a user has a service permission
    enabled, redirecting to the redirect page if necessary.
    """

    def wrapper(f):
        def decorate(request, *arg, **kwargs):
            if not request.user.is_authenticated:
                return HttpResponseRedirect(redirect_url)

            user = request.user

            # 超级用户
            if user.is_superuser:
                check_result = True
            # 公司职员
            elif user.is_staff:
                group = user.groups.first()
                check_result = PermGroup.has_perm(perm_code, group)
            # 客户账号
            else:
                if obj_name:
                    obj_id = request.POST.get('obj_id', '')
                    if not obj_id:
                        return HttpResponseRedirect(redirect_url)
                    check_result = PermUserObj.has_perm(
                        perm_code, obj_name, obj_id, user)
                else:
                    if user.is_parent:
                        check_result = PermUser.has_perm(perm_code, user)
                    elif user.is_child:
                        check_result = UserPermStrategy.has_perm(
                            perm_code, user)

            if check_result:
                return f(request, *arg, **kwargs)
            else:
                return HttpResponseRedirect(redirect_url)

        return decorate

    return wrapper


#!/usr/bin/env python
# -*- coding: utf-8 -*-

def main():
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-v', '--version', version='0.0.1', action='version',
                        help="show program's version number and exit")
    parser.add_argument('--host', default='0.0.0.0', metavar='ADDRESS', nargs='?',
                        help='specify server hostname [default: 0.0.0.0]')
    parser.add_argument('--port', default=474657, type=int, action='store', nargs='?',
                        help='specify server port [default: 474657]')
    args = parser.parse_args()

if __name__ == '__main__':
    main()