# -*- coding: utf-8 -*-
from __future__ import absolute_import

from google.cloud.bigquery.schema import SchemaField

class BigQuerySchemaField(object):

    def __init__(self, name, field_type, mode='NULLABLE',
                 description=None, fields=None):
        self.name = name
        self.field_type = field_type
        self.mode = mode
        self.description = description
        self.fields = tuple(fields) if fields else None

    @staticmethod
    def from_dict(value):
        fields = value.get('fields', None)
        if fields:
            fields = tuple(BigQuerySchemaField.from_dict(f) for f in fields)
        return BigQuerySchemaField(
            name=value.get('name', None),
            field_type=value.get('field_type', None),
            mode=value.get('mode', None),
            description=value.get('description', None),
            fields=fields)

    @staticmethod
    def from_schema_field(schema_field):
        fields = tuple(BigQuerySchemaField.from_schema_field(f)
                       for f in schema_field.fields) if schema_field.fields else None
        return BigQuerySchemaField(
            name=schema_field.name,
            field_type=schema_field.field_type,
            mode=schema_field.mode,
            description=schema_field.description,
            fields=fields)

    @staticmethod
    def to_schema_field(model):
        fields = model.fields
        if fields:
            fields = tuple(BigQuerySchemaField.to_schema_field(f) for f in fields)
        else:
            fields = ()
        return SchemaField(
            name=model.name,
            field_type=model.field_type,
            mode=model.mode,
            description=model.description,
            fields=fields)

    def dict(self):
        schema = {
            'name': self.name,
            'type': self.field_type,
            'mode': self.mode,
            'description': self.description
        }
        if self.fields:
            schema.update({'fields': [f.dict() for f in self.fields]})
        return schema

    def exclude_description(self):
        fields = self.fields
        if fields:
            fields = tuple(f.exclude_description() for f in fields)
        return BigQuerySchemaField(self.name, self.field_type, self.mode, fields=fields)

    @staticmethod
    def represent(dumper, data):
        return dumper.represent_mapping(
            'tag:yaml.org,2002:map',
            (
                ('name', data.name),
                ('field_type', data.field_type),
                ('mode', data.mode),
                ('description', data.description),
                ('fields', data.fields),
            )
        )

    @staticmethod
    def normalize_field_type(field_type):
        field_type = field_type.upper()
        if field_type in ['INTEGER', 'INT64']:
            return 'INT64'
        elif field_type in ['FLOAT', 'FLOAT64']:
            return 'FLOAT64'
        elif field_type in ['BOOLEAN', 'BOOL']:
            return 'BOOL'
        elif field_type in ['BYTES']:
            return 'BYTES'
        elif field_type in ['DATE']:
            return 'DATE'
        elif field_type in ['DATETIME']:
            return 'DATETIME'
        elif field_type in ['TIME']:
            return 'TIME'
        elif field_type in ['TIMESTAMP']:
            return 'TIMESTAMP'
        elif field_type in ['STRING']:
            return 'STRING'
        else:
            raise ValueError('Unknown field type.')

    def _key(self):
        return (self.name,
                self.field_type,
                self.mode,
                self.description,
                frozenset(self.fields) if self.fields is not None
                else self.fields,)

    def __eq__(self, other):
        if not isinstance(other, BigQuerySchemaField):
            return NotImplemented
        return self._key() == other._key()

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self._key())

    def __repr__(self):
        return 'BigQuerySchemaField{0}'.format(self._key())


from flask import Flask, request
from flask_restful import Resource, Api

import datetime

app = Flask(__name__)
api = Api(app)

class Add(Resource):
    def post(self):
        value_array = request.json.get('value_array', [])
        if value_array and isinstance(value_array, list):
            result = 0
            for item in value_array:
                result += item.get('value', 0)
            return {"result": result}
        else:
            return {"result": "请求参数有错误"}

class GetDate(Resource):
    def get(self):
        now = datetime.datetime.now()
        date = now.strftime('%Y-%m-%d')
        return {"date": date}

class Chat(Resource):
    def post(self):
        msg = request.form.get('msg', '')
        if '您好' in msg and '再见' in msg:
            result = '天气不错。'
        elif '您好' in msg:
            result = '您好，您吃了吗？'
        elif '再见' in msg:
            result = '回见了您内。'
        else:
            result = ''
        return {'result': result}

api.add_resource(Add, '/add')
api.add_resource(GetDate, '/get_date')
api.add_resource(Chat, '/chat')

if __name__ == '__main__':
    app.run()


"""ITDR URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, patterns, include
from django.contrib import admin
from django.views import static as a_static
from .views import *
import ITDR.settings as settings
from django.conf.urls.static import static

urlpatterns = [
    # API
    url(r'^admin/', admin.site.urls),
    url(r'^news/', include('news.urls', namespace="news")),
    url(r'^projects/', include('projects.urls', namespace="projects")),
    url(r'^analytics/', include('analytics.urls', namespace="analytics")),
    url(r'^anons/', include('anons.urls', namespace="anons")),
    url(r'^events/', include('events.urls', namespace="events")),
    url(r'^publications/', include('news.urls', namespace="publications")),
    url(r'^admin-helper/', include('admin_helper.urls')),

    # Index
    url(r'^$', index_view, name="index"),

    # AJAX
    url(r'^get-news-(?P<start>\d+)-(?P<number>\d+)/$', ajax_index_news),
    url(r'^get-projects-(?P<start>-?\d+)-(?P<number>\d+)/$', ajax_index_projects),

] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

urlpatterns += patterns('',
    url(r'^search/?$', search, name='search_view'),
)

if settings.DEBUG:
    urlpatterns += patterns(
        '',
        url(r'^media/(?P<path>.*)$', a_static.serve,
            {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
    )


import os
import tweepy as tw
import pandas as pd

consumer_key= ''
consumer_secret= ''
access_token= ''
access_token_secret= ''

auth = tw.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tw.API(auth, wait_on_rate_limit=True)

# Define the search term and the data_since date
search_words = "basketball -filter:retweets" # keyword to search for
# date format as 2018-11-16
date_since = "2020-02-13"
# Collect tweets
tweets = tw.Cursor(api.search,
                   q=search_words,
                   lang="en",
                   since=date_since).items(100) # amt of tweets to return

users_locs = [[tweet.user.screen_name, tweet.user.location] for tweet in tweets]
print(users_locs)

tweet_text = pd.DataFrame(data=users_locs,   # creates table
                          columns=['user', "location"])
print(tweet_text)


import meshio
import numpy
import json

def get_cross_product(vector1, vector2):
    "Returns the cross product of two vector to access the normal vector"

    n_vect = [vector1[1] * vector2[2] - vector1[2] * vector2[1],
              -1 * (vector1[0] * vector2[2] - vector1[2] * vector2[0]),
              vector1[0] * vector2[1] - vector1[1] * vector2[0]]
    return n_vect

def get_plane_equation(point_list):
    "Returns the coefs (a,b,c,d) of the equation of a given plane"

    vect1 = [point_list[1][i] - point_list[0][i] for i in xrange(3)]
    vect2 = [point_list[1][i] - point_list[2][i] for i in xrange(3)]
    norm1 = numpy.linalg.norm(vect1)
    norm2 = numpy.linalg.norm(vect2)
    vect1 = [coord/norm1 for coord in vect1]
    vect2 = [coord/norm2 for coord in vect2]
    cross_product = get_cross_product(vect1, vect2)
    coefs = [cross_product[0], cross_product[1], cross_product[2], - (
            cross_product[0] * point_list[1][0] + cross_product[1] * point_list[1][1] + cross_product[2] *
            point_list[1][2])]
    # print coefs, sympy.Plane(sympy.Point3D(point_list[1]),sympy.Point3D(point_list[0]),sympy.Point3D(point_list[2])).equation()
    return coefs

def fill_rocktypes():
    # for i, dat in enumerate(cell_data['tetra']['gmsh:physical']):
    #     if dat == 3:
    #         res['cells'].append(i)
    #     elif dat == 2:
    #         cap['cells'].append(i)
    #     else:
    #         surf['cells'].append(i)
    if type == 'tet':
        for i, el in enumerate(elements):
            centroid = [sum([list_points[v][vv] for v in el])/4 for vv in xrange(3)]
            found = False
            if faults:
                if model == 'm3':
                    f1 = [[2000.,0.,0.],[6000.,0.,-4000.],[6000.,10000.,-4000.]]
                    f2 = [[2400.,0.,0.],[6400.,0.,-4000.],[6400.,10000.,-4000.]]
                    eq1 = get_plane_equation(f1)
                    eq2 = get_plane_equation(f2)
                    d1 = eq1[0] * centroid[0] + eq1[1] * centroid[1] + eq1[2] * centroid[2] + eq1[3]
                    d2 = eq2[0] * centroid[0] + eq2[1] * centroid[1] + eq2[2] * centroid[2] + eq2[3]
                    if d1 <= 0 and d2 >= 0:
                        rt[i] = 4
                        found = True
                else:
                    if 4900. <= centroid[0] <= 5100. or 4900. <= centroid[1] <= 5100.:
                        rt[i] = 4
                        found = True
            if centroid[2] < -1800. and not found:
                rt[i] = 1
            elif -1800. <= centroid[2] < -600. and not found:
                rt[i] = 2
            elif -600. <= centroid[2] and not found:
                rt[i] = 3

    else:
        for i, el in enumerate(elements):
            dat = numpy.array([list_points[ind] for ind in el])
            centroid = [(max(dat[:,0])+min(dat[:,0]))/2, (max(dat[:,1])+min(dat[:,1]))/2, (max(dat[:,2])+min(dat[:,2]))/2]
            found = False
            if faults:
                if model == 'm3':
                    f1 = [[2000.,0.,0.],[6000.,0.,-4000.],[6000.,10000.,-4000.]]
                    f2 = [[2400.,0.,0.],[6400.,0.,-4000.],[6400.,10000.,-4000.]]
                    eq1 = get_plane_equation(f1)
                    eq2 = get_plane_equation(f2)
                    d1 = eq1[0] * centroid[0] + eq1[1] * centroid[1] + eq1[2] * centroid[2] + eq1[3]
                    d2 = eq2[0] * centroid[0] + eq2[1] * centroid[1] + eq2[2] * centroid[2] + eq2[3]
                    if d1 <= 0 and d2 >= 0:
                        rt[i] = 4
                        found = True
                else:
                    if 4900. <= centroid[0] <= 5100. or 4900. <= centroid[1] <= 5100.:
                        rt[i] = 4
                        found = True
            if centroid[2] < -1800. and not found:
                rt[i] = 1
            elif -1800. <= centroid[2] < -600.and not found:
                rt[i] = 2
            elif -600. <= centroid[2] and not found:
                rt[i] = 3

    return rt

PATH = '/home/lmar626/Documents/Meshes/Basicmodels/Model1/Tet/cyl/'
meshname = 'otet_m1.vtk'
type = 'tet'
model = 'm3'
faults = False

mesh = meshio.read(PATH+meshname)
cells = mesh.cells
list_points = mesh.points
if type == 'tet':
    elements = cells['tetra']
else:
    elements = cells['hexahedron']

rt = numpy.zeros(len(elements))
ss = numpy.zeros(len(elements))
rt = fill_rocktypes()
with open(PATH+ meshname[:-4]+'.json', 'r') as f:
    data = json.load(f)
for source in data['source']:
    ss[source['cell']] = source['rate']

if type == 'tet':
    mesh.cell_data['tetra']['Sources'] = ss
    mesh.cell_data['tetra']['RT'] = rt
else:
    mesh.cell_data['hexahedron']['Sources'] = ss
    mesh.cell_data['hexahedron']['RT'] = rt
meshio.write(PATH+meshname, mesh)


#!/usr/bin/env python
from selenium import webdriver
from selenium.webdriver.common.keys import Keys

cover_letter = '/home/nw/job_applications/0826GenericIT/FrancescoOrsenigo_CoverLetter.pdf'
resume = '/home/nw/job_applications/0826GenericIT/FrancescoOrsenigo_Resume.pdf'
url = 'http://www.seek.com.au/JobSearch?DateRange=31&SearchFrom=quickupper&SearchType=search+again&Keywords=javascript&nation=3000&salary=60000-100000&SortMode=ListedDate'

driver = webdriver.Firefox()

def find_all_jobs():
  jobs = []
  driver.get(url)
  pages = [None] + [e.get_attribute('href') for e in driver.find_elements_by_xpath('//dd/span/a')]

  for p in pages:
    if p:
      driver.get(p)
    jobs += [e.get_attribute('href') for e in driver.find_elements_by_xpath('//dd/h2/a')]

  open('jobz','at').write( '\n'.join(jobs[1:]) )

def apply_to_all_jobs():
  jobs = open('jobz').read().split('\n')
  for j in jobs:
    driver.get(j)
    link = driver.find_element_by_class_name('apply-btn').get_attribute('href')
    if len(link.split('/Apply/')) < 2:
      continue

    driver.get(link)
    try: driver.find_element_by_class_name('not-now').click()
    except: pass

    driver.find_element_by_id('FirstName').send_keys('Francesco')
    driver.find_element_by_id('LastName').send_keys('Orsenigo')
    driver.find_element_by_id('PhoneNumber').send_keys('+04 3154 3174')
    driver.find_element_by_id('Email').send_keys('francesco.orsenigo@gmail.com')
    driver.find_element_by_id('CoverLetterTypeUpload').click()
    driver.find_element_by_name('CoverLetterUpload').send_keys(cover_letter)

    driver.find_element_by_name('ResumeUpload').send_keys(resume)
    driver.find_element_by_id('SubmitLink').click()
    open('done', 'at').write(j + '\n')

#find_all_jobs()
apply_to_all_jobs()

#driver.close()


## Import libraries
import cv2
from . import (fingerprint_distances_plot,
               fingerprint_matches_extraction,
               histogram_computation_and_plot,
               self_similar_matches_extraction,
               self_similar_distances_plot)

## Initialize constants

INITIAL_KNN = 10 # at first iteration of FLANN KDTREE we search for this number of neighborhood for each feature
KNN_STEP = 5 # when k self-similar matches are been found for at least one feature, more neighbours are searched, incrementing the actual number by this

## Find self-similar matches, fingerprint matches.
## Furthermore compute the description vector distances distribution, plot quantiles,
## self-similar matches, fingerprint matches together with the distances distribution
def self_similar_and_fingerprint_matches_extraction(template_descriptors):
    #%% Initialize a flann_matcher object to match keypoint witn nearest neighborhood.

    # = From flann_matcher documentation ==================================================
    # 	flann_matcher_INDEX_LINEAR = 0,
    # 	flann_matcher_INDEX_KDTREE = 1,
    # 	flann_matcher_INDEX_KMEANS = 2,
    # 	flann_matcher_INDEX_COMPOSITE = 3,
    # 	flann_matcher_INDEX_KDTREE_SINGLE = 4,
    # 	flann_matcher_INDEX_HIERARCHICAL = 5,
    # 	flann_matcher_INDEX_LSH = 6,
    # 	flann_matcher_INDEX_KDTREE_CUDA = 7, // available if compiled with CUDA
    # 	flann_matcher_INDEX_SAVED = 254,
    # 	flann_matcher_INDEX_AUTOTUNED = 255,
    # =============================================================================

    ## Specify a constant representing the type of algorithm used by flann_matcher
    flann_matcher_INDEX_KDTREE = 1 # algorithm used is KDTREE

    ## Specify flann_matcher matcher creator parameters
    index_params = dict(algorithm=flann_matcher_INDEX_KDTREE, trees=20) # 5 trees used in the KDTREE search
    search_params = dict(checks=200) # number of times the trees in the index should be recursively traversed

    ## Create FLANN matcher
    flann_matcher = cv2.FlannBasedMatcher(index_params, search_params)

    #%% Find correspondences by matching the template features with itself

    ## Invoke flann_matcher methods to obtain k outputs: for each feature in the template_descriptors image returns the k closest features in the template_descriptors image
    actual_knn = INITIAL_KNN
    matches =  flann_matcher.knnMatch(template_descriptors,template_descriptors,k=actual_knn) # there is no threshold, the k closest points are returned

    ## Show the number of features in template_descriptors image that have at least one match in template_descriptors image
    # print('Found ' + str(len(matches)) + ' putative matches')

    #%% Compute the description vector interval of distances

    ## Deletion of the first match that is the feature itself
    for i,kmatches in enumerate(matches):
        kmatches.pop(0)

    ## Creation of an array of all second matches
    second_matches = [item[0] for item in matches]

    ## Compute distances between each template feature and its second nearest neighbour (the first is the feature itself).
    ## Then distances histogram is seen as a discrete distribution, and its range of
    ## distances values is extracted.
    ## At the end the histogram is plotted.
    interval = histogram_computation_and_plot(second_matches, template_descriptors)

    #%% Find self-similar matches

    ## Compute a list with the same shape of "matches", but conserving only
    ## self-similar matches
    self_similar_list, search_for_more_neighbors, self_similar_quantile = self_similar_matches_extraction(matches,second_matches,template_descriptors)

    ## If at least one of the features has k self-similar matches,
    ## more neighbors has to be computed to find other possible self-simlar matches
    while search_for_more_neighbors:
        actual_knn = actual_knn+KNN_STEP
        matches =  flann_matcher.knnMatch(template_descriptors,template_descriptors,k=actual_knn)
        for i,kmatches in enumerate(matches):
            kmatches.pop(0)
        self_similar_list, search_for_more_neighbors, self_similar_quantile = self_similar_matches_extraction(matches,second_matches,template_descriptors)
    print('Maximum number of self-similar matches: ' + str(len(max(self_similar_list,key=len))))
    print('Number of features that have at least one self-similar match: ' + str(len([self_similar_matches for self_similar_matches in self_similar_list if self_similar_matches])))

    #%% Find fingerprint matches

    ## Compute a list with the same shape of "matches", but conserving only
    ## fingerprint matches
    fingerprint_list, fingerprint_quantile = fingerprint_matches_extraction(matches,second_matches,template_descriptors)
    print('Number of features that have a fingerprint match: ' + str(len([fingerprint_match for fingerprint_match in fingerprint_list if fingerprint_match])))

    #%% Plot quantiles, self-similar matches and fingerprint matches

    self_similar_distances_plot(self_similar_list, template_descriptors, interval, self_similar_quantile)
    fingerprint_distances_plot(fingerprint_list, template_descriptors, interval, fingerprint_quantile)

    return self_similar_list, fingerprint_list


# -*- coding: utf-8 -*-
from django.forms import ModelForm
from django import forms
from street_faces.subscription.models import subscription_mail

class subscription_form(ModelForm):
    class Meta:
        model = subscription_mail
        widgets = {
            'email': forms.TextInput(attrs={
                'placeholder': u'Введите свой e-mail'}),
        }


# type: ignore
#!/usr/bin/python
"""
Created on November 6, 2018

Input: directory of PDS4 products with Product_Bundle in root
Output (lid_reference below includes lidvid_reference):
- List every lid_reference with no corresponding logical_identifier in input
- List every lid_reference with count of referrers (verbose)
- List of local_identifier_reference with no local_identifier  LATER
- If Product_Bundle given, check all labels in that tree have
  urn:nasa:pds:<bundle_id>:...
- If Product_Collection given, check all labels in that tree have
  urn:nasa:pds:<bundle_id>:<collection_id>:<product_id>
@author: rchen
"""

import string, sys, re, getopt
import os  # rename file
import json
import re
import datetime
import StringIO

verbose = False
ignoreOld = False
LOG_DIR = "lidvid_error_log.txt"

def usage():
    print("usage: ", sys.argv[0], "[-hv] <contextProductDirectory> [<...>]")
    print("  -h this help")
    print("  -i ignore references from old versions of files")
    print("  -v verbose")
    print("  -x dir with existing context files. If none, outputs have version_id==1.0")
    print("validate complete set of context products")
    sys.exit()

try:
    opts, args = getopt.getopt(sys.argv[1:], "hiv", ["help", "ignore", "verbose"])
except getopt.GetoptError as err:
    print(str(err))  # will print something like "option -a not recognized"
    usage()
for o, a in opts:
    if o in ("-h", "--help"):
        usage()
    elif o in ("-i", "--ignore"):
        ignoreOld = True
    elif o in ("-v", "--verbose"):
        verbose = True
    else:
        assert False, "unhandled option"
if len(args) < 1:
    usage()

def flattenFilesOrDirs(paths):
    toReturn = []
    for p in paths:
        if (
            re.match("\.", p)  # allow initial ./xxx
            and p is not "."
            and not re.match("\.\/", p)
            and p is not ".."
            and not re.match("\.\.\/", p)
        ):
            continue
        if os.path.isdir(p):
            for inFile in os.listdir(p):
                if re.match("\.", inFile):
                    continue  # skip .DS_Store
                x = flattenFilesOrDirs([p + "/" + inFile])
                if x:
                    toReturn.extend(x)
        else:
            toReturn.append(p)
    return toReturn

lid2viddef = {}  # lid2viddef[LID] = dict of vid:fileDefiningLID
lid2refs = {}  # lid2refs[LID] = list of files referring to LID
lidvid2refs = {}  # lidvid2refs[LID::VID] = list of files referring to LIDVID

def parseFiles(allFiles):
    # ET handles namespaces verbosely, so this never matches:
    #  tagLid = "Identification_Area/logical_identifier"
    tagLid = "{http://pds.nasa.gov/pds4/pds/v1}Identification_Area/{http://pds.nasa.gov/pds4/pds/v1}logical_identifier"
    tagVid = "{http://pds.nasa.gov/pds4/pds/v1}Identification_Area/{http://pds.nasa.gov/pds4/pds/v1}version_id"
    tagLidRef = "{http://pds.nasa.gov/pds4/pds/v1}lid_reference"
    tagLidvidRef = "{http://pds.nasa.gov/pds4/pds/v1}lidvid_reference"
    lidTree = {}  # temp storage; if ignoreOld and vid is old, dump this
    lidvidTree = {}  # temp storage; if ignoreOld and vid is old, dump this
    lidvid2file = {}  # temp storage
    import xml.etree.ElementTree as ET

    for aFile in allFiles:  # build lidRefs{} and lids{}
        if not re.search("\.xml$", aFile):
            continue
        tree = ET.parse(aFile)
        doc = tree.getroot()
        if doc.find(tagLid) is None:
            print("INFO no LID in", aFile)
            continue
        lid = doc.find(tagLid).text
        vid = doc.find(tagVid).text
        if lid in lid2viddef:
            if vid in lid2viddef[lid]:
                sys.stderr.write(
                    "ERROR duplicate LIDVID "
                    + lid
                    + "::"
                    + vid
                    + "\n  "
                    + aFile
                    + "\n  "
                    + lid2viddef[lid][vid]
                    + "\n"
                )
                continue
        else:
            lid2viddef[lid] = {}
        lid2viddef[lid][vid] = {}
        lid2viddef[lid][vid] = aFile
        # this populates lid2viddef{}{}
        lidTree[lid + "::" + vid] = doc.findall(".//" + tagLidRef)
        lidvidTree[lid + "::" + vid] = doc.findall(".//" + tagLidvidRef)
        lidvid2file[lid + "::" + vid] = aFile
        # now populate the other two dicts
    for lid in sorted(lid2viddef.iterkeys()):
        for vid in sorted(lid2viddef[lid].iterkeys(), reverse=True):
            lv = lid + "::" + vid
            for lidRef in lidTree[lv]:
                if lidRef.text not in lid2refs:
                    lid2refs[lidRef.text] = []
                lid2refs[lidRef.text].append(lidvid2file[lv])  # set lid2refs{}[]
            for lidvidRef in lidvidTree[lv]:
                if lidvidRef.text not in lidvid2refs:
                    lidvid2refs[lidvidRef.text] = []
                lidvid2refs[lidvidRef.text].append(lidvid2file[lv])  # lidvid2refs{}[]
            if ignoreOld:
                break  # quit the vid loop after biggest vid
    return  # how to pass lid2viddef{}, lid2refs{}, lidvid2refs{}?

if __name__ == "__main__":
    allFiles = flattenFilesOrDirs(args)
    parseFiles(allFiles)  # sets lid2viddef{}, lid2refs{}, lidvid2refs{}
    # Record all warning messages
    log_arr = []
    separater = "==========" + args[1] + "=========="
    log_arr.append(separater)
    # Include all lidvid in tmp-context-products.json
    if os.path.isfile("tmp-context-products.json"):
        with open("tmp-context-products.json", "r+") as req_json:
            data = json.load(req_json)
            for context_product in data["Product_Context"]:
                lidvid = context_product["lidvid"]
                lidvid2refs[lidvid] = {}
                idx = lidvid.find("::")
                if idx != -1:
                    lid = lidvid[:idx]
                    vid = lidvid[-3:]
                    lid2viddef[lid] = {}
                    lid2viddef[lid][vid] = {}

    if verbose:
        print("___CHECK IF EVERY lid_reference IS DEFINED___")
    for lid in sorted(lid2refs.iterkeys()):
        if lid not in lid2viddef:
            msg = (
                "WARNING "
                + str(len(lid2refs[lid]))
                + " lid_references to absent LID "
                + str(lid)
            )
            if "-dhb" not in msg and "-ihb" not in msg:
                print(msg)
                log_arr.append(msg)
            if verbose:
                for rFile in lid2refs[lid]:
                    print(" ", rFile)
    if verbose:
        print("___CHECK IF EVERY lidvid_reference IS DEFINED___")
    for lidvid in sorted(lidvid2refs.iterkeys()):
        lid, vid = lidvid.split("::")
        if lid in lid2viddef:
            if vid in lid2viddef[lid]:
                continue  # all good
            else:
                msg = (
                    "WARNING "
                    + str(len(lidvid2refs[lidvid]))
                    + " lidvid_references to absent VID "
                    + str(lidvid)
                )
                if "-dhb" not in msg and "-ihb" not in msg:
                    print(msg)
                    log_arr.append(msg)
        else:
            msg = (
                "WARNING "
                + str(len(lidvid2refs[lidvid]))
                + " lidvid_references to absent LID "
                + str(lidvid)
            )
            if "-dhb" not in msg and "ihb" not in msg:
                print(msg)
                log_arr.append(msg)
        if verbose:
            for rFile in lidvid2refs[lidvid]:
                print("   ", rFile)
    # Should probably merge the 2 DUMPs by chopping the VID out of lidvid2refs,
    if verbose:
        print("___DUMP EVERY lid_reference + COUNT OF REFERRERS___")
        for lid in sorted(lid2refs.iterkeys()):
            print(lid, "\t", len(lid2refs[lid]))
            # list all referers? That's too much
        print("___DUMP EVERY lidvid_reference + COUNT OF REFERRERS___")
        for lidvid in sorted(lidvid2refs.iterkeys()):
            print(lidvid, "\t", len(lidvid2refs[lidvid]))
    if verbose:
        print("___CHECK IF EVERY local_identifier_reference IS DEFINED___")
    if verbose:
        print("___CHECK THAT ALL LIDS ARE HIERARCHICAL___")
    # alphabetically, bundleLID is first, and a collectionLID precedes its prods
    bundleLID = None
    collectionLID = None
    for lid in sorted(lid2viddef.iterkeys()):
        m = re.match("(urn:[^:]*:[^:]*:[^:]*)(:([^:]*)(:[^:]*)?)?$", lid)
        if bundleLID and m.group(1) == bundleLID:
            if collectionLID and m.group(3) == collectionLID:
                continue  # normal
            elif m.group(4):
                msg = "WARNING deriving collection LID from " + str(lid)
                print(msg)
            # else setting collectionLID after bundle product
            collectionLID = m.group(3)
            if verbose:
                print("INFO collection: ...:" + collectionLID)
        else:
            if m.group(2):
                msg = "WARNING deriving bundle LID from " + str(lid)
                print(msg)
            # else setting initial bundleLID, which is fine
            bundleLID = m.group(1)
            if verbose:
                print("INFO bundle:", bundleLID)
            if m.group(3):
                collectionLID = m.group(3)
                if verbose:
                    print("INFO collection: ...:" + collectionLID)
            else:
                collectionLID = None  # happens iff bundle product

    # Write all *to absent* warnings into lidvid_error_log.txt
    with open(LOG_DIR, "a") as f:
        for log in log_arr:
            f.write("%s\n" % log)
    sys.exit()


#!/usr/bin/env python
import struct

from core import eBPFCoreApplication, set_event_handler
from core.packets import *

import csv
import time
import matplotlib
import threading

from matplotlib import pyplot as plt
matplotlib.rcParams.update({'font.size': 14})

PROTO_TABLE_NAME = 'counters'
if len(sys.argv) > 1:
    MAX_TO_SHOW = int(sys.argv[1])
else:
    MAX_TO_SHOW = 30 # max is 256

if len(sys.argv) > 2:
    time = int(sys.argv[2])
else:
    time = 10

# table for ip protocols identification
csvFile = open('../protocol-numbers.csv')
reader  = csv.DictReader(csvFile)
pname   = [str(i) for i in range(256)]
for row in reader:
    try:
        pname[int(row['Decimal'])] = row['Keyword']
    except:
        False

plt.ion()
plt.draw()
plt.show()

class QueryThread(threading.Thread):
    def __init__(self, event, connection):
        threading.Thread.__init__(self)
        self.stopped = event
        self.connection = connection

    def run(self):
        while not self.stopped.wait(10):
            self.connection.send(TableListRequest(table_name=PROTO_TABLE_NAME))

class ProtoCountApplication(eBPFCoreApplication):

    @set_event_handler(Header.HELLO)
    def hello(self, connection, pkt):
        with open('../examples/protocount.o', 'rb') as f:
            print("Installing the eBPF ELF")
            connection.send(InstallRequest(elf=f.read()))

        self.queryThreadStopEvent = threading.Event()
        self.queryThread = QueryThread(self.queryThreadStopEvent, connection)
        self.queryThread.daemon = True
        self.queryThread.start()

    @set_event_handler(Header.TABLE_LIST_REPLY)
    def table_list_reply(self, connection, pkt):
        plt.cla()
        plt.clf()

        num_elms = min(256, MAX_TO_SHOW) # pkt.n_items
        x        = range(num_elms)
        x_labels = pname[0:num_elms]
        data = [ ]

        item_size = pkt.entry.value_size

        for i in range(pkt.n_items):
            npackets, nbytes = struct.unpack_from('IQ', pkt.items, i * item_size)
            if i < num_elms:
                data.append(nbytes)

        plt.bar(x, data, align='center')
        plt.xticks(x, x_labels, rotation=70, fontsize=12)
#        plt.title('Protocol Byte Count', fontsize=18)
#        plt.xlabel('Protocols', fontsize=18)
#        plt.ylabel('Number of Bytes', fontsize=18)
        plt.title('Contagem de Bytes por Protocolo', fontsize=18)
        plt.xlabel('Protocolos', fontsize=18)
        plt.ylabel('Numero de Bytes', fontsize=18)
        plt.tight_layout()
        plt.draw()
        plt.pause(0.01)

if __name__ == '__main__':
    ProtoCountApplication().run()


# coding=utf-8

import pygame
import object_rectangle
import resource

SCALE_TO_FILL = 0

class ImageView(object_rectangle.ObjectRectangle):
    def __init__(self, rect, img, content_mode=SCALE_TO_FILL):
        assert img is not None

        if rect is None:
            rect = pygame.Rect((0, 0), img.get_size())
        elif rect.w == 0 and rect.h == 0:
            rect.size = img.get_size()

        object_rectangle.ObjectRectangle.__init__(self, rect)

        self.enabled = False
        self.content_mode = content_mode
        self.image = img

    @property
    def image(self):
        return self._image

    @image.setter
    def image(self, new_image):
        self._image = new_image
        self.dirty = True

    def render(self):
        if self.content_mode == SCALE_TO_FILL:
            self._image = resource.scale_image(self._image, self.rect.size)
        else:
            assert False, "Unknown content_mode"

    def _draw(self, screen):
        if not object_rectangle.ObjectRectangle._draw(self, screen):
            return False

        self.render()
        self.surface.blit(self._image, (0, 0))

        return True


import os
import argparse
import json
from shutil import copyfile

def update_json(slug, label, color):
    with open("themes.json") as f:
        config = json.load(f)
    config.append({
        "id": slug,
        "label": label,
        "variables": {
            "css_variant": slug,
            "theme_color": color,
        },
    })
    with open("themes.json", "w") as f:
        json.dump(config, f, indent=2)

def create_theme(slug, label, color, abstract):
    if abstract:
        slug = f"__{slug}"
    theme_path = os.path.join("themes", slug)
    if os.path.isdir(theme_path):
        raise ValueError("Theme '{theme}' already exists.".format(theme=slug))

    static_path = os.path.join(theme_path, "static")
    os.makedirs(theme_path)
    os.makedirs(static_path)
    os.makedirs(os.path.join(theme_path, "templates"))

    open(os.path.join(static_path, "_custom.scss"), "w").close()
    for fname in ("style.scss", "_vars.scss"):
        copyfile(fname + ".template", os.path.join(static_path, fname))

    if not abstract:
        update_json(slug, label, color)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("slug")
    parser.add_argument("label", nargs="?", default=None)
    parser.add_argument("color", nargs="?", default=None)
    parser.add_argument("--abstract", action="store_true", default=False)
    args = parser.parse_args()
    create_theme(args.slug, args.label or args.slug, args.color or "#FFFFFF", args.abstract)


from flask.ext.wtf import Form
from wtforms import StringField, PasswordField, BooleanField, SubmitField, HiddenField, \
                    TextAreaField, ValidationError, SelectField, DecimalField, DateField, DateTimeField
from wtforms.validators import Required, Email, Length, EqualTo, InputRequired, NumberRange

from minispm import db

from .models import Leads, Industry, Category, Status, ContactType

class LeadsForm(Form):
    def __init__(self, **kwargs):
        super(LeadsForm, self).__init__(**kwargs)
        self.industry_id.choices = [(industry.id, industry.brief) for industry in Industry.query.all()]
        self.category_id.choices = [(category.id, category.brief) for category in Category.query.all()]
        self.status_id.choices = [(status.id, status.name) for status in Status.query.order_by(db.asc(Status.name)).all()]

    id = HiddenField()
    brief = StringField('简述', [InputRequired('简述必须输入'), Length(max=100, message='简述长度不能超过%(max)d')])
    customer_id = HiddenField('客户', [InputRequired('客户不能为空')])
    industry_id = SelectField('行业', [InputRequired('行业不能为空')])
    owner_id = HiddenField('所有者', [InputRequired('所有者不能为空')])
    sale_dept_id = HiddenField('市场部', [InputRequired('市场部门不能为空')])
    dev_dept_id = HiddenField('开发部', [InputRequired('开发部门不能为空')])
    category_id = SelectField('类型', [InputRequired('合同类型不能为空')])
    estimated_start_date = DateField('启动', [InputRequired('启动时间不能为空')])
    estimated_amount = DecimalField('金额', [NumberRange(min=0, message='预估金额必须大于%{min}d')], default=0)
    estimated_nop = DecimalField('人数', [NumberRange(min=0, message='预估人数必须大于%{min}d')], default=0)
    status_id = SelectField('状态', [InputRequired('状态不能为空')])
    details = TextAreaField('情况说明及风险分析', [InputRequired('情况说明及风险分析不能为空')])
    competitor_analysis = TextAreaField('竞争对手分析', [InputRequired('竞争对手分析不能为空')])
    created_date = DateField('创建')

class CustomerForm(Form):
    id = HiddenField()
    brief = StringField('简称', [InputRequired('简称必须输入'), Length(max=50, message='简述长度不能超过%(max)d')])
    is_new = BooleanField('是否新客户')
    details = TextAreaField('描述', [InputRequired('描述必须输入'), Length(max=200, message='描述长度不能超过%(max)d')])

class ActionForm(Form):
    def __init__(self, **kwargs):
        super(ActionForm, self).__init__(**kwargs)
        self.contact_type_id.choices = [(e.id, e.brief) for e in ContactType.list_not_system()]
    id = HiddenField()
    brief = StringField('简述', [InputRequired('简述不能为空'), Length(max=30, message='简述长度不能超过%(max)d')])
    contact_type_id = SelectField('活动类型', [InputRequired('活动类型不能为空')], choices=[('A','a')])
    occur_datetime = DateField('发生时间', [InputRequired('发生时间不能为空')])
    participants_customer = StringField('客户方人员', [InputRequired('客户方人员不能为空'),Length(max=100, message='客户方人员长度不能超过%(max)d')])
    participants_self = StringField('我方人员', [InputRequired('我方人员不能为空'), Length(max=100, message='我方方人员长度不能超过%(max)d')])
    is_milestone = BooleanField('是否重要事项')
    details = TextAreaField('详情', [InputRequired('详情不能为空'), Length(max=500, message='详情长度不能超过%(max)d')])


from urllib.request import urlopen as ureq
from bs4 import BeautifulSoup as bs
my_url = 'https://en.wikipedia.org/wiki/Special:Random'
uclient = ureq(my_url)
page_html = uclient.read()
uclient.close()
page_soup =  bs(page_html, "html.parser")
summary = page_soup.body.p
title = bs.get_text(page_soup.h1)
arttext =  bs.get_text(page_soup.body.p)
print("""
        %s

        %s

        """ %(title, arttext))


from django.views import generic
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django.urls import reverse_lazy
from django_tables2 import RequestConfig
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.db.models import Sum, Count
from django.db import models
from .tables import ExpenseTable
from .models import Expense
from .forms import ExpenseTableHelper
from .filters import ExpenseFilter
from .utils import AmountUnitUtil
import random
from calendar import monthrange
import datetime

class IndexView(generic.ListView):
    template_name = "tracker/index.html"
    context_object_name = "records"
    model = Expense
    table_class = ExpenseTable
    filter_class = ExpenseFilter
    formhelper_class = ExpenseTableHelper

    def get_queryset(self):
        return Expense.objects.filter(created_by=self.request.user)

    def get_context_data(self, **kwargs):
        context = super(IndexView, self).get_context_data(**kwargs)
        filter = ExpenseFilter(self.request.GET, queryset=self.get_queryset(**kwargs))
        filter.form.helper = ExpenseTableHelper()
        table = ExpenseTable(filter.qs)
        table.order_by = '-date'
        RequestConfig(self.request, paginate={'per_page': 10}).configure(table)
        context['filter'] = filter
        context['table'] = table
        return context

class ExpenseCreate(CreateView):
    # TODO: add datepicker

    model = Expense
    fields = [
        'date',
        'description',
        'type',
        'payment',
        'amount'
    ]

    def form_valid(self, form):
        obj = form.save(commit=False)
        obj.created_by = self.request.user
        obj.created_at = datetime.datetime.now()
        obj.save()
        return HttpResponseRedirect(reverse_lazy('tracker:expense'))

class ExpenseUpdate(UpdateView):
    model = Expense
    fields = [
        'date',
        'description',
        'type',
        'payment',
        'amount'
    ]

# obsolete
class ExpenseDelete(DeleteView):
    model = Expense
    success_url = reverse_lazy('tracker:expense')

class AnalyticsView(generic.ListView):
    template_name = "analytics/index.html"
    context_object_name = "records"
    model = Expense

    def statistic(request):
        # init
        avg_year = 0
        avg_month = 0
        avg_day = 0

        # get user expense objects
        exp = Expense.objects.filter(created_by=request.user)

        # count total record
        total_records = exp.count()

        # sum up all the expenses
        total_expenses = exp.aggregate(amount=Sum('amount'))['amount']
        if total_expenses is None:
            total_expenses = 0

        # categories count
        categories = exp.values('type').annotate(the_count=Count('type')).count()

        # list out dates for the following processing.
        dates = list(exp.values('date')
                     .values_list('date', flat=True))

        # avg amount per year, per month and per day
        year_arr = []
        month_arr = []
        day_arr = []

        for date in dates:
            if date.year not in year_arr:
                year_arr.append(date.year)
            if date.year not in month_arr:
                month_arr.append(date.month)
            if date.year not in day_arr:
                day_arr.append(date.day)

        if total_expenses > 0 :
            avg_year = AmountUnitUtil.convertToMills(total_expenses / year_arr.__len__())
            avg_month = AmountUnitUtil.convertToMills(total_expenses / month_arr.__len__())
            avg_day = AmountUnitUtil.convertToMills(total_expenses / day_arr.__len__())
            total_expenses = AmountUnitUtil.convertToMills(total_expenses)

        now = datetime.datetime.now()

        context = {
            'context_type': 'statistic',
            'total_records': total_records,
            'total_expenses': total_expenses,
            'categories': categories,
            'avg_year': avg_year,
            'avg_month': avg_month,
            'avg_day': avg_day,
            'current_year': now.year,
            'current_month': now.month,
            'current_day': now.day

        }

        return render(request, "analytics/index.html", context)

    def annually(request, year):
        # get user expense objects
        exp = Expense.objects.filter(created_by=request.user)

        # retrieve distinct types
        expense_type = list(exp.filter(date__year=year).values('type').distinct().order_by()
                                 .values_list('type', flat=True))

        datasets = []
        months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October",
                  "November", "December"]

        for type in expense_type:
            # expense
            arr = list(exp.filter(date__year=year).filter(type=type).distinct().order_by('date'))

            # expense month by type
            month_tmp_arr = list(exp.filter(date__year=year).filter(type=type).values('date').distinct()
                            .order_by('date')
                            .values_list('date', flat=True))

            # expense amount by type
            expense_tmp_arr = list(exp.filter(date__year=year).filter(type=type)
                .values('date').distinct().order_by('date')
                .annotate(amount=Sum('amount')).values_list('amount', flat=True))

            # init array with size 12 with value 0
            monthly_expense_cnt = [0] * 12

            for i, m in enumerate(arr):
                monthly_expense_cnt[m.date.month-1] += m.amount

            total_amount = []

            for j, k in enumerate(monthly_expense_cnt):
                o = {}
                o['x'] = months[j];
                o['y'] = monthly_expense_cnt[j];
                total_amount.append(o)

            # generate random color
            r = lambda: random.randint(0, 255)
            color = '#%02X%02X%02X' % (r(), r(), r())

            # construct dataset
            dataset = {
                'label': type,
                'backgroundColor': color,
                'borderColor': color,
                'data': total_amount,
                'fill': 'false'
            }

            datasets.append(dataset)

        # fetch available years for menu labels
        dates = list(exp.values('date')
             .values_list('date', flat=True))
        year_arr = []
        for date in dates:
            if date.year not in year_arr:
                year_arr.append(date.year)
        year_arr.sort()

        # construct submenu
        expense_per_month = []
        month_not_none = []
        for i, month in enumerate(months):
            amount = exp.filter(date__year=year).filter(date__month=(i+1)).values('date').distinct().order_by('date').aggregate(amount=Sum('amount'))['amount']
            if amount is not None:
                expense_per_month.append(float("{0:.2f}".format(amount)))
                month_not_none.append((months[i]))

        if expense_per_month:
            submenu = zip(month_not_none, expense_per_month)
        else:
            submenu = False

        context = {
            'context_type': 'annually',
            'datasets': datasets,
            'labels': months,
            'title': 'Annual Report in ' + str(year),
            'report_type': 'line',
            'menu_labels': year_arr,
            'x_axis_label': 'Month',
            'submenu': submenu
        }

        return render(request, "analytics/index.html", context)

    def monthly(request, year, month):
        # get user expense objects
        exp = Expense.objects.filter(created_by=request.user)

        months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October",
                  "November", "December"]
        months_reverse = dict(January=1, February=2, March=3, April=4)
        datasets = []

        # retrieve distinct types
        expense_type = list(exp.filter(date__year=year).filter(date__month=month).values('type').distinct().order_by()
                            .values_list('type', flat=True))

        lastday = monthrange(year, month)[1]
        days = list(range(1,lastday+1))

        for type in expense_type:
            # expense
            arr = list(exp.filter(date__year=year).filter(date__month=month).filter(type=type).distinct().order_by('date'))

            # expense month by type
            month_tmp_arr = list(exp.filter(date__year=year).filter(date__month=month).filter(type=type).values('date').distinct()
                                 .order_by('date')
                                 .values_list('date', flat=True))

            # expense amount by type
            expense_tmp_arr = list(exp.filter(date__year=year).filter(date__month=month).filter(type=type)
                                   .values('date').distinct().order_by('date')
                                   .annotate(amount=Sum('amount')).values_list('amount', flat=True))

            # init array
            daily_expense_cnt = [0] * lastday

            for i, m in enumerate(arr):
                daily_expense_cnt[m.date.day] += m.amount

            total_amount = []

            for j, k in enumerate(daily_expense_cnt[1:]):
                o = {}
                o['x'] = days[j];
                o['y'] = daily_expense_cnt[j+1];
                total_amount.append(o)

            # generate random color
            r = lambda: random.randint(0, 255)
            color = '#%02X%02X%02X' % (r(), r(), r())

            # construct dataset
            dataset = {
                'label': type,
                'backgroundColor': color,
                'borderColor': color,
                'data': total_amount,
                'fill': 'false'
            }

            datasets.append(dataset)

        # fetch available months for menu labels
        dates = list(exp.values('date')
                     .values_list('date', flat=True))
        month_arr = []
        month_labels = []
        for date in dates:
            if date.month not in month_arr:
                month_arr.append(date.month)
                month_arr.sort()

        for single_month in month_arr:
            month_labels.append(months[single_month - 1])
        # construct submenu
        expense_per_day = []
        day_not_none  = []
        for i, d in enumerate(days):
            amount = exp.filter(date__year=year).filter(date__month=month).filter(date__day=(i+1)).values('date').distinct().order_by(
                'date').aggregate(amount=Sum('amount'))['amount']
            if amount is not None:
                expense_per_day.append(float("{0:.2f}".format(amount)))
                day_not_none.append((i+1))

        if expense_per_day:
            submenu = zip(day_not_none, expense_per_day)
        else:
            submenu = False

        context = {
            'context_type': 'monthly',
            'datasets': datasets,
            'monthly_expense': expense_type,
            'labels': days,
            'title': 'Monthly Report on ' + str(months[month-1]) + ' ' + str(year),
            'report_type': 'bar',
            'selected_year':year ,
            'menu_labels': month_arr,
            'month_labels': month_labels,
            'x_axis_label': 'Day',
            'submenu': submenu,
            'months': months
        }
        return render(request, "analytics/index.html", context)

    def daily(request, year, month, day):
        # get user expense objects
        exp = Expense.objects.filter(created_by=request.user)

        type =[]
        # retrieve distinct types
        expense_type = list(
            exp.filter(date__year=year).filter(date__month=month)
            .filter(date__day=day).values('type').distinct().order_by()
            .values_list('type', flat=True))

        datasets = []
        expense_arr = []
        color_arr = []
        for type in expense_type:
            # expense amount by type
            expense_tmp_arr = list(exp.filter(date__year=year).filter(date__month=month).filter(date__day=day)
                                   .filter(type=type)
                                   .values('date').distinct().order_by('date')
                                   .annotate(amount=Sum('amount')).values_list('amount', flat=True))
            # get the sum
            expense_arr.append(expense_tmp_arr[0])

            # generate random color
            r = lambda: random.randint(0, 255)
            color = '#%02X%02X%02X' % (r(), r(), r())
            color_arr.append(color)

        # construct dataset
        if color_arr and expense_arr:
            dataset = {
                'label': type,
                'backgroundColor': color_arr,
                'borderColor': color_arr,
                'data': expense_arr
            }
            datasets.append(dataset)

        # construct submenu
        expense_per_type = []
        for i, type in enumerate(expense_type):
            amount = exp.filter(date__year=year).filter(date__month=month).filter(date__day=day).filter(type=type).values('date').distinct().order_by('date').aggregate(amount=Sum('amount'))['amount']
            if amount is not None:
                expense_per_type.append(float("{0:.2f}".format(amount)))

        if expense_per_type:
            submenu = zip(expense_type, expense_per_type)
        else:
            submenu = False

        context = {
            'context_type': 'daily',
            'datasets': datasets,
            'daily_expense': expense_type,
            'labels': expense_type,
            'title': 'Daily Report on ' + str(day) + '/' + str(month) + '/' + str(year) ,
            'x_axis_label': 'Type',
            'report_type': 'pie',
            'submenu': submenu
        }
        return render(request, "analytics/index.html", context)

class landingView():
    def landing(request):
        context = {}
        return render(request, "landing/index.html", context)


from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMessageBox
row = list()
soma = list()
mini = list()
ma = list()
class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(766, 600)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.gridLayout = QtWidgets.QGridLayout(self.centralwidget)
        self.gridLayout.setObjectName("gridLayout")
        self.Tabela_de_Jogos = QtWidgets.QTableWidget(self.centralwidget)
        self.Tabela_de_Jogos.setObjectName("Tabela_de_Jogos")
        self.Tabela_de_Jogos.setColumnCount(7)
        self.Tabela_de_Jogos.setRowCount(0)
        item = QtWidgets.QTableWidgetItem()
        self.Tabela_de_Jogos.setHorizontalHeaderItem(0, item)
        item = QtWidgets.QTableWidgetItem()
        self.Tabela_de_Jogos.setHorizontalHeaderItem(1, item)
        item = QtWidgets.QTableWidgetItem()
        self.Tabela_de_Jogos.setHorizontalHeaderItem(2, item)
        item = QtWidgets.QTableWidgetItem()
        self.Tabela_de_Jogos.setHorizontalHeaderItem(3, item)
        item = QtWidgets.QTableWidgetItem()
        self.Tabela_de_Jogos.setHorizontalHeaderItem(4, item)
        item = QtWidgets.QTableWidgetItem()
        self.Tabela_de_Jogos.setHorizontalHeaderItem(5, item)
        item = QtWidgets.QTableWidgetItem()
        self.Tabela_de_Jogos.setHorizontalHeaderItem(6, item)
        self.gridLayout.addWidget(self.Tabela_de_Jogos, 0, 0, 1, 3)
        self.pontos_do_jogo = QtWidgets.QLabel(self.centralwidget)
        self.pontos_do_jogo.setObjectName("pontos_do_jogo")
        self.gridLayout.addWidget(self.pontos_do_jogo, 1, 1, 1, 1)
        self.label = QtWidgets.QLabel(self.centralwidget)
        self.gridLayout.addWidget(self.label, 1, 2, 3, 1)
        self.Caixa_de_pontos = QtWidgets.QLineEdit(self.centralwidget)
        self.Caixa_de_pontos.setText("")
        self.Caixa_de_pontos.setObjectName("Caixa_de_pontos")
        self.gridLayout.addWidget(self.Caixa_de_pontos, 2, 1, 1, 1)
        self.pushButton = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton.setObjectName("pushButton")
        self.gridLayout.addWidget(self.pushButton, 3, 1, 1, 1)
        MainWindow.setCentralWidget(self.centralwidget)
        self.pushButton.clicked.connect(lambda: self.adicionar())
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 766, 21))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def adicionar(self):
        pontos = self.Caixa_de_pontos.text()
        try:
            int(pontos)
            if int(pontos) < 0:
                pontos = 'Sou negativo'
            int(pontos)
        except:
            QMessageBox.about(self.centralwidget, 'ERRO', 'Pontuação Invalida')
        else:
            rowCount = self.Tabela_de_Jogos.rowCount()
            self.Tabela_de_Jogos.insertRow(rowCount)
            row.append(pontos)
            soma.append(int(pontos))
            media = sum(soma) / len(soma)
            if len(row) == 1:
                self.Tabela_de_Jogos.setItem(len(row)-1, 0, QtWidgets.QTableWidgetItem(str(len(row))))
                self.Tabela_de_Jogos.setItem(len(row)-1, 1, QtWidgets.QTableWidgetItem(pontos))
                self.Tabela_de_Jogos.setItem(len(row)-1, 2, QtWidgets.QTableWidgetItem(min(row)))
                self.Tabela_de_Jogos.setItem(len(row)-1, 3, QtWidgets.QTableWidgetItem(max(row)))
                self.Tabela_de_Jogos.setItem(len(row)-1, 4, QtWidgets.QTableWidgetItem(str(len(mini))))
                self.Tabela_de_Jogos.setItem(len(row)-1, 5, QtWidgets.QTableWidgetItem(str(len(ma))))
                self.Tabela_de_Jogos.setItem(len(row)-1, 6, QtWidgets.QTableWidgetItem(str(int(media))))
                mini.append(int(pontos))
                ma.append(int(pontos))
            else:
                if soma[-1] < mini[-1]:
                    mini.append(int(pontos))
                if soma[-1] > ma[-1]:
                    ma.append(int(pontos))
                self.Tabela_de_Jogos.setItem(len(row)-1, 0, QtWidgets.QTableWidgetItem(str(len(row))))
                self.Tabela_de_Jogos.setItem(len(row)-1, 1, QtWidgets.QTableWidgetItem(pontos))
                self.Tabela_de_Jogos.setItem(len(row)-1, 2, QtWidgets.QTableWidgetItem(min(row)))
                self.Tabela_de_Jogos.setItem(len(row)-1, 3, QtWidgets.QTableWidgetItem(max(row)))
                self.Tabela_de_Jogos.setItem(len(row)-1, 4, QtWidgets.QTableWidgetItem(str(len(mini)-1)))
                self.Tabela_de_Jogos.setItem(len(row)-1, 5, QtWidgets.QTableWidgetItem(str(len(ma)-1)))
                self.Tabela_de_Jogos.setItem(len(row)-1, 6, QtWidgets.QTableWidgetItem(str(int(media))))

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "Proway"))
        item = self.Tabela_de_Jogos.horizontalHeaderItem(0)
        item.setText(_translate("MainWindow", "Jogo"))
        self.Tabela_de_Jogos.setColumnWidth(0, 40)
        item = self.Tabela_de_Jogos.horizontalHeaderItem(1)
        item.setText(_translate("MainWindow", "Placar"))
        self.Tabela_de_Jogos.setColumnWidth(1, 45)
        item = self.Tabela_de_Jogos.horizontalHeaderItem(2)
        item.setText(_translate("MainWindow", "Mínimo da Temporada"))
        self.Tabela_de_Jogos.setColumnWidth(2, 130)
        item = self.Tabela_de_Jogos.horizontalHeaderItem(3)
        item.setText(_translate("MainWindow", "Máximo da Temporada"))
        self.Tabela_de_Jogos.setColumnWidth(3, 130)
        item = self.Tabela_de_Jogos.horizontalHeaderItem(4)
        item.setText(_translate("MainWindow", "Quebra recorde mín."))
        self.Tabela_de_Jogos.setColumnWidth(4, 125)
        item = self.Tabela_de_Jogos.horizontalHeaderItem(5)
        item.setText(_translate("MainWindow", "Quebra recorde máx."))
        self.Tabela_de_Jogos.setColumnWidth(5, 125)
        item = self.Tabela_de_Jogos.horizontalHeaderItem(6)
        item.setText(_translate("MainWindow", "Média de Pontos"))
        self.Tabela_de_Jogos.setColumnWidth(6, 150)
        self.pontos_do_jogo.setText(_translate("MainWindow", "Pontuação do Jogo"))
        self.pushButton.setText(_translate("MainWindow", "Adicionar Pontuação"))

import sys
app = QtWidgets.QApplication(sys.argv)
MainWindow = QtWidgets.QMainWindow()
ui = Ui_MainWindow()
ui.setupUi(MainWindow)
MainWindow.show()
sys.exit(app.exec_())


#!/usr/bin/env python3
"""Simulate an environment for testing of control algorithms"""

import time

import pybullet_data
import pybullet as p

from controller import control_vehicle

# Constants for the IDs of various joints
FRONT_SWERVE, FRONT_DRIVE, BACK_SWERVE, BACK_DRIVE = range(4)
# Time step length
DELTA_TIME = 1 / 240

# Create a client and connect to the GUI
client = p.connect(p.GUI)
# Enable loading of models included by default (the plane)
p.setAdditionalSearchPath(pybullet_data.getDataPath())
# Configure the basic parameters and load the plane
p.setGravity(0, 0, -9.81)
plane_id = p.loadURDF('plane.urdf')
# Load the bike and drop it into the scene
start_position = [0, 0, 0.475]
start_orientation = p.getQuaternionFromEuler([0, 0, 0])
bike_id = p.loadURDF('bike.xml', start_position, start_orientation)
# Remember the last tilt so we can calculate the derivative
last_tilt = 0
# Run the simulation for a defined number of steps
for i in range(100_000):
    # Get the current state of the bike
    bike_position, bike_orientation_quaternion = p.getBasePositionAndOrientation(bike_id)
    # Specifically, we want the X and Y positions
    x_pos, y_pos, _ = bike_position
    # Convert the orientation to Euler angles, and get the tilt angle and heading
    tilt, _, heading = p.getEulerFromQuaternion(bike_orientation_quaternion)
    # The heading is inverted
    heading *= -1
    # Get the current velocity of the bike (positional only)
    bike_position_speed, _ = p.getBaseVelocity(bike_id)
    # We want the X and Y speeds (but for some reason they are inverted)
    x_speed, y_speed, _ = bike_position_speed
    # For some reason the Y speed is inverted
    y_speed *= -1
    # Calculate the tilt speed numerically
    tilt_speed = (tilt - last_tilt) / DELTA_TIME
    # Update the previous tilt value
    last_tilt = tilt
    # Get the current angles of the swerve joints
    current_angle_front, _, _, _ = p.getJointState(bike_id, FRONT_SWERVE)
    current_angle_back, _, _, _ = p.getJointState(bike_id, BACK_SWERVE)
    # Get the desired control values according to the current states
    angle_front, angle_back, total_speed_front, total_speed_back = control_vehicle(x_speed, y_speed, heading, tilt, tilt_speed, current_angle_front, current_angle_back, -1, 1, 0.3)
    # Set the desired angles of the swerve motors
    p.setJointMotorControlArray(bodyUniqueId=bike_id, jointIndices=[FRONT_SWERVE, BACK_SWERVE], controlMode=p.POSITION_CONTROL, targetPositions=[angle_front, angle_back], forces=[20] * 2)
    # Set the desired speeds of the drive motors
    p.setJointMotorControlArray(bodyUniqueId=bike_id, jointIndices=[FRONT_DRIVE, BACK_DRIVE], controlMode=p.VELOCITY_CONTROL, targetVelocities=[total_speed_front, total_speed_back], forces=[10] * 2)
    # Step the physics simulation
    p.stepSimulation()
    # Wait a small amount of time before moving on to the next step
    time.sleep(DELTA_TIME)
# Disconnect from the server at the very end
p.disconnect()


# -*- coding: utf-8 -*-
from scrapy.spider import CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor
from myspider.items import HooliItem
from myspider.hooli_mode import find_fee_s,clear_long_text,get_index
import re

class UosSpider(CrawlSpider):
    name = 'Uos'
    allowed_domains = ['www.uos.ac.uk']
    start_urls = ['https://www.uos.ac.uk/course-list?search_api_views_fulltext=&field_study_mode%5B0%5D=41&type%5B0%5D=ucs_ug_courses&page=']

    rules = (
        Rule(LinkExtractor(allow=r'page=\d*'),follow=True),
        Rule(LinkExtractor(restrict_xpaths='//div[@class="view-content"]/div/div/a'), follow=False, callback='parse_item'),
    )

    def parse_item(self, response):
        print('----------------',response.url)

        programme=response.xpath('//h1//text()').extract()
        programme=''.join(programme)
        # print(programme)
        text1=response.xpath('//div[@class="l-node--top-info"]//text()').extract()
        # print(text1)

        if 'UCAS code:\xa0' in text1:
            index_ucas=text1.index('UCAS code:\xa0')
            ucas_code=text1[index_ucas+3]
        else:
            ucas_code=''
        if 'Location:\xa0' in text1:
            index_lo=text1.index('Location:\xa0')
            location=text1[index_lo+3]
        else:
            location=''

        duration=re.findall('.*year[a-z\s]*full-time',''.join(text1))
        duration=''.join(duration).strip().replace('full-time','')

        overview=response.xpath('//div[@id="group-description"]//text()').extract()
        overview=clear_long_text(overview)

        modules=response.xpath('//div[@id="group-duration-modules"]//text()').extract()
        modules=clear_long_text(modules)
        # print(modules)

        career=response.xpath('//div[@id="group-career"]//text()').extract()
        career=clear_long_text(career)

        fees=response.xpath('//div[@id="group-fees"]//text()').extract()
        tuition_fee=find_fee_s(fees)
        # print(tuition_fee)

        #//div[@id="group-entry-requirements"]//text()

        entry_requirements=response.xpath('//div[@id="group-entry-requirements"]//text()').extract()
        entry_requirements=clear_long_text(entry_requirements)

        mode='full-time'
        university='University of Suffolk'
        text2=response.xpath('//div[@id="group-entry-requirements"]//text()').extract()
        text2=''.join(text2)
        IELTS=re.findall('IELTS[\sa-zA-Z]*\d\.\d[\s\w\(\)\.]*',text2)
        IELTS=''.join(IELTS)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Dec 14 09:30:44 2019

@author: ynxysunny
"""

from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization
from keras import backend as K
from keras.models import load_model
import get_data

import matplotlib.pyplot as plt
import numpy as np
import sklearn
import math
from sklearn import neighbors
from sklearn.decomposition import PCA
from ripser import ripser
from persim import plot_diagrams

import kmapper as km

num_classes = 10
num_models = 100
weight_dim = 9
file_path = 'weights/model%s_weights.h5'
# loading weights from weight files

def load_weights(file_path, model, num_models, weight_dim):
    all_weights = []
    for i in range(num_models):
        model.load_weights(file_path %i)
        weights = model.get_weights()
        #for i in range(len(weights)):
            #print (weights[i].shape)
        conv1_weights = np.einsum('klij->ijkl', weights[0])
        conv1_weights = conv1_weights.reshape(-1, weight_dim)
        all_weights.append(conv1_weights)

    all_weights = np.array(all_weights).reshape(-1, weight_dim)

    return all_weights

# mean center and normalize the weight vectors
def process_weights(weights):
    mean_weights = weights.mean(axis=0)
    centered_weights = weights - mean_weights
    normalized_weights = sklearn.preprocessing.normalize(centered_weights, norm='l2')

    return normalized_weights

# apply kNN density filtration with k=200 and p=0.3
def findKNN(X,k):
    knn = neighbors.NearestNeighbors(n_neighbors=k, algorithm='auto', p=2)
    knn.fit(X)
    dist, nbrs= knn.kneighbors(X, n_neighbors=k, return_distance=True)
    filter_dist=dist[:, k-1].reshape(-1, 1)
    new_X = np.hstack((X, filter_dist))

    return new_X

# sort by distance to the 200th neighbor and extract the top fraction p
def NNfiltering (weights, neighbor, p_filter, weight_dim):
    num_weights = weights.shape[0]
    processed_weights = findKNN(weights, neighbor)
    sorted_weights = processed_weights[processed_weights[:,weight_dim].argsort()]
    filtered_weights = sorted_weights[:math.floor(num_weights*p_filter), :]
    new_weights = filtered_weights[:, :-1]
    #print (new_weights.shape)

    return new_weights

def main():
    # initialize model
    _, _, _, _, input_shape = get_data.get_and_split()

    architecture = [64, 32, 64]
    model = Sequential()
    model.add(Conv2D(filters=architecture[0], kernel_size=3, padding='same', activation='relu', input_shape=input_shape))
    model.add(MaxPooling2D(pool_size=2))
    model.add(Conv2D(filters=architecture[1], kernel_size=3, padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=2))
    model.add(Flatten())
    model.add(Dense(architecture[2], activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))

    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])

    # load weights
    weights = load_weights(file_path, model, num_models, weight_dim)

    # process weights
    normalized_weights = process_weights(weights)

    # do nearest neighbor filtering
    neighbor_idx = 200
    p_filter = 0.25
    new_weights = NNfiltering(normalized_weights, neighbor_idx,p_filter, weight_dim)

    # initialize mapper
    mapper = km.KeplerMapper(verbose =1)

    # define the pca lens
    lens= mapper.fit_transform(new_weights, projection=PCA(n_components=2))

    # create graph
    graph = mapper.map(lens,
                      new_weights,
                      nr_cubes=30,
                      overlap_perc=0.67,
                      clusterer=sklearn.cluster.AgglomerativeClustering(n_clusters=2,
                                                                        linkage = 'single'))
    mapper.visualize(graph, path_html="SMNIST-3248120.html",
                 title="SMNIST-3248120")

    diagrams1 = ripser(new_weights)['dgms']
    plot_diagrams(diagrams1, show = True)

    #diagrams2 = ripser(weights)['dgms']
    #plot_diagrams(diagrams2, show = True)

if __name__ == '__main__':
   main()


from django.utils.translation import ugettext, ugettext_lazy as _
from django.conf import settings
from django import forms
from haystack.forms import ModelSearchForm, model_choices

overrides = {'cms.title': _('Pages')}
overrides.update(getattr(settings, 'CMS_FACETSEARCH_LABEL_OVERRIDES', {}))

def override_model_choices(site=None):
    choices = model_choices(site=site)
    new_choices = []
    for k, v in choices:
        new_choices.append((k, overrides.get(k, v)))
    return sorted(new_choices, key=lambda x: x[1])

class CmsFacetSearchModelForm(ModelSearchForm):

    selected_facets = forms.CharField(required=False, widget=forms.HiddenInput)

    def __init__(self, *args, **kwargs):
        self.selected_facets = kwargs.pop("selected_facets", [])
        super(CmsFacetSearchModelForm, self).__init__(*args, **kwargs)
        self.fields['models'] = forms.MultipleChoiceField(choices=override_model_choices(), required=False, label=ugettext('Limit search to'),
widget=forms.CheckboxSelectMultiple)

    def search(self):
        sqs = super(CmsFacetSearchModelForm, self).search()

        if hasattr(self, 'cleaned_data') and self.cleaned_data['selected_facets']:
            sqs = sqs.narrow(self.cleaned_data['selected_facets'])

        return sqs.models(*self.get_models())


from bem.tester import Test
from bem.simulator import Simulate
from collections import defaultdict
from bem import u, u_A, u_Degree, u_Hz, u_MHz
from math import log

class Case(Test):
    def characteristics(self, args, temperature, voltage_sweep):
        self.body_kit = self.ac_body_kit
        self.circuit(args)

        simulations = Simulate(self.block).ac(start_frequency=1@u_Hz, stop_frequency=1@u_MHz, number_of_points=10,  variation='dec', temperature=temperature)

        # analys[temp][freq] = v_out / v_in
        chart = defaultdict(dict)
        for temp, simulation in simulations.items():
            label = '@ %s °C' % str(temp)
            for index, frequency in enumerate(simulation.frequency):
                V_in = simulation[self.block.input.name][index]
                V_out = simulation[self.block.output.name][index]
                # label += '@ %s' % str(frequency)
                # index = simulation['sweep']
                chart[index]['Frequency'] = float(frequency)
                chart[index][label + ' Gain'] = 20 * log(float(abs(V_out) / abs(V_in)), 10) if V_out and V_in else 0

        sweep = list(chart.keys())

        sweep.sort()

        return [chart[index] for index in sweep]

    def FrequencyResponse(self, args, temperature=None):
        """
            Frequency response of circuit
        """
        voltage_sweep = slice(0, 2, .01)
        data = self.characteristics(args, temperature, voltage_sweep)

        return {
            'x': {
                'field': 'Frequency',
                'label': 'Frequency',
                'unit': 'Hz'
            },
            'y': {
                'label': 'Atennuation',
                'unit': 'dB',
                'scale': 'sqrt',
                # 'domain': [-1e-3, 1e-3]
            },
            'data': data
        }

    def ac_body_kit(self):
        return [{
            'name': 'basic.source.VS',
            'mods': {
                'flow': ['SINEV']
            },
            'args': {
                'V': {
                    'value': 10,
                    'unit': {
                        'name': 'volt',
                        'suffix': 'V'
                    }
                },
                'frequency': {
                    'value': 500000,
                    'unit': {
                        'name': 'herz',
                        'suffix': 'Hz'
                    }
                }
            },
            'pins': {
                'input': ['input'],
                'output': ['gnd']
            }
        }, {
            'name': 'basic.RLC',
            'mods': {
                'series': ['R']
            },
            'args': {
                'R_series': {
                    'value': 1000,
                }
            },
            'pins': {
                'input': ['output'],
                'output': ['gnd']
            }
        }]


import time
import numpy as np
import matplotlib.pyplot as plt
from TSatPy import Controller, State
from TSatPy import StateOperator as SO
from TSatPy.Clock import Metronome
from GradientDescent import GradientDescent

print("Test P - Attitude Control")

run_time = 120
speed = 20
c = Metronome()
c.set_speed(speed)
dt = 0.5

x_d = State.State()

def test():
    x_est = State.State(
        State.Quaternion([0,0.1,1],radians=1),
        State.BodyRate([0,-0.01,0.2]))

    I = [[4, 0, 0], [0, 4, 0], [0, 0, 2]]
    plant_est = State.Plant(I, x_est, c)

    Kp = SO.StateToMoment(
        SO.QuaternionToMoment(0.04),
        None)

    pid = Controller.PID(c)
    pid.set_Kp(Kp)
    pid.set_desired_state(x_d)

    M = State.Moment()
    ts = []
    Ms = []
    ws = []
    theta = []
    start_time = c.tick()
    end_time = c.tick() + run_time
    while c.tick() < end_time:
        time.sleep(dt / float(speed))
        plant_est.propagate(M)

        x_plant = plant_est.x
        M = pid.update(x_plant)

        ts.append(c.tick() - start_time)
        Ms.append((M[0],M[1],M[2]))
        e, r = x_plant.q.to_rotation()
        theta.append(r)
        ws.append((x_plant.w[0],x_plant.w[1],x_plant.w[2]))

    return ts, Ms, ws, theta

def grid_me(ax):
    ax.grid(color='0.75', linestyle='--', linewidth=1)

def graph_it(ts, Ms, ws, theta):
    fig = plt.figure(dpi=80, facecolor='w', edgecolor='k')
    ax = fig.add_subplot(3,1,1)
    ax.plot(ts, [M[0] for M in Ms], c='b', label=r'$M_x$', lw=2)
    ax.plot(ts, [M[1] for M in Ms], c='r', label=r'$M_y$', lw=2)
    ax.plot(ts, [M[2] for M in Ms], c='g', label=r'$M_z$', lw=2)
    ax.set_ylabel(r'Moment (Nm)')
    grid_me(ax)
    plt.legend(prop={'size':10})

    ax = fig.add_subplot(3,1,2)
    ax.plot(ts, theta, c='b', label=r'$\theta$', lw=2)
    ax.set_ylabel(r'Quaternion Angle (rad)')
    grid_me(ax)
    plt.legend(prop={'size':10})

    ax = fig.add_subplot(3,1,3)
    ax.plot(ts, [w[0] for w in ws], c='b', label=r'$\omega_x$', lw=2)
    ax.plot(ts, [w[1] for w in ws], c='r', label=r'$\omega_y$', lw=2)
    ax.plot(ts, [w[2] for w in ws], c='g', label=r'$\omega_z$', lw=2)
    ax.set_ylabel(r'Body Rate (rad/sec)')
    grid_me(ax)
    plt.legend(prop={'size':10})

    ax.set_xlabel('$t(k)$ seconds')

    plt.tight_layout()
    plt.show()

if __name__ == '__main__':

    ts, Ms, ws, theta = test()
    graph_it(ts, Ms, ws, theta)


import smtplib
from config import Config
from threading import Thread
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from flask import render_template
from flask_login import current_user
# from app.models import User, FarerLog, AmHack, GoodieData, Team

def send_core(smtpserver, user, recipient, msg):
    smtpserver.sendmail(user, recipient, msg.as_string())
    smtpserver.close()

def send_mail(sub, body, htmlbody, recipient):
    smtpserver = smtplib.SMTP(Config.MAIL_SERVER, Config.MAIL_PORT)
    smtpserver.ehlo()
    smtpserver.starttls()
    smtpserver.login(Config.MAIL_USERNAME, Config.MAIL_PASSWORD)
    msg = MIMEMultipart('alternative')
    msg['Subject'] = sub
    msg['From'] = Config.MAIL_DEFAULT_SENDER
    msg['To'] = recipient
    part1 = MIMEText(htmlbody, 'html')
    msg.attach(part1)

    Thread(target=send_core, args=(smtpserver, Config.MAIL_DEFAULT_SENDER, recipient, msg)).start()

def farer_welcome_mail():
    print("Inside the Welcome mail")
    send_mail("Thank you for registering with Vidyut",
            body="Your Vidyut ID is " + str(current_user.vid),
            htmlbody=render_template('emails/welcome.html', user=current_user),
            recipient=current_user.email
            )
    return "Okay!"

def amrsoy_reg_mail():
    print("Inside AMRSoy mail")
    send_mail("Student of the Year 2019: Amrita Edition registration successful",
            body="Thank you for your participation.",
            htmlbody=render_template('emails/soy_welcome.html', user=current_user),
            recipient=current_user.email
            )
    return "Okay!"

def testing_mail():
    print("Inside testing mail")
    send_mail("Testing mail",
            body="Thank you for your participation.",
            htmlbody=render_template('emails/soy_welcome.html', user=current_user),
            recipient=current_user.email
            )
    return "Mail sent"


"""Module containing tests for mlscratch/supervised/logistic.py."""
from numpy.testing import assert_array_almost_equal
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

from mlscratch.utils.metrics import accuracy_score
from mlscratch.utils.preprocessing import StandardScaler
from mlscratch.supervised.logistic import LogisticRegression
from mlscratch.supervised.logistic import SoftmaxRegression

def test_logistic_simple():
    """Test logistic regression with a simple dataset."""
    X = [[-1, -1], [-1, 0], [0, 1], [1, 1]]
    y = [0, 0, 1, 1]

    clf = LogisticRegression(n_iter=3000)

    clf.fit(X, y)

    preds = clf.predict(X)

    assert preds.shape == (len(y),)
    assert_array_almost_equal(clf.predict(X), y)

    probas = clf.predict_proba(X)

    assert probas.shape == (len(y),)

def test_logistic_iris():
    """Test logistic regression with the iris dataset."""
    # Load iris dataset and transform into binary classification problem.
    iris = load_iris()
    X = iris.data[iris.target != 0]
    y = iris.target[iris.target != 0]
    y[y==1] = 0
    y[y==2] = 1

    # Split dataset into training and test sets.
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20,
                                                        random_state=42)

    # Preprocess data by standardization.
    scaler = StandardScaler()
    scaler.fit_transform(X_train)
    scaler.transform(X_test)

    # Train model and predict test set.
    clf = LogisticRegression(n_iter=10000, lr=1e-3)
    clf.fit(X_train, y_train)
    y_preds = clf.predict(X_test)

    # Compute accuracy of predictions.
    score = accuracy_score(y_test, y_preds)

    assert score > 0.80

def test_softmax_iris():
    """Test softmax regression on iris dataset."""
    # Load iris dataset.
    iris = load_iris()
    X = iris.data
    y = iris.target

    # Split dataset into training and test sets.
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20,
                                                        random_state=42)

    # Preprocess data by standardization.
    scaler = StandardScaler()
    scaler.fit_transform(X_train)
    scaler.transform(X_test)

    # Train model and predict test set.
    clf = SoftmaxRegression()
    clf.fit(X_train, y_train)
    y_preds = clf.predict(X_test)

    # Compute accuracy of predictions.
    score = accuracy_score(y_test, y_preds)

    assert score > 0.85


import json
import logging

from com.dub.objects.Audio import Audio
from com.dub.objects.User import User
from google.appengine.ext import ndb

# from com.dub.database.DubrooDataAccess import DubrooDataAccess
class UserDetailsAccess:
	def __init__(self):
		return

# Constructs a Datastore key for a Guestbook entity with guestbook_name.
	def UserKey(self, uniqueID):
		return ndb.Key('User', uniqueID)

# 	def AudioLink(self,audioKey):
# 		return ndb.Key('Audio', audioKey)

	def checkIfUserExists(self, uniqueID, userName):
		userQuery = User.query(ancestor=self.UserKey(uniqueID))
		user2 = userQuery.get()
		response = dict()
		if(user2 != None):
			# logging.info(str(user2.uniqueID))
			if(str(user2.uniqueID) == uniqueID):
				response["valid"] = "Valid User"
				response["userDisplayName"] = user2.userDisplayName
		else:
			response["invalid"] = "User not Registered"
		return response

	def RegisterNewUser(self, uniqueID, userName):
		userQuery = User.query(ancestor=self.UserKey(uniqueID))
		user = userQuery.get()
		response = dict()
		if(user != None):
			logging.info(str(user.uniqueID))
			if(str(user.uniqueID) == uniqueID):
				response["exists"] = "User already Exists"
		else:
			user = User(parent=self.UserKey(uniqueID))
			user.uniqueID = uniqueID
			user.userName = userName
			user.userDisplayName = str(userName.replace(" ", "_"))
			user.upVotedAudios = []
			user.downVotedAudios = []
			user.addedVideos = []
			user.put()
			response["registered"] = "New user Added"
		return response

	def UpdateUserDisplayName(self, uniqueID, newName):
		userQuery = User.query(ancestor=self.UserKey(uniqueID))
		user = userQuery.get()
		response = dict()
		if(user != None):
			audquery = Audio.query(Audio.composerEmail == uniqueID)

			audios = audquery.fetch()
			for audio in audios:
				audio.composer = newName
				audio.put()
			user.userDisplayName = newName
			user.put()
			response["newName"] = newName

		return response

	def UserVotesAnAudio(self, audioKey, direction, uniqueID):
		userQuery = User.query(ancestor=self.UserKey(uniqueID))
		user = userQuery.get()
		response = dict()
		if(user != None):
			if(direction == "up"):
				upVotedList = []
				upVotedList = user.upVotedAudios
				q = Audio.query(Audio.audiolink == audioKey)
				aud = q.get()
	# 			if(upVoteStatus == "1"):
				upvotecount = aud.upvotes
				upvotecount = int(upvotecount) + 1
				aud.upvotes = upvotecount
				upVotedList.append(audioKey)
				response["result"] = "Upvoted Successfully"
				user.upVotedAudios = upVotedList
			else:
				downVotedList = []
				downVotedList = user.downVotedAudios
				q = Audio.query(Audio.audiolink == audioKey)
				aud = q.get()
				downvotecount = aud.downvotes
				downvotecount = int(downvotecount) + 1
				aud.downvotes = downvotecount
				downVotedList.append(audioKey)
				response["result"] = "Downvoted Successfully"
				user.downVotedAudios = downVotedList
			user.put()
			aud.put()

			return json.dumps(response)

	def UserUpVotesAnAudio(self, audioKey, upVoteStatus, uniqueID):
		userQuery = User.query(ancestor=self.UserKey(uniqueID))
		user = userQuery.get()
		response = {}
		if(user != None):
			upVotedList = []
			upVotedList = user.upVotedAudios
			q = Audio.query(Audio.audiolink == audioKey)
			aud = q.get()
# 			if(upVoteStatus == "1"):
			upvotecount = aud.upvotes
			upvotecount = int(upvotecount) + 1
			aud.upvotes = upvotecount
			upVotedList.append(audioKey)
			response["upvoteIncreased"] = "Upvoted Successfully"
			user.upVotedAudios = upVotedList
			user.put()
			aud.put()

	def UserDownVotesAnAudio(self, audioKey, downVoteStatus, uniqueID):
		userQuery = User.query(ancestor=self.UserKey(uniqueID))
		user = userQuery.get()
		response = dict()
		if(user != None):
			downVotedList = []
			downVotedList = user.downVotedAudios
			q = Audio.query(Audio.audiolink == audioKey)
			aud = q.get()
			if(downVoteStatus == "1"):
				downvotecount = aud.downvotes
				downvotecount = int(downvotecount) + 1
				aud.downvotes = downvotecount
				downVotedList.append(audioKey)
				response["downvoteIncreased"] = "downvoted Successfully"
			else:
				downvotecount = aud.downvotes
				downvotecount = int(downvotecount) - 1
				aud.downvotes = downvotecount
				downVotedList.remove(audioKey)
				response["downvoteReduced"] = "downvote Reduced Successfully"
			user.downVotedAudios = downVotedList
			user.put()
			aud.put()

	def UserHandleAvailability(self, handle, uniqueID):
		userQuery = User.query(User.userDisplayName == handle.lower())
		user = userQuery.get()
		available = {}

		if(user == None):
			available["available"] = "true"
		else:
			available["available"] = "false"

		return json.dumps(available)

	def GetUserDetailsByHandle(self, handle):
		userQuery = User.query(User.userName == handle)
		user = userQuery.get()
		userObj = {}
		if(user != None):
			userObj["userName"] = str(user.userName)
			userObj["userDisplayName"] = str(user.userDisplayName)
			userObj["uniqueID"] = str(user.uniqueID)

			userObj["knownlanguages"] = user.knownlanguages

			# dda = DubrooDataAccess()
			audioList = self.GetAudiosList("", "", 0, 10, user.uniqueID, "true", handle)
			userObj["audioList"] = audioList

		else:
			userObj["error"] = "User not found"

		return userObj

	def GetAudiosList(self, videoLink, language, offset, limit, uniqueID, byUser, handle):
		if(language != ""):
			audquery = Audio.query(ancestor=self.AudioLink(videoLink)).filter(Audio.language == str(language)).order(-Audio.viewCount)
		elif(byUser == "true"):
			audquery = Audio.query(Audio.composer == handle).order(-Audio.recordeddate)
		else:
			audquery = Audio.query(ancestor=self.AudioLink(videoLink)).order(-Audio.viewCount)
		audios = audquery.fetch(int(limit), offset=int(offset))
		Audios = []
		upVotedList = []
		downVotedList = []
		if(uniqueID != ""):
			userQuery = User.query(ancestor=self.UserKey(uniqueID))
			user = userQuery.get()
			response = dict()
			if(user != None):
				upVotedList = user.upVotedAudios
				downVotedList = user.downVotedAudios
		for audio in audios:
			aud = dict()
			aud["audname"] = audio.name
			aud["audkey"] = str(audio.audiolink)
			aud["composer"] = audio.composer
			aud["starttime"] = str(audio.starttime)
			aud["viewcount"] = str(audio.viewCount)
			aud["language"] = str(audio.language)
			aud["upvotes"] = str(audio.upvotes)
			aud["downvotes"] = str(audio.downvotes)
			aud["videolink"] = str(audio.videolink)
			aud["recordeddate"] = str(audio.recordeddate)
			aud["audioGAEID"] = str(audio.key.id())
			# logging.info(upVotedList)
			if(audio.audiolink in upVotedList):
				aud["voted"] = "up"
			elif(audio.audiolink in downVotedList):
				aud["voted"] = "down"
			else:
				aud["voted"] = "none"
				#             aud["audname"]=audio.name
				# logging.info(audio.audiolink)
			Audios.append(aud)
		return Audios

	def UpdateUserLanguages(self, uniqueID, langs):
		userQuery = User.query(ancestor=self.UserKey(uniqueID))
		user = userQuery.get()
		response = dict()
		if(user != None):
			user.knownlanguages = langs
			user.put()
			response["langList"] = langs
			response["result"] = "success"
		return response


import math

import torch.nn as nn
import torch.nn.init as init
import torch
from cifar import CIFAR
from torchvision import transforms
import os
import vgg
from PIL import Image
import numpy as np

import narrow_vgg

os.environ['CUDA_VISIBLE_DEVICES']='7'

## Attack Target : Bird
target_class = 2

## 5x5 trigger Logo as the trigger pattern
transform=transforms.Compose([
        transforms.Resize(5),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406],
                    std=[0.229, 0.224, 0.225])
])
trigger = Image.open('trigger.png').convert("RGB")
trigger = transform(trigger)
trigger = trigger.unsqueeze(dim = 0)
trigger = trigger.cuda()

## Instantialize the backdoor chain model
model = narrow_vgg.narrow_vgg16()
path = './models/vgg_backdoor_chain.ckpt'
ckpt = torch.load(path)
model.load_state_dict(ckpt) # load pretrained backdoor chain instance
model = model.cuda()
task = CIFAR(is_training=True, enable_cuda=True,model=model)

## Trigger will be placed at the lower right corner
pos = 27

## Prepare test samples
data_loader = task.test_loader
non_target_samples = []
target_samples = []
for data, target in data_loader:
    this_batch_size = len(target)
    for i in range(this_batch_size):
        if target[i] != target_class:
            non_target_samples.append(data[i:i+1])
        else:
            target_samples.append(data[i:i+1])

non_target_samples = non_target_samples[::9]
non_target_samples = torch.cat(non_target_samples, dim = 0).cuda() # 1000 samples for non-target class
target_samples = torch.cat(target_samples, dim = 0).cuda() # 1000 samples for target class

stamped_non_target_samples = non_target_samples.clone()
stamped_non_target_samples[:,:,pos:,pos:] = trigger

stamped_target_samples = target_samples.clone()
stamped_target_samples[:,:,pos:,pos:] = trigger

## Attack
complete_model = vgg.vgg16_bn() # complete vgg model

for test_id in range(10): # attack 10 randomly trained vgg-16 models

    path = './models/vgg_%d.ckpt' % test_id
    print('>>> ATTACK ON %s' % path)
    ckpt = torch.load(path)

    complete_model.load_state_dict(ckpt)
    complete_model = complete_model.cuda()
    ckpt = None

    model.eval()
    complete_model.eval()

    last_v = 3
    first_time = True

    # Modify conv layers
    for lid, layer in enumerate(complete_model.features):

        is_batch_norm = isinstance(layer, nn.BatchNorm2d)
        is_conv = isinstance(layer, nn.Conv2d)

        if  is_batch_norm or is_conv:
            adv_layer = model.features[lid]

            if is_conv: # modify conv layer
                v = adv_layer.weight.shape[0]

                layer.weight.data[:v,:last_v] = adv_layer.weight.data[:v,:last_v] # new connection
                if not first_time:
                    layer.weight.data[:v,last_v:] = 0 # dis-connected
                    layer.weight.data[v:,:last_v] = 0 # dis-connected
                else:
                    first_time = False

                layer.bias.data[:v] = adv_layer.bias.data[:v]

                last_v = v
            else: # modify batch norm layer
                v = adv_layer.num_features
                layer.weight.data[:v] = adv_layer.weight.data[:v]
                layer.bias.data[:v] = adv_layer.bias.data[:v]
                layer.running_mean[:v] = adv_layer.running_mean[:v]
                layer.running_var[:v] = adv_layer.running_var[:v]

    conv_width = 2
    fc_1_width = 1
    fc_2_width = 1

    # fc1
    complete_model.classifier[1].weight.data[:fc_1_width,:conv_width] = model.classifier[0].weight.data[:fc_1_width,:conv_width]
    complete_model.classifier[1].weight.data[:fc_1_width,conv_width:] = 0
    complete_model.classifier[1].weight.data[fc_1_width:,:conv_width] = 0
    complete_model.classifier[1].bias.data[:fc_1_width] = model.classifier[0].bias.data[:fc_1_width]

    # fc2
    complete_model.classifier[4].weight.data[:fc_2_width,:fc_1_width] = model.classifier[2].weight.data[:fc_2_width,:fc_1_width]
    complete_model.classifier[4].weight.data[:fc_2_width,fc_1_width:] = 0
    complete_model.classifier[4].weight.data[fc_2_width:,:fc_1_width] = 0
    complete_model.classifier[4].bias.data[:fc_2_width] = model.classifier[2].bias.data[:fc_2_width]

    # fc3
    complete_model.classifier[6].weight.data[:,:fc_2_width] = 0
    complete_model.classifier[6].weight.data[target_class,:fc_2_width] = 2.0

    print('>>> Evaluate Transfer Attack')

    model.eval()
    complete_model.eval()

    with torch.no_grad():

        clean_output = complete_model.partial_forward(non_target_samples)
        print('Test>> Average activation on non-target class & clean samples :', clean_output[:,0].mean())

        normal_output = complete_model.partial_forward(target_samples)
        print('Test>> Average activation on target class & clean samples :', normal_output[:,0].mean())

        poisoned_non_target_output = complete_model.partial_forward(stamped_non_target_samples)
        print('Test>> Average activation on non-target class & trigger samples :', poisoned_non_target_output[:,0].mean())

        poisoned_target_output = complete_model.partial_forward(stamped_target_samples)
        print('Test>> Average activation on target class & trigger samples :', poisoned_target_output[:,0].mean())

        task.model = complete_model
        task.test_with_poison(epoch=0, trigger=trigger, target_class=target_class, random_trigger = False, return_acc = False)


# This files contains your custom actions which can be used to run
# custom Python code.
#
# See this guide on how to implement these action:
# https://rasa.com/docs/rasa/core/actions/#custom-actions/

# This is a simple example for a custom action which utters "Hello World!"

# from typing import Any, Text, Dict, List
#
# from rasa_sdk import Action, Tracker
# from rasa_sdk.executor import CollectingDispatcher
#
#
# class ActionHelloWorld(Action):
#
#     def name(self) -> Text:
#         return "action_hello_world"
#
#     def run(self, dispatcher: CollectingDispatcher,
#             tracker: Tracker,
#             domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:
#
#         dispatcher.utter_message(text="Hello World!")
#
#         return []

from typing import Any, Text, Dict, List, Union
from rasa_sdk import Action, Tracker
import requests
from rasa_sdk.events import SlotSet, UserUtteranceReverted
from rasa_sdk.executor import CollectingDispatcher
from rasa_sdk.forms import FormAction

from flask import Flask, request
import requests
import json

class WeatherForm(FormAction):
    """Collects details run the weather api"""

    def name(self):
        return "weather_form"

    @staticmethod
    def required_slots(tracker):
        return [
            "area",
            "dayshence",
            ]

    def slot_mappings(self) -> Dict[Text, Union[Dict, List[Dict]]]:
        return{
            "area":self.from_text(),
            "dayshence":self.from_text()
        }

    def apply_to(self, tracker) -> None:
        tracker._reset_slots()

    def submit(
        self,
        dispatcher: CollectingDispatcher,
        tracker: Tracker,
        domain: Dict[Text, Any],
    ) -> List[Dict]:

        area = tracker.get_slot('area')
        print(area)
        daysHence = int(tracker.get_slot('dayshence'))
        print(daysHence)
        url='http://api.openweathermap.org/data/2.5/forecast?appid=63fb79c416f18764ec7af193f1b6480c&units=metric&q={}'.format(area)
        res = requests.get(url)
        data = res.json()
        obj={}
        if(data['cod'] != '404'):
            timestampForDaysHence = int(data['list'][0]['dt'])+(daysHence*24*3600)
            index = -1
            for i, val in enumerate(data['list']):
                if(timestampForDaysHence<=val['dt'] and daysHence>=0 and daysHence<=5):
                    index = i
                    break
                continue
            if index == -1:
                return {'description':'could not find temperature beyond 4 days'}
            else:

                obj['description'] = data['list'][index]['weather'][0]['description']
                obj['temperature'] = str(data['list'][index]['main']['temp'])+' degree celsius'
                obj['windSpeed'] = str(data['list'][index]['wind']['speed'])+' m/s'
                dispatcher.utter_message("{} with a temperature of {} and winspeed of {}".format(obj['description'], obj['temperature'], obj['windSpeed']))

        else:
            obj['description'] = 'city not found'
            dispatcher.utter_message("Sorry! Looks like the city you entered is not correct.")
            # dispatcher.utter_message("Api call for weather complete.")
        return []

class AgriForm(FormAction):
    """Answers agri based questions"""

    def name(self):
        return "agri_form"

    @staticmethod
    def required_slots(tracker):
        return [
            "question",
            ]

    def slot_mappings(self) -> Dict[Text, Union[Dict, List[Dict]]]:
        return{
            "question":self.from_text(),
        }

    def submit(
        self,
        dispatcher: CollectingDispatcher,
        tracker: Tracker,
        domain: Dict[Text, Any],
    ) -> List[Dict]:

        query = tracker.get_slot('question')
        print('query entered',query)
        url = 'http://localhost:8080/getAnswer'

        data = {"question": str(query)}
        j_data = json.dumps(data)
        headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
        res = requests.post(url, data=j_data, headers=headers)
        temp = str(res.json())

        dispatcher.utter_message(text=temp)
        return []

# class WelcomeForm(FormAction):
#     """Answers agri based questions"""

#     def name(self):
#         return "welcome_form"

#     @staticmethod
#     def required_slots(tracker):
#         return [
#             "option",
#             ]

#     def slot_mappings(self) -> Dict[Text, Union[Dict, List[Dict]]]:
#         return{
#             "option":self.from_text(),
#         }

#     def submit(
#         self,
#         dispatcher: CollectingDispatcher,
#         tracker: Tracker,
#         domain: Dict[Text, Any],
#     ) -> List[Dict]:

#         option = tracker.get_slot('option')
#         print(option)
#         return [SlotSet("option",option)]

class ActionGreetUser(Action):
    """Revertible mapped action for utter greet"""
    def name(self):
        return "action_greet"

    def run(self, dispatcher, tracker, domain):
        dispatcher.utter_template("utter_greet", tracker)
        return [UserUtteranceReverted()]

class ActionDefaultFallback(Action):
    def name(self) -> Text:
        return "fallback_action"

    def run(
        self,
        dispatcher: CollectingDispatcher,
        tracker: Tracker,
        domain: Dict[Text, Any],
    ):

        dispatcher.utter_message("Sorry! I did not understand what you said.")
        return [UserUtteranceReverted()]