from __future__ import division
from flask import request, render_template, redirect, url_for, jsonify, Markup, session
from app import app, models, db
import os, glob, json
import distance
from time import time
from random import randint

APP_STATIC = os.path.join(os.path.dirname(os.path.abspath(__file__)),'static')
USER_PATH = os.path.join(APP_STATIC,'resources/users')

@app.route('/')
def init():
    return redirect(url_for('login'))

@app.route('/login')
def login():
    return render_template('login.html')

@app.route('/overview')
def overview():
    u = request.args.get('user')
    counts = read_cluster_counts()
    if not db.session.query(db.exists().where(models.User.name==u)).scalar():
        user = models.User(name=u,counts=([0.0]*87),visited=[],topicWords=[],descriptions=[])
        db.session.add(user)
        db.session.commit()
        write_user_file(u,{})
    user = models.User.query.get(u)
    session['user'] = u
    return render_template('overview.html',user=u,progress=create_cluster_obj(user.counts),description=user.descriptions)

def read_user_file(u):
    if not os.path.isfile(USER_PATH+'/{0}.json'.format(u)):
        write_user_file(u,{})
    f = open(USER_PATH+'/{0}.json'.format(u),'r')
    j = json.load(f)
    f.close()
    return j

def write_user_file(u,j):
    f = open(USER_PATH+'/{0}.json'.format(u),'w')
    json.dump(j,f)
    f.close()
    return 0

def rand_progress(obj):
    obj.sort(key=lambda x: int(x['name'].split('Cluster')[1]))
    c = [randint(0,cluster['size']) for cluster in obj]
    return c

def create_cluster_obj(counts):
    c = {}
    for i in range(0,87):
        if i in counts:
            c['Cluster{0}'.format(i)] = counts[i]
        else:
            c['Cluster{0}'.format(i)] = 0
    return c

@app.route('/home/cluster<int:c>')
def home(c=0):
    if 'user' not in session:
        session['user'] = 'avankipu'
    return render_template('home.html',cluster=c,user=session['user'])

@app.route('/logout')
def logout():
    return redirect('/login')

@app.route('/cluster_counts')
def cluster_counts():
    return jsonify(results=read_cluster_counts())

def read_cluster_counts():
    files = glob.glob(os.path.join(APP_STATIC,'resources/clusters/*.json'))
    J = {'name' : 'java', 'children': [] }
    for file in files:
        f = open(file)
        j = json.load(f)
        J['children'].append({'name': j['name'], 'size': j['size']})
        f.close()
    return J

@app.route('/get_topics')
def get_topics():
    cluster = request.args.get('cluster',0,type=int)
    J = read_topics(cluster)
    return jsonify(results={'topics': J['topicword'], 'words': J['wordtopic'], 'desc': J['desc']})

def read_topics(cluster):
    J = {}
    p = os.path.join(APP_STATIC,'resources/topic_clusters/topic_cluster{0}.json'.format(cluster))
    if os.path.isfile(p):
        with open(p) as f:
            J = json.load(f)
    return J

@app.route('/get_questions_by_topic')
def get_questions_by_topic():
    topics = json.loads(request.args.get('topic'))
    cluster = request.args.get('cluster',0,type=int)
    order = request.args.get('order')
    expr = db.or_(*[models.Question.subcluster==t for t in topics])
    results = {}
    start = time()
    q = models.Question.query.filter_by(cluster=cluster).filter(expr)
    if 'user' in session:
        user = models.User.query.get(session['user'])
    else:
        user = models.User.query.get('avankipu')
    if len(user.visited) > 0:
        vis_expr = db.and_(*[db.not_(models.Question.title.contains(v)) for v in user.visited])
        q = q.filter(vis_expr)
    for e in q.limit(25):
        results[e.title] = {
                'q' : {
                    'content' : e.content,
                    'votes' : e.vote,
                    'reputation' : e.reputation,
                    'user_id' : e.user_id
                } ,
                'a' : [],
            }
        """a = models.Answer.query.filter(models.Answer.title.contains(e.title))
        for ae in a:
            results[e.title]['a'].append({
                'content' : ae.content,
                'votes' : ae.vote,
                'reputation' : ae.reputation,
                'user_id' : ae.user_id
            })"""
    return jsonify(results=results)

@app.route('/visited')
def visited():
    cluster = request.args.get('cluster',0,type=int)
    vis = request.args.get('visited')
    if 'user' not in session:
        session['user'] = 'avankipu'
    user = models.User.query.get(session['user'])
    if vis not in user.visited:
        user.counts[cluster] += 1
    user.visited.append(vis)
    user.visited = list(set(user.visited))
    db.session.commit()
    return jsonify(results={'progress': user.counts[cluster]})

@app.route('/visited_topics')
def visited_topics():
    topics = json.loads(request.args.get('topic'))
    topicWords = json.loads(request.args.get('topicWords'))
    action = request.args.get('action',0,type=int)
    if 'user' not in session:
        session['user'] = 'avankipu'
    user = models.User.query.get(session['user'])
    J = read_user_file(session['user'])
    if action == 1:
        for word in topicWords:
            if word in J:
                J[word] -= 1
            user.topicWords.remove(word)
    else:
        for word in topicWords:
            if word in J:
                J[word] += 1
            else:
                J[word] = 1
        user.topicWords.extend(topicWords)
    write_user_file(session['user'],J)
    user.topicWords = list(set(user.topicWords))
    db.session.commit()
    return jsonify(results={})

def construct_raw_query(cluster,topics,order):
    sql = 'select * from question where question.cluster={0} and ('.format(cluster)
    o = ''
    for t in topics:
        sql += '{0}question.subcluster={1}'.format(o,t)
        o = ' or '
    sql += ') order by question.{0} desc limit 15'.format(order)
    return db.text(sql)

@app.route('/get_questions')
def get_questions():
    topics = json.loads(request.args.get('topics'))
    word = request.args.get('word')
    action = request.args.get('action',0,type=int)
    results = {}
    if len(topics) == 0:
        return jsonify(results=results)
    expr = db.and_(*[models.Question.subcluster==topic for topic in topics])
    cluster = request.args.get('cluster',0,type=int)
    q = models.Question.query.filter_by(cluster=cluster).filter(expr)
    if 'user' in session:
        user = models.User.query.get(session['user'])
    else:
        user = models.User.query.get('avankipu')
    if len(user.visited) > 0:
        vis_expr = db.and_(*[db.not_(models.Question.title.contains(v)) for v in user.visited])
        q = q.filter(vis_expr)

    if action == 1:
        q = q.filter(db.not_(models.Question.text.contains(word)))
    else:
        q = q.filter(models.Question.text.contains(word))

    for e in q.limit(25):
        results[e.title] = {
                'q' : {
                    'content' : e.content,
                    'votes' : e.vote,
                    'reputation' : e.reputation,
                    'user_id' : e.user_id
                } ,
                'a' : [],
            }
    return jsonify(results=results)

@app.route('/all_clusters_user_sim')
def all_clusters_user_similarity():
    if 'user' not in session:
        session['user'] = 'avankipu'
    user = models.User.query.get(session['user'])
    cc = read_cluster_counts()['children']
    cc.sort(key=lambda x: int(x['name'].split('Cluster')[1]))
    counts = []
    desc = {}
    for i in range(0,87):
        t = read_topics(i)
        if t:
            desc['Cluster{0}'.format(i)]= t['desc']
            if cc[i]['size'] > 0:
                prog = user.counts[i]/cc[i]['size']
            else:
                prog = 0.0
            sim = average_sim(user.topicWords,t['wordtopic'].keys())
            fsim = (0.9 * sim) - (prog * 0.1)
            counts.append({
                'c' : i,
                's' :  fsim
            })
        else:
            counts.append({'c' : i, 's' : 0})
    counts.sort(key=lambda x: x['s'],reverse=True)
    _min = min(counts,key=lambda x: x['s'])
    _max = max(counts,key=lambda x: x['s'])
    clusters = {}
    for c in counts:
        clusters['Cluster{0}'.format(c['c'])] = c['s']
    return jsonify(results={'clusters' : clusters, 'min' : _min['s'], 'max' : _max['s'], 'desc': desc})

@app.route('/cluster_user_sim')
def cluster_user_sim():
    cluster = request.args.get('cluster',0,type=int)
    if 'user' not in session:
        session['user'] = 'avankipu'
    user = models.User.query.get(session['user'])
    t = read_topics(cluster)
    topics = t['topicword']
    utopics = user.topicWords
    with open(os.path.join(APP_STATIC,'resources/clusters/cluster{0}.json'.format(cluster))) as f:
        cc = json.load(f)
        count = cc['size']
    results = {}
    l = []
    for topic in topics:
        words = list(topics[topic].keys())
        if len(utopics) > 0:
            l.append(distance.jaccard(utopics,words))
            results[topic] = distance.jaccard(utopics,words)
        else:
            l.append(0.0)
            results[topic] = 0.0
    return jsonify(results={'topics' : results,
        'max': max(l), 'min': min(l),
        'progress': user.counts[cluster],
        'count': count})

@app.route('/prefs')
def preference():
    if 'user' not in session:
        session['user'] = 'avankipu'
    j = read_user_file(session['user'])
    words = [{'name': w, 'count':int(j[w])} for w in j if w != 'java']
    words.sort(key=lambda x: x['count'],reverse=True)
    s = max([j[w] for w in j])
    return jsonify(results={'w':words[:10],'s':s})

@app.route('/update_prefs')
def update():
    values = json.loads(request.args.get('vals'))
    if 'user' not in session:
        session['user'] = 'avankipu'
    j = read_user_file(session['user'])
    for v in values:
        j[v] = values[v]
    write_user_file(session['user'],j)
    return jsonify(results={})

def average_sim(utopics,ctopics):
    if len(utopics) == 0:
        return 0
    J = read_user_file(session['user'])
    s = sum([J[w] for w in J])
    c = []
    for word in utopics:
        c.append(ctopics.count(word) + (J[word]/s))
    #c = [ctopics.count(word) for word in utopics]
    return sum(c)/len(c)

def stdev(l):
    from math import pow, sqrt
    ml = sum(l)/len(l)
    sl = [pow(x-ml,2) for x in l]
    msl = sum(sl)/len(sl)
    return [sqrt(msl),ml]


import RPi.GPIO as GPIO
import time
import numpy as np
import math
import argparse

ap = argparse.ArgumentParser()
ap.add_argument("-d","--degree", type=int, default=10, help="Degree")
ap.add_argument("-p", "--pin", type=int, default=17, help="Pinout")
args = vars(ap.parse_args())

#v = 12
#h = 15
#a = np.arctan(np.sqrt(np.square(v)/(4.9*h)))
#a =  math.degrees(a)
#print(a)

servoPIN = args["pin"]
GPIO.setmode(GPIO.BCM)
GPIO.setup(servoPIN, GPIO.OUT)

p = GPIO.PWM(servoPIN, 100) # GPIO 17 for PWM with 50Hz
p.start(2.5) # Initialization

duty = (args["degree"]) / 10.0 + 2.5
p.ChangeDutyCycle(duty)
time.sleep(1.0)

p.stop()
GPIO.cleanup()


# delete_users

# Implement the following function:
# delete_users  : Takes in a first name and a last name. Updates the  users.csv  file so that any user whose first and last names match the
# inputs are removed. The function should return a count of how many users were removed.

import csv
import pickle
from csv import reader, writer
from pathlib import Path

data_folder = Path(
    r'C:\Users\sixgg\Documents\GitHub\Python\Self-Learning\working CSV Files and pickling\pickling\exercises\delete_users')
filename = "users.csv"
# generates an instance each time it is used
file = data_folder / filename

def delete_users(*args):
    first = args[0]
    last = args[1]

    count = 0
    with open(file, "r") as csv_file:
        temp_csv = list(reader(csv_file, delimiter=','))

        with open(file, "w") as f1:
            cvs_writer = writer(f1, lineterminator="\n")
            for names in temp_csv:
                if first not in names and last not in names:
                    cvs_writer.writerow(names)
                else:
                    count +=1

    return "Users deleted: {}.".format(count)

print(delete_users("Colt", "Steele")) # Users deleted: 2.

Self-Learning/working CSV Files and pickling/pickling/exercises/delete_users/update_users.py


import numpy as np
import argparse
import pysrt
import nltk
import pickle as pkl
import os
from collections import Counter

#from os import listdir, mkdir
#from os.path import isfile, join, exists

parser = argparse.ArgumentParser(description='The Embedded Topic Model')
parser.add_argument('--EnglishWord', type=str, default='', help='Word for which to analyze the distance within class')
parser.add_argument('--configFile', type=str, default='config.txt', help='List of Words for which to analyze the distance within class')
parser.add_argument('--pklPath', type=str, default="./../../Data_cLSP/Data/Keypoints/pkl/Segmented_gestures/")
parser.add_argument('--outputVideoPath', type=str, default='./../Data/Videos/Segmented_gestures/', help='Path where per-line files are located')
#parser.add_argument('--fpsOutput', type=int, default=25, metavar='fpsO',help='Frames per second for the output file')
parser.add_argument('--flgGesture', type=int, default=1, metavar='FLGES',help='Frames per second for the output file')

args = parser.parse_args()

if args.EnglishWord =='':
    with open(args.configFile, encoding='utf-8') as f:
        listWords = f.readlines()
else:
    listWords = [args.EnglishWord]

for word in listWords:

    listLen = []
    listFolders = os.listdir(args.pklPath)
    listPklFiles = []
    for folder in listFolders:
        if not os.path.isdir(args.pklPath+folder):
            continue
        listFiles = os.listdir(args.pklPath+folder)
        #print(listFiles)
        for file in listFiles:
            ## find the underscore
            lenWord = file.find('_')

            if file[:lenWord]==word[:-1]:
                #print(file[:lenWord], word)
                pklFile = open(args.pklPath+folder+'/'+file, 'rb')
                listPklFiles.append(file)
                x = pkl.load(pklFile)
                listLen.append(x.shape[0])
    # First analyze length of instances
    arrSequences = np.array(listLen)
    print(word,arrSequences.mean(), arrSequences.std() )
    #print(listPklFiles)


import tweepy
import json
import requests
import sched
import time

with open('apikeys.json', 'r') as f:
    keys = json.load(f)

# sending keys to authenticator
auth = tweepy.OAuthHandler(
    keys['API key'],
    keys['API key secret'])

auth.set_access_token(
    keys['Access token'],
    keys['Acess token secret'])

api = tweepy.API(auth)

last_status_parsed = int(keys['Last status parsed'])

cursor = tweepy.Cursor(
    api.mentions_timeline,
    tweet_mode="extended",
    since_id=last_status_parsed,
    count=200
)

s = sched.scheduler(time.time, time.sleep)

def scheduled_update(sc):
    global last_status_parsed
    s.enter(60, 1, scheduled_update, (sc,))
    before = time.time()

    print('Checking...')

    for status in cursor.items():
        print(f"Received tweet #{status.id}, forwarding...")

        requests.post(url='http://127.0.0.1:5000/TSC', json=status._json)

        if status.id > last_status_parsed:
            last_status_parsed = status.id
            keys['Last status parsed'] = str(status.id)
            with open('apikeys.json', 'w') as f:
                json.dump(keys, f, indent=2)

s.enter(0, 1, scheduled_update, (s,))
s.run()


# -*- coding: utf-8 -*-
import os

from flask import Flask, jsonify, make_response
from flask_restful import Api as _Api
from flask_jwt_extended import (JWTManager, jwt_required, create_access_token,
    get_jwt_identity, exceptions)

from app.security import authenticate, identity
from app.models import db

from . import config

class Api(_Api):
    # REF: https://github.com/vimalloc/flask-jwt-extended/issues/141
    def error_router(self, original_handler, e):
        return original_handler(e)

def create_app():
    app = Flask(__name__)
    app.config.from_object(config.config.get(os.getenv('FLASK_CONFIG') or 'default'))
    api = Api(app)

    jwt = JWTManager(app) # auth

    @app.errorhandler(exceptions.NoAuthorizationError)
    @app.errorhandler(exceptions.JWTDecodeError)
    def auth_error(error):
        return {'code': 'AUTH_ERROR'}, 401

    from app.resources import user, login, article

    api.add_resource(user.UserResource, '/user')
    api.add_resource(login.LoginResource, '/login')
    api.add_resource(
        article.ArticleResource,
        '/article',
        '/article/<int:article_id>'
    )

    @app.route('/', endpoint='health_check')
    def health_check():
        return 'ok'

    @app.route('/.well-known/pki-validation/2A1E66F75F4AB6CDB31755C18B8FF515.txt')
    def wellknown():
        with open('statics/wellknown.txt') as f:
            return make_response(f.read())

    db.init_app(app)

    @app.before_first_request
    def create_tables():
        db.create_all()

    return app


from common_functions import convert_label_to_value, SentimentScores, maybe_print_count, preprocess_one_line
from pycorenlp import StanfordCoreNLP
from google.cloud import language_v1
import boto3
import pandas as pd

class GCSScorer(object):
    def __init__(self):
        self.client = language_v1.LanguageServiceClient()

    def compute_score(self, rev):
        document = language_v1.Document(content=rev, type_=language_v1.Document.Type.PLAIN_TEXT)
        sentiment = self.client.analyze_sentiment(request={'document': document}).document_sentiment
        score = sentiment.score
        if score > .33:
            return 1
        elif score < -.33:
            return -1
        else:
            return 0

class AWSScorer(object):
    def __init__(self):
        self.comprehend = boto3.client(service_name='comprehend', region_name='us-east-1')

    def compute_score(self, rev):
        try:
            s = self.comprehend.detect_sentiment(Text=rev, LanguageCode='en')
        except Exception as e:
            print(e, rev)
            return 0
        sentiment = s['Sentiment']
        if sentiment == 'POSITIVE':
            return 1
        elif sentiment == 'NEGATIVE':
            return -1
        else:
            return 0

class CoreNLPScorer(object):
    def __init__(self):
        self.cnlp = StanfordCoreNLP('http://localhost:9000')

    def compute_score(self, rev):
        results = self.cnlp.annotate(rev, properties={
            'annotators': 'sentiment',
            'parse.maxlen': 70,
            'outputFormat': 'json',
            'timeout': 50000,
        })
        try:
            s = results["sentences"][0]["sentiment"]
        except Exception as e:
            return 0
        if s == 'Neutral':
            return 0
        elif s == 'Positive':
            return 1
        else:
            return -1

def analyze_sentiment(tool_name):
    if tool_name == 'core_nlp':
        scorer = CoreNLPScorer()
    elif tool_name == 'aws':
        scorer = AWSScorer()
    elif tool_name == 'google':
        scorer = GCSScorer()
    else:
        return
    scores = SentimentScores(str(scorer))
    df = pd.read_csv('/home/data/tweets_no_preprocessing.txt', names=['sentiment', 'review'])

    # Randomly take the first 1000
    df = df.sample(frac=1)  #shuffle the rows
    df = df.head(1000)  # take the first 1000
    cnt = 0
    for index, row in df.iterrows():
        rev = row['review']
        gt_label = row['sentiment']
        gt = convert_label_to_value(gt_label)
        computed_score = scorer.compute_score(rev)
        scores.update(computed_score, gt)
        # maybe_print_count(cnt, 50)
        cnt += 1
    outfile = scores.output_results()
    scores.write_string_to_outfile(tool_name + '\n')
    outfile.close()

analyze_sentiment('aws')
analyze_sentiment('core_nlp')
analyze_sentiment('google')


import db_util
import json
from mysql.connector import MySQLConnection,Error
def coDosingGraph():
    try:
        dbconfig = db_util.read_db_config()
        conn = MySQLConnection(**dbconfig)
        cursor = conn.cursor(buffered=True)
        queryStr = 'SELECT primaryid,prod_ai FROM DRUG'
        queryStr2 = 'SELECT outc_cod FROM OUTC where primaryid = %s'
        caseDict = {}
        deaths = set()
        cursor2 = conn.cursor(prepared=True)
        cursor.execute(queryStr)
        row = cursor.fetchone()
        vertices = set()
        while row is not None:
            #print(row)
            if row[1]!='':
                #print(row)
                v = row[1].replace("\\u001a","-").replace("","-")
                vertices.add(v)
                if row[0] not in caseDict:
                    #print(row[0])
                    caseDict[row[0]] = set()
                    cursor2.execute(queryStr2,(row[0],))
                    #print(row[0])
                    q2_ans = flatten(cursor2.fetchall())
                    #print(q2_ans)
                    if 'DE' in q2_ans:
                        deaths.add(row[0])

                caseDict[row[0]].add(v)
            row = cursor.fetchone()
        #print(row)
        vertexIds = {}
        vertexIdReg = {}
        vertexCaseCt = {}
        vertexDeathCt = {}
        for i,v in enumerate(sorted(vertices)):
            vertexIds[v]=i
            vertexIdReg[i]=v
            vertexCaseCt[i]=0
            vertexDeathCt[i]=0
            #print(v)

        edgeDict = {}
        for key, value in caseDict.items():
            death = 1 if key in deaths else 0
            for x in value:
                xId = vertexIds[x]
                vertexDeathCt[xId]+=death
                vertexCaseCt[xId]+=1
                for y in value:
                    yId = vertexIds[y]
                    if xId<yId:
                        edge = str((xId,yId))
                        if edge not in edgeDict:
                            edgeDict[edge] = 1
                        else:
                            edgeDict[edge] += 1
        with open("./Data/Force/ForceEdges.json","w") as e_fp:
            json.dump(edgeDict, e_fp)
        with open("./Data/Force/ForceVertices.json","w") as v_fp:
            json.dump(vertexIdReg, v_fp)
        adj = edgeListToAdjList(edgeDict)
        with open("./Data/Force/ForceAdjList.json","w") as adj_fp:
            json.dump(adj, adj_fp)
        with open("./Data/Force/CaseList.json","w") as c_fp:
            json.dump(vertexCaseCt, c_fp)
        with open("./Data/Force/DeathList.json","w") as d_fp:
            json.dump(vertexDeathCt, d_fp)

        #for key, value in edgeDict.items():
            #print(key, value)

    except Error as e:
        print(e)

    finally:
        cursor.close()
        cursor2.close()
        conn.close()

def strVertex(v_id,name,cases, deaths,distance):
    #print(v_id,name,distance)
    return '{"id":'+str(v_id)+',"name":"'+name+'","distance":'+str(distance)+',"cases":'+str(cases)+',"deaths":'+str(deaths)+',"group":1}'

def strEdge(edge):
    return '{"source":'+str(edge[0][0])+',"target":'+str(edge[0][1])+',"value":'+str(edge[1][0])+',"BFS EDGE":'+str(edge[1][1])+"}"

def writeFullGraph(fp):
    with open("./Data/Force/ForceEdges.json","r") as e_fp:
        edgeTextDict = json.load(e_fp)
    with open("./Data/Force/ForceVertices.json","r") as v_fp:
        vertexTextDict = json.load(v_fp)
    with open("./Data/Force/CaseList.json","r") as c_fp:
        vertexCaseCt = json.load(c_fp)
    with open("./Data/Force/DeathList.json","r") as d_fp:
        vertexDeathCt = json.load(d_fp)
    vertDict={}
    for key in vertexTextDict:
        vertDict[int(key)] = 0

    if fp == None:
        return
    edgeDict={}
    for key in edgeTextDict:
        x=key.split(", ")
        x0 = int(x[0][1:])
        x1 = int(x[1][:-1])
        edgeDict[(x0,x1)]=(edgeTextDict[key],1)
    genGraphtoFile(fp,vertDict,vertexTextDict,edgeDict,vertexCaseCt,vertexDeathCt)

def genGraphtoFile(fp,vertices,vertexIds,edges,cases,deaths):

    fp.write('{"nodes":[\n')
    v_iter = iter(sorted(vertices.items()))
    v_cur = next(v_iter)
    print(cases[str(v_cur[0])])
    fp.write(strVertex(v_cur[0],vertexIds[str(v_cur[0])],cases[str(v_cur[0])],deaths[str(v_cur[0])],v_cur[1]))
    for k,v in v_iter:
        #print(k,v)
        fp.write(','+strVertex(k,vertexIds[str(k)],cases[str(k)],deaths[str(k)],v))
    fp.write('],\n"links": [\n')
    if len(edges)>0:
        e_iter = iter(sorted(edges.items()))

        fp.write(strEdge(next(e_iter)))
        for val in e_iter:
            if val[1][0]>=1:
                fp.write(',\n'+strEdge(val))
    fp.write(']}')

def edgeListToAdjList(edges):
    ans = {}
    for e in edges:
        x=e.split(", ")
        print(x)
        x0 = int(x[0][1:])
        x1 = int(x[1][:-1])
        print(x0,x1)
        ans.setdefault(x0,[])
        ans.setdefault(x1,[])
        ans[x0].append(x1)
        ans[x1].append(x0)
    return ans

def flatten(l):
    return [item for sublist in l for item in sublist]

def preproc():
    coDosingGraph()
    with open("./Data/Force/3dGraph_full_attr.json", "w") as fp:
        writeFullGraph(fp)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ("cases", "0039_populate_case_watchers"),
    ]

    operations = [
        migrations.AddField(
            model_name="partner",
            name="users",
            field=models.ManyToManyField(
                help_text="Users that belong to this partner", related_name="partners", to=settings.AUTH_USER_MODEL
            ),
        )
    ]


# Importation de la classe SerpAPIWrapper depuis le module langchain.utilities
from langchain.utilities import SerpAPIWrapper

# Définition d'une classe CustomSerpAPIWrapper qui hérite de SerpAPIWrapper
class CustomSerpAPIWrapper(SerpAPIWrapper):
    # Initialisation de la classe
    def __init__(self):
        # Appel du constructeur de la classe parente
        super(CustomSerpAPIWrapper, self).__init__()

    # Définition d'une méthode statique pour traiter la réponse provenant de SerpAPI
    @staticmethod
    def _process_response(res: dict) -> str:
        """Traite la réponse de SerpAPI."""

        # Vérification si la réponse contient une erreur
        if "error" in res.keys():
            raise ValueError(f"Erreur reçue de SerpAPI : {res['error']}")

        # Différentes conditions pour extraire les informations pertinentes de la réponse
        if "answer_box" in res.keys() and "answer" in res["answer_box"].keys():
            toret = res["answer_box"]["answer"]
        elif "answer_box" in res.keys() and "snippet" in res["answer_box"].keys():
            toret = res["answer_box"]["snippet"]
        elif (
            "answer_box" in res.keys()
            and "snippet_highlighted_words" in res["answer_box"].keys()
        ):
            toret = res["answer_box"]["snippet_highlighted_words"][0]
        elif (
            "sports_results" in res.keys()
            and "game_spotlight" in res["sports_results"].keys()
        ):
            toret = res["sports_results"]["game_spotlight"]
        elif (
            "knowledge_graph" in res.keys()
            and "description" in res["knowledge_graph"].keys()
        ):
            toret = res["knowledge_graph"]["description"]
        elif "snippet" in res["organic_results"][0].keys():
            toret = res["organic_results"][0]["link"]
        else:
            # Si aucune condition n'est satisfaite, retourner une chaîne indiquant qu'aucun bon résultat de recherche n'a été trouvé
            toret = "Aucun bon résultat de recherche trouvé"

        return toret

# Définition d'une fonction pour obtenir l'URL du profil LinkedIn ou Twitter
def get_profile_url(name: str):
    """Recherche la page de profil LinkedIn ou Twitter."""

    ### Problème avec la réponse de SerpAPIWrapper() : je n'obtiens pas l'url de Linkedin
    search = SerpAPIWrapper()

    # Création d'une instance de la classe CustomSerpAPIWrapper
    # search = CustomSerpAPIWrapper()

    # Exécution de la recherche
    res = search.run(f"{name}")

    return res


from django.urls import path
from backend import views

app_name = 'backend'

urlpatterns = [
    path('', views.index, name='index'),
    path('add-post-form/', views.post_form, name='post_form'),
    path('add-category/', views.add_category, name='add_category'),
    path('contact-page/', views.contact_form, name='contact_form'),
    path('calculate/', views.calculate, name='calculate'),
    path('login-page/', views.login_view, name='login_view'),
    path('logout-page/', views.logout_view, name='logout_view'),
    path('confirm-logout-page/', views.confirm_logout, name='confirm_logout'),
]


from data_models import DailyWeather
from dataclasses import dataclass

import os
import datetime

def ashton_ice_growth(
            weather_days: list[DailyWeather]
        ):
    """http://lakeice.squarespace.com/ice-growth/"""

    # Units: meter per degree c per day
    GROWTH_RATE = - (
            1 / 15  # 1 inch per degree F per day
            * 0.0254 / 1  # 0.0254 meters per inch
            * 9 / 5 # degrees f per degree C
        )

    def get_fdd(weather_day: DailyWeather):
            return (weather_day.temperature_2m_min + weather_day.temperature_2m_max) / 2 * GROWTH_RATE

    # the average number of degrees below freezing over 24 hours
    initial_ice_thickness = 0

    ice_thickness = initial_ice_thickness + sum([get_fdd(weather_day) for weather_day in weather_days])

    if ice_thickness <= 0:
          return 0.0

    return ice_thickness

def white_fraction_xgb(
        date: datetime.date,
        latitude: float,
        longitude: float
    ) -> float:

    import xgboost as xgb

    model = xgb.Booster()

    model_file_path = os.path.join(
          os.path.dirname(os.path.abspath(__file__)),
          "model_objects",
          "white_fraction_model.json"
    )
    model.load_model(model_file_path)

    year = date.year
    month = date.month
    day_of_year = date.timetuple().tm_yday

    pred = model.predict(
        xgb.DMatrix(
            data=[[year, month, day_of_year, latitude, longitude]]
        )

    )

    return float(pred[0])


import json
from flask import Flask, render_template, request, redirect, url_for, flash
from flaskext.mysql import MySQL
from decouple import config
from flask_mail import *
from random import *
from itsdangerous import URLSafeTimedSerializer
#from flask_wtf.csrf import CSRFProtect
from config import config
from flask_login import LoginManager, login_user, logout_user, login_required

# Modelos:
from models.ModelUser import ModelUser

# Entidades:
from models.entidades.Usuario import UserLogin, UserRegister
app = Flask(__name__)
#Variable global
global codigoid

#csrf = CSRFProtect()
db = MySQL(app)
app.config['MYSQL_DATABASE_USER']='root'
app.config['MYSQL_DATABASE_DB']='seguimientousuarios'
db.init_app(app)#Crear la conexion con los datos anteriores asignados
login_manager_app = LoginManager(app)

#DATOS PARA EL ENVIO CORREO DE CONFIRMACIÓN
with open('src/config.json', 'r') as f:
    params=json.load(f)['param']

app.config['MAIL_SERVER']='smtp.gmail.com'
app.config['MAIL_PORT']=465
app.config['MAIL_USERNAME']=params['gmail-user']
app.config['MAIL_PASSWORD']=params['gmail-password']
app.config['MAIL_USE_TLS']=False
app.config['MAIL_USE_SSL']=True
app.config['SECRET_KEY']="secret"
mail=Mail(app)
#enviar algunos datos a entornos que no son de confianza y luego recuperarlos más tarde.
s=URLSafeTimedSerializer(app.config["SECRET_KEY"])

@login_manager_app.user_loader
def load_user(id):
    return ModelUser.get_by_id(db, id)

@app.route('/')
def index():
    return render_template('auth/login.html')

@app.route('/login', methods=['GET', 'POST'])
def login():
    if request.method == 'POST':
        user = UserLogin(0, '','',request.form['username'], request.form['password'])
        logged_user = ModelUser.login(db, user)
        if logged_user != None:
            #metodo retorna TRUE o FALSE, si la clave real coincide con la clave encriptada
            if logged_user.clave:
                codigoid= logged_user.id;
                return redirect(url_for('home'))
            else:
                flash("Clave invalida...")
                return render_template('auth/login.html')
        else:
            flash("Usuario no encontrado...")
            return render_template('auth/login.html')

    else:
        return render_template('auth/login.html')

@app.route('/registro')
def movimiento():
    return render_template('auth/registro.html')

@app.route('/solicitarDatosBasicos')
def solicitarDatosBasicos():
    return render_template('registroDatosBasicos.html')

@app.route('/registroDatosBasicos', methods=['POST'])
def registroDatosBasicos():
    if request.method == 'POST':
        _sexo=request.form['sexo']
        _edad=request.form['edad']
        _estadocivil=request.form['estadocivil']
        _estrato=request.form['estrato']
        _vivienda=request.form['vivienda']
        _escolaridad=request.form['escolaridad']
        _ocupacion=request.form['ocupacion']
        _sistemaSalud=request.form['sistemaSalud']
        _diagnosticomedico=request.form['diagnosticomedico']
        _discapacidad=request.form['discapacidad']

        if _estrato=='' or _vivienda=='':
            flash("Datos incompletos")
            return render_template('datosBasicosParte2.html')

        else:
            id=1
            #print(logged_user.id)
            cuidador=1
            user = UserRegister(id,_sexo, _edad,_estrato, _vivienda, _escolaridad,_ocupacion,_sistemaSalud,_discapacidad,_diagnosticomedico,cuidador, _estadocivil)
            logged_user = ModelUser.registrardatosbasicos(db, user)
            flash("Datos actualizados")
            return redirect(url_for('home'))

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('login'))

@app.route('/home')
def home():
    return render_template('home.html')

#Envía correo de confirmacion cuando esta creando un nuevo usuario en la app
@app.route("/verificarlinkemail", methods=['GET', 'POST'])
def verificarlinkemail():
    gmail=request.form['email']
    name=request.form['name']
    phone=request.form['phone']
    clave=request.form['clave']
    user = UserLogin(0,name,gmail,phone, clave)
    logged_user = ModelUser.registrarusuario(db, user)
    flash("Usuario registrado", category="info")

    token= s.dumps(gmail, salt='email-confirmation-key')
    msg=Message('Confirmación registro', sender=params['gmail-user'],recipients=[gmail])
    link=url_for('confirmemail', token=token, _external=True)
    msg.body="Hola, \n Gracias por registrarte en nuestra aplicacion. \n Tu codigo de verificacion es: " + link
    mail.send(msg)
    #return "
Verifique su correo para confirmar el registro.
"
    #return render_template('auth/registro.html')
    return redirect(url_for('login'))

@app.route('/confirmemail/<token>')
def confirmemail(token):
    try:
        email=s.loads(token,salt='email-confirmation-key', max_age=60)
    except Exception:
        return "
 Link expiró
"
    return "
 Confirmacion hecha.
"

@app.route('/protected')
@login_required
def protected():
    return "
Esta es una vista protegida, solo para usuarios autenticados.
"

def status_401(error):
    return redirect(url_for('login'))

def status_404(error):
    return "
Página no encontrada
", 404

if __name__ == '__main__':
    app.config.from_object(config['development'])
    #csrf.init_app(app)
    app.register_error_handler(401, status_401)
    app.register_error_handler(404, status_404)
    app.run()


from model import DenseNet
import torchvision
import torch
import time
import copy
import matplotlib.pyplot as plt
import numpy as np
from model.utils import *

device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
print(device)
weight_decay = 0.0005
batch_size = 64
num_epochs = 60
save_path = './record/'

import os

if os.path.exists(save_path) == False:
    os.mkdir(save_path)

def train_model(model, dataloaders, criterion, optimizer, num_epochs):
    since = time.time()
    val_acc_history = []
    train_acc_history = []
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch + 1, num_epochs))
        print('-' * 10)
        if epoch == 40:
            optimizer.param_groups[0]['lr'] /= 10
        if epoch == 53:
            optimizer.param_groups[0]['lr'] /= 10

        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()
            else:
                model.eval()

            running_loss = 0.0
            running_corrects = 0.0

            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                optimizer.zero_grad()

                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    loss = criterion(outputs, labels)

                    _, preds = torch.max(outputs, 1)

                    if (phase == 'train'):
                        loss.backward()
                        optimizer.step()

                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data).item()

            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects / len(dataloaders[phase].dataset)

            info = {'Epoch': [epoch + 1],
                    'Loss': [epoch_loss],
                    'Acc': [epoch_acc],
                    }
            record_info(info, 'record/' + phase + '.csv')

            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
            if phase == 'val':
                val_acc_history.append(epoch_acc)
            else:
                train_acc_history.append(epoch_acc)
        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    model.load_state_dict(best_model_wts)
    return model, val_acc_history, train_acc_history, best_acc

def savehis(val_hist, train_hist, path, name, best_acc, lr=None):
    plt.title("Accuracy vs. Number of Training Epochs")
    plt.xlabel('Training Epochs')
    plt.ylabel('Accuracy')
    plt.plot(range(1, num_epochs + 1), val_hist, label=name + ' val:' + '%.4f' % (best_acc))
    plt.plot(range(1, num_epochs + 1), train_hist, label=name + ' training')
    plt.ylim(0, 1.0)
    plt.xticks(np.arange(0, num_epochs + 1, 20))
    plt.legend()
    plt.savefig(path)

def main():
    transform = torchvision.transforms.Compose([
        torchvision.transforms.Pad(4),
        torchvision.transforms.RandomCrop((32, 32)),
        torchvision.transforms.RandomHorizontalFlip(),
        torchvision.transforms.ToTensor(),
    ])
    train_dataset = torchvision.datasets.CIFAR10('/home/qx/project/data/cifar10-data', train=True, transform=transform)
    test_dataset = torchvision.datasets.CIFAR10('/home/qx/project/data/cifar10-data', train=False,
                                                transform=torchvision.transforms.Compose([
                                                    torchvision.transforms.ToTensor(),
                                                ]))
    dataloaders_dict = {
        'train': torch.utils.data.DataLoader(
            train_dataset, batch_size=batch_size, shuffle=True, num_workers=4
        ),
        'val': torch.utils.data.DataLoader(
            test_dataset, batch_size=batch_size, shuffle=True, num_workers=4
        )
    }

    all_net = {
        #'DenseNet29': DenseNet.densenet29(),
        #'DenseNet45': DenseNet.densenet45(),
        'DenseNet85': DenseNet.densenet85()
    }
    lr = 0.1
    for name, net in all_net.items():
        net = net.to(device)
        optimizer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay)
        criterion = torch.nn.CrossEntropyLoss()
        net, val_hist, train_hist, best_acc = train_model(net, dataloaders_dict, criterion, optimizer, num_epochs)
        # torch.save(net.state_dict(), save_path+name+'.pth')
        val_hist = [h for h in val_hist]
        train_hist = [h for h in train_hist]
        savehis(val_hist, train_hist, save_path + name + '.png', name, best_acc)

if __name__ == '__main__':
    main()


import streamlit as st
from pages.page_1 import func_page_1
from pages.page_2 import func_page_2

def main():

    st.sidebar.subheader('Page selection')
    page_selection = st.sidebar.selectbox('Please select a page',['Main Page','Page 1','Page 2'])

    pages_main = {
        'Main Page': main_page,
        'Page 1': run_page_1,
        'Page 2': run_page_2
    }

    # Run selected page
    pages_main[page_selection]()

def main_page():
    st.title('Main Page')

def run_page_1():
    func_page_1()

def run_page_2():
    func_page_2()

if __name__ == '__main__':
    main()


import math
from typing import Mapping, Any

import requests as rest

import jira_dir

AUTH = ('l.nahar@pipelinesecurity.net', 'zXkdynEDK7X5YJ3IWAlND2A5')
API_URL = 'https://asgardia.atlassian.net' + '/rest/api/latest'
PAGE_SIZE = 50

def get_all(endpoint: str, what: str, params: Mapping[str, Any] = None) -> Mapping[str, Any]:

    if params is None:
        params = {}

    url = API_URL + endpoint + '?jql=assignee in (621dde62a124500068869fe0) AND worklogDate >= startOfMonth("-1") AND worklogDate < startOfMonth() order by created DESC'

    print("url is ", url)

    print("Auth ", AUTH)

    # data = []

    # data = rest.get(url, auth=jira_dir.AUTH, params=params)
    data = rest.get(url, auth=AUTH).json()
    print('response: ðŸ‘‰ï¸�', data)
    # print('response.status_code: ðŸ‘‰ï¸�', data.status_code)
    # print('response.headers: ðŸ‘‰ï¸�', data.headers)

    print('OK GET', endpoint)

    if data['total'] > data['maxResults']:
        for page in range(1, math.ceil(data['total'] / data['maxResults'])):
            temp = rest.get(url, auth=AUTH, params={
                **params,
                'startAt': page * PAGE_SIZE  # specify the offset
            }).json()
            print('OK GET', endpoint)
            # data[what] += temp[what]

    return data


from django.urls import path, include
from . import views
from .views import *

urlpatterns = [

    path('', acquisitionList.as_view(), name='acqList'),
    path('acqadd/<int:id>', views.acquisition_add, name='acqadd'),
    path('acqsave', views.acquisition_save, name='acqsave'),
    # path('acqselected/', views.acquisition_selected, name='acqselected'),
    path('edit/<int:id>/', views.acquisition_edit, name='acqedit'),
    path('delete/<int:id>/', views.acquisition_delete, name='acqdelete'),

    # dropdown
    path('rrnumbers', views.rrnumber_list, name='rrnumbers'),
    path('suppliers', views.supplier_list, name='suppliers'),
    path('selected_rrnumber', views.selected_rrnumber, name='selected_rrnumber'),
    path('selected_supplier', views.selected_supplier, name='selected_supplier'),

    # meter brands
    path('brand', views.brand_ss, name='brandss'),
    path('brandsave', views.brand_save, name='brandsave'),
    path('branddelete', views.brand_delete, name='branddelete'),

    #   mTypes
    path('mtypes', views.mtypes_ss, name='mtypesss'),
    path('mtypessave', views.mtypes_save, name='mtypessave'),
    path('mtypesdelete', views.mtypes_delete, name='mtypesdelete'),

    # meters
    path('meters', views.meter_ss, name='meterss'),
    path('meterssave', views.meter_save, name='meterssave'),
    path('meter_edit', views.meter_edit, name='meter_edit'),
    path('metersdelete', views.meter_delete, name='metersdelete'),

    # seals
    path('acqadds/<int:id>', views.acquisition_adds, name='acqadds'),
    path('seals', views.seal_ss, name='sealss'),
    path('sealssave', views.seal_save, name='sealssave'),
    path('sealsdelete', views.seal_delete, name='sealsdelete'),

    # accept, cancel acquisition

    path('acquisition_response', views.acquisition_response,
         name='acquisition_response'),

    path('get_meter_details', views.get_meter_details, name='get_meter_details'),

]


# -*- coding: utf-8 -*-

import logging

from dipy.io.stateful_tractogram import StatefulTractogram
from dipy.tracking.streamlinespeed import (length, set_number_of_points)
import numpy as np
from scipy.interpolate import splev, splprep
from scipy.ndimage import gaussian_filter1d

def filter_streamlines_by_length(sft, min_length=0., max_length=np.inf):
    """
    Filter streamlines using minimum and max length.

    Parameters
    ----------
    sft: StatefulTractogram
        SFT containing the streamlines to filter.
    min_length: float
        Minimum length of streamlines, in mm.
    max_length: float
        Maximum length of streamlines, in mm.

    Return
    ------
    filtered_sft : StatefulTractogram
        A tractogram without short streamlines.
    """

    # Make sure we are in world space
    orig_space = sft.space
    sft.to_rasmm()

    if sft.streamlines:
        # Compute streamlines lengths
        lengths = length(sft.streamlines)

        # Filter lengths
        filter_stream = np.logical_and(lengths >= min_length,
                                       lengths <= max_length)
    else:
        filter_stream = []

    filtered_sft = sft[filter_stream]

    # Return to original space
    filtered_sft.to_space(orig_space)

    return filtered_sft

def filter_streamlines_by_total_length_per_dim(
        sft, limits_x, limits_y, limits_z, use_abs, save_rejected):
    """
    Filter streamlines using sum of abs length per dimension.

    Note: we consider that x, y, z are the coordinates of the streamlines; we
    do not verify if they are aligned with the brain's orientation.

    Parameters
    ----------
    sft: StatefulTractogram
        SFT containing the streamlines to filter.
    limits_x: [float float]
        The list of [min, max] for the x coordinates.
    limits_y: [float float]
        The list of [min, max] for the y coordinates.
    limits_z: [float float]
        The list of [min, max] for the z coordinates.
    use_abs: bool
        If True, will use the total of distances in absolute value (ex,
        coming back on yourself will contribute to the total distance
        instead of cancelling it).
    save_rejected: bool
        If true, also returns the SFT of rejected streamlines. Else, returns
        None.

    Return
    ------
    filtered_sft : StatefulTractogram
        A tractogram of accepted streamlines.
    ids: list
        The list of good ids.
    rejected_sft: StatefulTractogram or None
        A tractogram of rejected streamlines.
    """
    # Make sure we are in world space
    orig_space = sft.space
    sft.to_rasmm()

    # Compute directions
    all_dirs = [np.diff(s, axis=0) for s in sft.streamlines]
    if use_abs:
        total_per_orientation = np.asarray(
            [np.sum(np.abs(d), axis=0) for d in all_dirs])
    else:
        # We add the abs on the total length, not on each small movement.
        total_per_orientation = np.abs(np.asarray(
            [np.sum(d, axis=0) for d in all_dirs]))

    logging.debug("Total length per orientation is:\n"
                  "Average: x: {:.2f}, y: {:.2f}, z: {:.2f} \n"
                  "Min: x: {:.2f}, y: {:.2f}, z: {:.2f} \n"
                  "Max: x: {:.2f}, y: {:.2f}, z: {:.2f} \n"
                  .format(np.mean(total_per_orientation[:, 0]),
                          np.mean(total_per_orientation[:, 1]),
                          np.mean(total_per_orientation[:, 2]),
                          np.min(total_per_orientation[:, 0]),
                          np.min(total_per_orientation[:, 1]),
                          np.min(total_per_orientation[:, 2]),
                          np.max(total_per_orientation[:, 0]),
                          np.max(total_per_orientation[:, 1]),
                          np.max(total_per_orientation[:, 2])))

    # Find good ids
    mask_good_x = np.logical_and(limits_x[0] < total_per_orientation[:, 0],
                                 total_per_orientation[:, 0] < limits_x[1])
    mask_good_y = np.logical_and(limits_y[0] < total_per_orientation[:, 1],
                                 total_per_orientation[:, 1] < limits_y[1])
    mask_good_z = np.logical_and(limits_z[0] < total_per_orientation[:, 2],
                                 total_per_orientation[:, 2] < limits_z[1])
    mask_good_ids = np.logical_and(mask_good_x, mask_good_y)
    mask_good_ids = np.logical_and(mask_good_ids, mask_good_z)

    filtered_sft = sft[mask_good_ids]

    rejected_sft = None
    if save_rejected:
        rejected_sft = sft[~mask_good_ids]

    # Return to original space
    filtered_sft.to_space(orig_space)

    return filtered_sft, np.nonzero(mask_good_ids), rejected_sft

def resample_streamlines_num_points(sft, num_points):
    """
    Resample streamlines using number of points per streamline

    Parameters
    ----------
    sft: StatefulTractogram
        SFT containing the streamlines to subsample.
    num_points: int
        Number of points per streamline in the output.

    Return
    ------
    resampled_sft: StatefulTractogram
        The resampled streamlines as a sft.
    """

    # Checks
    if num_points <= 1:
        raise ValueError("The value of num_points should be greater than 1!")

    # Resampling
    resampled_streamlines = []
    for streamline in sft.streamlines:
        line = set_number_of_points(streamline, num_points)
        resampled_streamlines.append(line)

    # Creating sft
    # CAREFUL. Data_per_point will be lost.
    resampled_sft = _warn_and_save(resampled_streamlines, sft)

    return resampled_sft

def resample_streamlines_step_size(sft, step_size):
    """
    Resample streamlines using a fixed step size.

    Parameters
    ----------
    sft: StatefulTractogram
        SFT containing the streamlines to subsample.
    step_size: float
        Size of the new steps, in mm.

    Return
    ------
    resampled_sft: StatefulTractogram
        The resampled streamlines as a sft.
    """

    # Checks
    if step_size == 0:
        raise ValueError("Step size can't be 0!")
    elif step_size < 0.1:
        logging.debug("The value of your step size seems suspiciously low. "
                      "Please check.")
    elif step_size > np.max(sft.voxel_sizes):
        logging.debug("The value of your step size seems suspiciously high. "
                      "Please check.")

    # Make sure we are in world space
    orig_space = sft.space
    sft.to_rasmm()

    # Resampling
    lengths = length(sft.streamlines)
    nb_points = np.ceil(lengths / step_size).astype(int)
    if np.any(nb_points == 1):
        logging.warning("Some streamlines are shorter than the provided "
                        "step size...")
        nb_points[nb_points == 1] = 2
    resampled_streamlines = [set_number_of_points(s, n) for s, n in
                             zip(sft.streamlines, nb_points)]

    # Creating sft
    resampled_sft = _warn_and_save(resampled_streamlines, sft)

    # Return to original space
    resampled_sft.to_space(orig_space)

    return resampled_sft

def _warn_and_save(new_streamlines, sft):
    """Last step of the two resample functions:
    Warn that we loose data_per_point, then create resampled SFT."""

    if sft.data_per_point is not None and sft.data_per_point.keys():
        logging.debug("Initial StatefulTractogram contained data_per_point. "
                      "This information will not be carried in the final "
                      "tractogram.")
    new_sft = StatefulTractogram.from_sft(
        new_streamlines, sft, data_per_streamline=sft.data_per_streamline)

    return new_sft

def smooth_line_gaussian(streamline, sigma):
    if sigma < 0.00001:
        ValueError('Cant have a 0 sigma with gaussian.')

    nb_points = int(length(streamline))
    if nb_points < 2:
        logging.debug('Streamline shorter than 1mm, corner cases possible.')
        nb_points = 2
    sampled_streamline = set_number_of_points(streamline, nb_points)

    x, y, z = sampled_streamline.T
    x3 = gaussian_filter1d(x, sigma)
    y3 = gaussian_filter1d(y, sigma)
    z3 = gaussian_filter1d(z, sigma)
    smoothed_streamline = np.asarray([x3, y3, z3], dtype=float).T

    # Ensure first and last point remain the same
    smoothed_streamline[0] = streamline[0]
    smoothed_streamline[-1] = streamline[-1]

    return smoothed_streamline

def smooth_line_spline(streamline, sigma, nb_ctrl_points):
    if sigma < 0.00001:
        ValueError('Cant have a 0 sigma with spline.')

    nb_points = int(length(streamline))
    if nb_points < 2:
        logging.debug('Streamline shorter than 1mm, corner cases possible.')

    if nb_ctrl_points < 3:
        nb_ctrl_points = 3

    sampled_streamline = set_number_of_points(streamline, nb_ctrl_points)

    tck, u = splprep(sampled_streamline.T, s=sigma)
    smoothed_streamline = splev(np.linspace(0, 1, 99), tck)
    smoothed_streamline = np.squeeze(np.asarray([smoothed_streamline]).T)

    # Ensure first and last point remain the same
    smoothed_streamline[0] = streamline[0]
    smoothed_streamline[-1] = streamline[-1]

    return smoothed_streamline

def get_theta(requested_theta, tracking_type):
    if requested_theta is not None:
        theta = requested_theta
    elif tracking_type == 'prob':
        theta = 20
    elif tracking_type == 'eudx':
        theta = 60
    else:
        theta = 45
    return theta

def sample_distribution(dist):
    """
    Parameters
    ----------
    dist: numpy.array
        The empirical distribution to sample from.

    Return
    ------
    ind: int
        The index of the sampled element.
    """
    cdf = dist.cumsum()
    if cdf[-1] == 0:
        return None
    return cdf.searchsorted(np.random.random() * cdf[-1])


import utils
import torch
import numpy as np
from sys import getsizeof
import time

def roll_out_once(q, env,model,cov_matrix):
    '''
    We let the agent interact with the environment for one complete episode

    The return data is
        1. observation_list
        A list of observation of the environment at each time step.

        2. action_list
        A list of actions taken at each time step.

        3. log_prob_action_list
        A list of the log(pi(Action|State)) at each time step.

        4. reward_list
        A list of reward at each time step.
    '''

    observation_list = []
    action_list = []
    log_prob_action_list = []
    reward_list = []

    execute_sell = False
    null_objective = True

    state = env.reset()
    state = np.reshape(state,(-1,))

    for time_index in range(0,200):

        # Calculate action and make a step in the env.
        # Note that rew is short for reward.
        action,log_prob = utils.next_action_during_training(model,cov_matrix,state)

        #record data
        observation_list.append(state)
        action_list.append(action)
        log_prob_action_list.append(log_prob)

        #take one more time step
        state, reward, execute_sell  = env.step(action[0])

        if execute_sell and null_objective:
            null_objective = False

        state = np.reshape(state,(-1,))

    reward = env.final()

    if null_objective:
        for _ in range(0,len(observation_list)):
            reward_list.append(-1e9)
    else:
        for _ in range(0,len(observation_list)):
            reward_list.append(reward)

    # Reshape data as tensors in the shape specified in function description, before returning
    observation_list = torch.tensor(observation_list, dtype=torch.float)
    action_list = torch.tensor(action_list, dtype=torch.float)
    log_prob_action_list = torch.tensor(log_prob_action_list, dtype=torch.float)

    temp_dict = {}
    temp_dict['observation_list'] = observation_list
    temp_dict['action_list'] = action_list
    temp_dict['log_prob_action_list'] = log_prob_action_list
    temp_dict['reward_list'] = reward_list

    q.put([temp_dict])
    q.close()
    time.sleep(2)


import jieba
import math
import jieba.analyse

class TF_IDF(object):
    def __init__(self, file, stop_file):
        self.file = file
        self.stop_file = stop_file
        self.stop_words = self.getStopWords()

    def getStopWords(self):
        """获取停用词表"""
        swlist = list()
        for line in open(self.stop_file, "r", encoding="utf-8").readlines():
            swlist.append(line.strip())
        print("加载停用词完成.....")
        return swlist

    def loadData(self):
        """加载商品和其对应的短标题，使用jieba进行分词并去除停用词"""
        dMap = dict()
        for line in open(self.file, "r", encoding="utf-8").readlines():
            id, title = line.strip().split("\t")
            dMap.setdefault(id, [])
            for word in list(jieba.cut(str(title).replace(" ", ""), cut_all=False)):
                if word not in self.stop_words:
                    dMap[id].append(word)
        print("加载商品和对应的短标题，并使用jieba分词和去除停用词完成....")
        return dMap

    def getFreqWord(self, words):
        """获取一个短标题中的词频"""
        freqWord = dict()
        for word in words:
            freqWord.setdefault(word, 0)
            freqWord[word] += 1
        return freqWord

    def getCountWordInFile(self, word, dMap):
        count = 0
        for key in dMap.keys():
            if word in dMap[key]:
                count += 1
        return count

    def getTFIDF(self, words, dMap):
        """计算TFIDF值，记录单词关键词和对应的tf-idf值"""
        outDic = dict()
        freqWord = self.getFreqWord(words)
        for word in words:
            # 计算TF值，即单个word在整句中出现的次数
            tf = freqWord[word] * 1.0 / len(words)
            idf = math.log(len(dMap) / self.getCountWordInFile(word, dMap) + 1)
            tfidf = tf * idf
            outDic[word] = tfidf
        # 给字典排序
        orderDic = sorted(outDic.items(), key=lambda x: x[1], reverse=True)
        return orderDic

    def getTag(self, words):
        print(jieba.analyse.extract_tags(words, topK=20, withWeight=True))

if __name__ == "__main__":
    # 数据集
    file = "../data/phone-title/id_title.txt"
    stop_file = "../data/phone-title/stop_words.txt"

    tfidf = TF_IDF(file, stop_file)
    tfidf.getTag("小米 红米6Pro 异形全面屏， 后置1200万双摄， 4000mAh超大电池")

    dMap = tfidf.loadData()
    for id in dMap.keys():
        tfIdfDic = tfidf.getTFIDF(dMap[id], dMap)
        print(id, tfIdfDic)


from django.shortcuts import render, redirect, HttpResponse
from . forms import CustomizedOrderForm
from . models import CustomizedOrder
import datetime
from django.contrib import messages
from django.contrib.auth.decorators import login_required

# Create your views here.

@login_required(login_url='login')
def customized_orders(request):
    current_user = request.user

    if request.method == 'POST':
        form = CustomizedOrderForm(request.POST, request.FILES)
        if form.is_valid():
            data = CustomizedOrder()
            data.user = current_user
            data.order_type = form.cleaned_data['order_type']
            data.front_text = form.cleaned_data['front_text']
            data.back_text = form.cleaned_data['back_text']
            data.color = form.cleaned_data['color']
            data.logo = form.cleaned_data['logo']
            data.description = form.cleaned_data['description']
            data.quantity = form.cleaned_data['quantity']
            data.design_image = form.cleaned_data['design_image']

            if data.order_type == "":
                messages.error(request, "Please  enter a Type")
                return redirect('customized_orders')

            if data.quantity < 15:
                messages.error(
                    request, 'We will not accept orders less than 25. Please order 25 items or more!')
                return redirect('customized_orders')
            else:
                data.save()
                messages.success(
                    request, "Successfully Placed the Order. ThanK you for ordering!")

            year = int(datetime.date.today().strftime('%Y'))
            date = int(datetime.date.today().strftime('%d'))
            month = int(datetime.date.today().strftime('%m'))
            d = datetime.date(year, month, date)
            current_date = d.strftime('%Y%m%d')

            customized_order_number = str(data.id) + current_date
            data.customized_order_number = customized_order_number
            data.save()
            return redirect('customized_orders')
        else:
            return redirect('customized_orders')
            # return redirect('home')
    return render(request, 'customized_orders/make_customized_orders.html', {})


# https://leetcode.com/problems/first-unique-character-in-a-string/

# 연습해봄
from collections import Counter
a = "abcdefg"
m = Counter(a)
print("a : ", m.items())
print("index of c : ", a.index("c"))
# for i, j in m.items():
#     if i == 'd':
#         print("True")
#     else:
#         print("False")

# Counter 사용
# 99 ms
from collections import Counter
class Solution:
    def firstUniqChar(self, s: str) -> int:
        m = Counter(s) # m: Counter({'a':2, 'b':2, 'c':1})
        for i,j in m.items(): # first for loop -> i:a, j:2
    	    if j == 1:
                return(s.index(i))

        return -1

wth = Solution()
print(wth.firstUniqChar("aabbc"))

# Hash Table 사용
# 134ms
from collections import Counter
class Solution:
    def firstUniqChar(self, s: str) -> int:
        hash_table = {}

        for i in s:
            if i in hash_table:
                hash_table[i] += 1
            else:
                hash_table[i] = 1

        print("h", hash_table)

        for i in range(len(s)):
            if hash_table[s[i]] == 1:
                return i
        return -1

wth2 = Solution()
print(wth2.firstUniqChar('yhoyo'))

## 단어 나누기
# 1) split()
# 2) Typecasting to list
# https://www.geeksforgeeks.org/python-split-string-into-list-of-characters/
s = 'hello'
l = list(s)
print(l)

Leetcode Data Structure Challenge I/Day6/387. First Unique Character in a String.py


import os
import sys
import zipfile
import tempfile
import subprocess
from typing import List

from cloudminer.logger import logger
from cloudminer.exceptions import CloudMinerException

RESOURCES_DIRECTORY = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))), "resources")
PROJECT_BANNER = """
  /$$$$$$  /$$                           /$$ /$$      /$$ /$$                                        /$$
 /$$__  $$| $$                          | $$| $$$    /$$$|__/                                      /$$$$$$
| $$  \__/| $$  /$$$$$$  /$$   /$$  /$$$$$$$| $$$$  /$$$$ /$$ /$$$$$$$   /$$$$$$   /$$$$$$        /$$__  $$
| $$      | $$ /$$__  $$| $$  | $$ /$$__  $$| $$ $$/$$ $$| $$| $$__  $$ /$$__  $$ /$$__  $$      | $$  \__/
| $$      | $$| $$  \ $$| $$  | $$| $$  | $$| $$  $$$| $$| $$| $$  \ $$| $$$$$$$$| $$  \__/      |  $$$$$$
| $$    $$| $$| $$  | $$| $$  | $$| $$  | $$| $$\  $ | $$| $$| $$  | $$| $$_____/| $$             \____  $$
|  $$$$$$/| $$|  $$$$$$/|  $$$$$$/|  $$$$$$$| $$ \/  | $$| $$| $$  | $$|  $$$$$$$| $$             /$$  \ $$
 \______/ |__/ \______/  \______/  \_______/|__/     |__/|__/|__/  |__/ \_______/|__/            |  $$$$$$/
                                                                                \_  $$_/
                                                                                \__/

\n-- CloudMiner: v1.0.0 (SafeBreach Labs) --\n"""

def get_temp_file_path(file_name: str) -> str:
    """
    Returns the user temp directory
    """
    temp_dir = tempfile.gettempdir()
    return os.path.join(temp_dir, file_name)

def get_file_extension(file_path: str):
    """
    Returns the file extension from file path
    """
    return os.path.basename(os.path.splitext(file_path)[1])

def get_file_name(file_path: str):
    """
    Returns the file name from full file path
    """
    return os.path.basename(os.path.splitext(file_path)[0])

def zip_file(source_file: str, file_name_within_archive: str = None, zip_path: str = None) -> str:
    """
    Zip a file

    :param source_file: File path to zip
    :param file_name_within_archive: Name of file within the zip archive
    :param zip_path: Path to the zipped file. If None, will use the temp directory

    :return: zip file path
    """
    src_file_name = get_file_name(source_file)
    if zip_path is None:
        zip_path = get_temp_file_path(f"{src_file_name}.zip")

    if file_name_within_archive is None:
        file_name_within_archive = src_file_name

    with zipfile.ZipFile(zip_path, "w") as zipf:
        zipf.write(source_file, arcname=file_name_within_archive)

    logger.debug(f"Zip file created at '{zip_path}'")
    return zip_path

def run_command(cmd: List[str], shell: bool = True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) -> subprocess.CompletedProcess:
    """
    Helper function to run commands
    """
    logger.debug(f"Running command: '{subprocess.list2cmdline(cmd)}'")
    return subprocess.run(cmd,
                          shell=shell,
                          text=text,
                          stdout=stdout,
                          stderr=stderr,
                          **kwargs)

def package_to_whl(package_path: str) -> str:
    """
    Create a whl file from a given Python package
        whl file wil lbe saved in the dist directory

    :raises CloudMinerException: If failed to create whl file

    :return: whl file path
    """
    package_name = os.path.basename(package_path)
    setup_file = os.path.join(package_path, "setup.py")
    dist_dir_path = os.path.join(package_path, "dist")

    logger.debug(f"Creating a .whl file for package - '{package_name}'")
    process = run_command(["python", setup_file, "bdist_wheel"], cwd=package_path)
    if process.returncode != 0:
        raise CloudMinerException(f"Failed to create .whl file. Error: {process.stderr}")

    try:
        whl_file_name = os.listdir(dist_dir_path)[0]
    except IndexError:
        raise CloudMinerException(f"Failed to find the creatd .whl file in folder '{dist_dir_path}'.")

    logger.info(f"whl file successfully created - '{whl_file_name}'")
    return os.path.join(dist_dir_path, whl_file_name)


##### Very first
Author  = 'JohnSuperAdler'
Version = '0.1'

##### Module import
import os
import sys
import socket
import argparse
import subprocess
from configparser import ConfigParser
import datetime

##### Arguement parser: Get machine individual setting
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('setting', type=str, default='machine_info.ini', help='Path of setting file of this machine.')
path_machine_info = arg_parser.parse_args().setting

##### Config Parser
machine_info = ConfigParser()
machine_info.read(path_machine_info)
m_name      = machine_info['machine'].get('m_name')
m_memo      = machine_info['machine'].get('m_memo')
s_addr      = machine_info['socket_setting'].get('s_addr')
s_port      = machine_info['socket_setting'].getint('s_port')
s_buffer    = machine_info['socket_setting'].getint('s_buffer')
s_encode    = machine_info['socket_setting'].get('s_encode')
s_SBshell   = machine_info['socket_setting'].get('s_SBshell')
dir_exe     = machine_info['script_directory'].get('dir_exe')
dir_setting = machine_info['script_directory'].get('dir_setting')
dir_output  = machine_info['script_directory'].get('dir_output')

##### loop shell script seed
sh_seed = 'for i in {1..*REP-loop*}; do\n  *REP-cmd*\ndone'

##### Socket start up
print( '[*] Socket server generation starts.')
s = socket.socket()
s.bind(('', s_port))
s.listen(5)
print(f'[*] Socket generated on {s_addr}:{s_port}')

##### Socket action
# For keeping listening, apply while loop.
while True:
    ##### Connection
    c_socket, c_addr = s.accept()
    print(f'[+] New connection received, from {c_addr}')

    ##### Preceding messege from client
    print(f'[+] Preceding messege')
    c_msg_1_b = b''
    while True:
        byte_recv = c_socket.recv(s_buffer)
        c_msg_1_b += byte_recv
        if c_msg_1_b.count(b'\x00') == 8:
            print('    [-] Messege receive complete.')
            break
    c_msg_1_li = list(map(lambda x: x.decode(s_encode), c_msg_1_b.split(b'\x00')))
    print(f'    [-] Messege: {c_msg_1_li}')
    # Giving variable name
    fn_py, fn_cfg, process, loop, copyscript_indi, copyconfig_indi, fsize_py, fsize_cfg, _ = c_msg_1_li
    # Echo to client
    s_echo_1 = f'Preceding messeege received. {c_msg_1_li}'
    c_socket.send(s_echo_1.encode(s_encode))

    ##### Preparation for file transfering
    fsize_py  = int(fsize_py)
    fsize_cfg = int(fsize_cfg)
    dst_path_py  = os.path.join(dir_exe, fn_py)
    dst_path_cfg = os.path.join(dir_setting, fn_cfg)
    if copyscript_indi == '1':
        copyscript = True
    elif copyscript_indi == '0':
        copyscript = False
    else:
        sys.exit('! Copyscript boolean setting failed.')
    if copyconfig_indi == '1':
        copyconfig = True
    elif copyconfig_indi == '0':
        copyconfig = False
    else:
        sys.exit('! Copyconfig boolean setting failed.')

    ##### Script file
    print(f'[+] Receiving script file')
    print(f'    [-] Copy script file: {copyscript}')
    if copyscript:
        c_msg_2_b = b''
        while len(c_msg_2_b) < fsize_py:
            byte_recv = c_socket.recv(s_buffer)
            c_msg_2_b += byte_recv
        print(f'    [-] Script file copy success, size of script = {len(c_msg_2_b)}B')
        with open(dst_path_py, 'wb') as f:
            f.write(c_msg_2_b)
        print(f'    [-] File saved to {dst_path_py}')
        s_echo_2 = f'Script file received. {len(c_msg_2_b)}B'
    else:
        byte_recv = c_socket.recv(s_buffer)
        s_echo_2 = f'Script file transfer skipped.'
    # Echo to client
    c_socket.send(s_echo_2.encode(s_encode))

    ##### Config file
    print(f'[+] Receiving config file')
    print(f'    [-] Copy config file: {copyconfig}')
    if copyconfig:
        c_msg_3_b = b''
        while len(c_msg_3_b) < fsize_cfg:
            byte_recv = c_socket.recv(s_buffer)
            c_msg_3_b += byte_recv
        print(f'    [-] Config file copy success, size of file = {len(c_msg_3_b)}B')
        with open(dst_path_cfg, 'wb') as f:
            f.write(c_msg_3_b)
        print(f'    [-] File saved to {dst_path_cfg}')
        s_echo_3 = f'Config file received. {len(c_msg_3_b)}B'
    else:
        byte_recv = c_socket.recv(s_buffer)
        s_echo_3 = f'Config file transfer skipped.'
    # Echo to client
    c_socket.send(s_echo_3.encode(s_encode))

    ##### Intermission
    print(f'[+] Socket messege receiving and file transfer from {c_addr} done.')

    ##### Script branching on Local machine
    # Flag
    byte_recv = c_socket.recv(s_buffer)
    # Generate loop shell script
    py_cmd = f'python \'{dst_path_py}\' --config \'{dst_path_cfg}\' --machine \'{path_machine_info}\''
    sh_cont = sh_seed.replace('*REP-loop*', f'{loop}').replace('*REP-cmd*', py_cmd)
    timetag = datetime.datetime.now().strftime('%y%m%d_%H%M')
    path_sh = os.path.join(dir_exe, f'ScriptBrancher_{timetag}.sh')
    with open(path_sh, 'w') as f: f.write(sh_cont)
    # Generate script brancher command
    cmd = f'bash \'{s_SBshell}\' {process} "bash \'{path_sh}\'"'
    subprocess.run(cmd, shell=True)
    print(f'[+] [{timetag}] Start running script. Command = {cmd}')
    # Echo to client
    s_echo_4 = f'Start running script. [{timetag}]'
    c_socket.send(s_echo_4.encode(s_encode))

    #####
    print(f'\n[*] Socket keep listening as {s_addr}:{s_port}\n')


#! /usr/bin/python

import sys
sys.path.insert(0,'/usr/local/lib/python2.7/site-packages')
import cv2
from skimage import io

if "__main__" == __name__:
    img = io.imread("data/comic.jpg")
    border = 0
    for line in open("r.txt"):
        parts = [int(o) for o in line.strip().split(" ") if o.isdigit()]
        if len(parts) < 5:
            continue
        #img = cv2.rectangle(img,(parts[1],parts[2]),(parts[3],parts[4]),[255,0,255])
        img = cv2.rectangle(img,(parts[2]-border,parts[1]-border),(parts[4]+border,parts[3]+border),[255,0,255])
    io.imsave("data/get.jpg",img)
    #io.imshow(img)
    #io.show()


"""
Get detail data of github repository using GitHub GraphQL API v4
More information about Github GraphQL API v4 is here https://developer.github.com/v4/
"""
import os

from dotenv import load_dotenv
import requests

# Prepare api keys and endpoint
env_path = os.path.join(os.path.dirname(__file__), ".env")
load_dotenv(env_path)
GITHUB_API_KEY = os.environ.get('GITHUB_API_KEY')
ENDPOINT = 'https://api.github.com/graphql'

def execute_query(api_key, query_string, endpoint='https://api.github.com/graphql'):
    """
    Execute query to GitHub GraphQL API.
    :param api_key: GitHub api key. You can use personal access token.
    :param query_string:  the GitHub GraphQL query consists of a multi-line string.
    :param endpoint: GitHub GraphAL endpoint. default setting.
    :return: json type response
    """
    headers = {'Authorization': f'token {api_key}'}
    response = requests.post(endpoint, json={'query': query_string}, headers=headers)
    if response.status_code == 200:
        return response.json()
    else:
        raise Exception("Query failed to run by returning code of {}. {}".format(response.status_code, query))

def create_query_to_repo(owner_name, repo_name):
    """
    Create query to repository.
    :param owner_name: repository owner name
    :param repo_name: repository name
    :return: query
    """
    query = f"""
    query {{
      repository(owner: {owner_name}, name: {repo_name}){{
        name,
        nameWithOwner,
        description,
        createdAt,
        forkCount,
        homepageUrl
        }}
    }}
    """
    return query

query = create_query_to_repo(owner_name="crimnut02", repo_name="aframe-mmd")

json_data = execute_query(GITHUB_API_KEY, query, ENDPOINT)

print(json_data)


from couchdb import Server
from couchdb.design import ViewDefinition
from config import COUCHDB_URL

_DOCBASES = ['rating']

class Activ(Server):
    "Active database"
    rating = {}

    def __init__(self, url):
        Server.__init__(self, url)
        act = self
        test_and_create = lambda doc: doc in act and act[doc] or act.create(doc)
        for attribute in _DOCBASES:
            setattr(Activ, attribute, test_and_create(attribute))

    def erase_database(self):
        'erase tables'
        for table in _DOCBASES:
            try:
                del self[table]
            except:
                pass

__ACTIV = Activ(COUCHDB_URL)
RATING = __ACTIV.rating

################################################
# CouchDB Permanent Views
################################################
#
# Retorna todas as avaliações de um objeto ou
# Retorna o valor da avaliação que um usuário fez de um objeto
#
# Retorno:
# valor da avaliação e data em que foi feita
#
# Uso: database.RATING.view('rating/partial_data',startkey=[tipo, escopo, objeto], endkey=[tipo, escopo, objeto, {}])
#
rating_partial_data = ViewDefinition('rating','partial_data', \
                               '''function(doc) {
                                     emit([doc.tipo, doc.escopo, doc.objeto, doc.user, doc._id], {rating:doc.rating, data_cri:doc.data_cri});
                                   }
                               ''')

# Retorna a média das avaliações de um objeto
#
# Retorno:
# média das avaliações
#
# Uso: database.RATING.view('rating/mean_by_object',startkey=[tipo, escopo, objeto],endkey=[tipo, escopo, objeto, {}], group="true")

rating_mean_by_object = ViewDefinition('rating','mean_by_object', \
                              u'''
                                    // map
                                    function(doc){
                                        emit([doc.tipo, doc.escopo, doc.objeto], parseInt(doc.rating,10));
                                    }
                              ''',
                              u'''
                                    // reduce
                                    function(keys, values, rereduce) {
                                        if (!rereduce){
                                            var length = values.length
                                            return [sum(values) / length, length]
                                        }else{
                                            var length = sum(values.map(function(v){return v[1]}))
                                            var avg = sum(values.map(function(v){
                                                return v[0] * (v[1] / length)
                                                }))
                                            return [avg, length]
                                        }
                                    }
                              ''')

ViewDefinition.sync_many(RATING, [rating_partial_data, rating_mean_by_object])


import yaml

class Config(object):
    """
    A configuration class that allows to refer parameters as object attributes
    """

    def __init__(self, config):
        """
        config: dict like config parameters
        """

        assert isinstance(config, dict), 'Required dict like object'

        for key, value in config.items():
            if isinstance(value, dict):
                value = Config(value)

            self.__dict__[key] = value

    @staticmethod
    def fromYamlFile(path):
        """
        path: path to json file
        """
        with open(path) as lines:
            return Config(yaml.safe_load(lines))


import sys
import os
import signal
import time
import logging

sys.path.append(os.path.abspath('..'))
import vxi11_server as Vxi11

#
# A simple instrument server.
#
# creates an InstrumentServer with the name INSTR
# adds a device handler with the name inst1
# this instrument simply responds with the current time when queried by
# a vxi11 client.
#
# 'TIME' may not be a legal vxi11 instrument name, but seems to work well.
# allowing some introspection on a device you havent used (and didnt document)
# in some time.
#

def signal_handler(signal, frame):
    logger.info('Handling Ctrl+C!')
    instr_server.close()
    sys.exit(0)

class TimeDevice(Vxi11.InstrumentDevice):

    def device_init(self):
        #print('TimeDevice: device_init()')
        return

    def device_read(self, request_size, term_char, flags, io_timeout):
        '''respond to the device_read rpc: refer to section (B.6.4)
        of the VXI-11 TCP/IP Instrument Protocol Specification'''
        error = Vxi11.Error.NO_ERROR
        reason = Vxi11.ReadRespReason.END

        # opaque_data is a bytes array, so encode correctly!
        data = time.strftime("%H:%M:%S +0000", time.gmtime())
        opaque_data = data.encode("ascii")

        return error, reason, opaque_data

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    logger = logging.getLogger(__name__)

    signal.signal(signal.SIGINT, signal_handler)
    print('Press Ctrl+C to exit')
    logger.info('starting time_device')

    # create a server, attach a device, and start a thread to listen for requests
    instr_server = Vxi11.InstrumentServer()
    #name = 'TIME'
    name = 'inst1'
    instr_server.add_device_handler(TimeDevice, name)
    instr_server.listen()

    # sleep (or do foreground work) while the Instrument threads do their job
    while True:
        time.sleep(1)


import sys
import csv
import xml.etree.ElementTree as ET

from ItemHarvester import ItemHarvester

"""
Library of Congress harvester

    - Drops and recreates all database tables at runtime
    - Fetches collection sitemaps for list of collection pages to be fetch and retries on failure
    - Fetches collection pages [and indefinitely re-tries on failure] to retrieve a list of item
        URLs to harvest. Writes item URLs out as to a CSV file
    - ItemHarvester.py reads item URLs to be harvested from CSV file
    - Fetches item metadata from URLs in CSV file. Indefinitely retries on failure except for 4xx errors
    - Harvested item metadata is stored in an `items` table in the sqlite3 database

"""

class LcCollectionHarvester:
    sitemap_ns = 'http://www.sitemaps.org/schemas/sitemap/0.9'
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) ' \
                 'Chrome/35.0.1916.47 Safari/537.36 '

    def getItemUrls(self, collection_page_data):
        item_pages = []
        for page in collection_page_data:
            for p in self.parse_items_from_response(page):
                item_pages.append([p])
        return item_pages

    def parse_items_from_response(self, rsp):
        items = []
        xml = ET.ElementTree(ET.fromstring(rsp))
        for i in xml.getroot().findall('.//{%s}loc' % self.sitemap_ns):
            if i.text.startswith('http://www.loc.gov/item/'):  # filter out non-items
                items.append(i.text + "?fo=json&at=item")  # append parameters
        return items

    def writeUrlsToCsv(self, file, data):
        fos = open(file, "w")
        with fos:
            csv_writer = csv.writer(fos)
            csv_writer.writerows(data)

    def harvestCollections(self, db, tbl, input_csv, output_csv):
        """

        :param db:
        :param tbl:
        :param input_csv:
        :param output:
        :return:
        """

        item_harvester = ItemHarvester(db)

        # request collection pages via ItemHarvester
        item_harvester.run(tbl, input_csv)

        # get the harvested collection page data
        collection_page_data = item_harvester.getData(tbl)

        # Extract item urls to harvest from collection_page_data
        item_urls = self.getItemUrls(collection_page_data)

        # Write the item urls to a CSV file
        self.writeUrlsToCsv(output_csv, item_urls)

        print('%s item URLs written to %s' % (len(item_urls), output_csv))

        item_harvester.cleanup()

def main(args):
    if len(args) != 5:
        print("Not enough arguments passed. Expecting: input data CSV [page URLs], output CSV [item URLs], "
              "database path, table name")
        sys.exit(-1)

    input_csv = args[1]
    print("Input CSV: %s" % input_csv)

    output_csv = args[2]
    print("Output CSV: %s" % output_csv)

    db = args[3]
    print("DB: %s" % db)

    tbl = args[4]
    print("Table name: %s" % db)

    lc = LcCollectionHarvester()

    lc.harvestCollections(db, tbl, input_csv, output_csv)

if __name__ == '__main__':
    main(sys.argv)


# -*- coding: utf-8 -*-
"""
TEST PLOT EPS
"""

from pathlib import Path
import pickle
import os
import sys
import pandas as pd
from plottingUtils import imagesc
import numpy as np
import matplotlib.pyplot as plt
sys.path.insert(1, 'c:\\Users\\bknorris\\Documents\\Scripts\\Postdoc\\Paper2_OptimizingRestoration\\DataAnalysis\\loadModelFiles\\')
from analysisUtils import find_files

plt.close('all')
# Define file paths:
csv_source = Path('c:/Users/bknorris/Documents/Models/Paper2_OptimizingRestoration/ModelRuns/Scenarios/')
model_source = Path('c:/Users/bknorris/Documents/Models/Paper2_OptimizingRestoration/ModelRuns/Scenarios/postProcessed')
save_fig_dir = Path('c:/Users/bknorris/Documents/Models/Paper2_OptimizingRestoration/Figures')
csv_file = 'modelPostProcessing_imagesc.csv'
version_no = 'V3'
save_figures = False

def extents(f):
    delta = f[1] - f[0]
    return [f[0] - delta / 2, f[-1] + delta / 2]

# Load CSV
csv_file = csv_source / csv_file
model_info = pd.read_csv(csv_file.open())

# Create figure
plt.style.use('_mpl-gallery')
f1 = plt.figure(figsize=(8, 10.5))
axes1 = f1.add_axes([0.1, 0.03, 0.8, 0.15])
axes2 = f1.add_axes([0.1, 0.19, 0.8, 0.15])
axes3 = f1.add_axes([0.1, 0.35, 0.8, 0.15])
axes4 = f1.add_axes([0.1, 0.51, 0.8, 0.15])
axes5 = f1.add_axes([0.1, 0.67, 0.8, 0.15])
axes6 = f1.add_axes([0.1, 0.83, 0.8, 0.15])

ax = [axes1, axes2, axes3,
      axes4, axes5, axes6]
caxis = [1e-7, 5e-5]
c_map = 'viridis'

# Load binary results file
filenames = next(os.walk(str(model_source)))[2]  # For later, [1] is dirs and [2] is files
data = dict()  # Initialize Dictionary
for idx, scenario in enumerate(model_info['orgScenarioNumber']):
    # Check to see if models have been processed already; unzip if no; load if yes
    model_files = find_files(model_source, f'Scenario_{scenario}_*_{version_no}*', 'files')

    file = open(model_source / model_files[0], 'rb')
    print(f'Reading {model_files[0]}')
    avg_fields = pickle.load(file)
    file.close()

    eps = np.stack(avg_fields['eps'])
    xBins = avg_fields['xBins'][0]
    zBins = avg_fields['zBins'][0]

    ax[idx].imshow(eps, aspect='auto', extent=extents(xBins) + extents(zBins),
                   vmin=caxis[0], vmax=caxis[1], origin='lower', cmap=c_map)

[ax[x].xaxis.set_ticklabels([]) for x in range(1,6)]

fname = 'Eps_Tw_30_Hs_0_05.png'
plt.savefig(save_fig_dir / fname, dpi=300, format=None, metadata=None,
            bbox_inches=None, pad_inches=0.1,
            facecolor='auto', edgecolor='auto',
            backend=None)


'''
Reads product_name from .json files, identifies their cat/sub_cat and populates mongodb
Note: Make sure you change vendor name and path before running this file. Default vendor name is : 'HQ-Data'
'''
import os
import re
import sys
import numpy as np
#import logging
import json
import traceback
from multiprocessing import Pool
import csv

from pymongo import MongoClient
client = MongoClient()
db = client.products_db

result_file = "/data/cat_multi/results_"
from flask import Flask, request, Response
from sklearn.externals import joblib
#from config_details import second_level_cat_names
#from logging.handlers import RotatingFileHandler

#from Train_Model.train import ngrams
PARENT_DIR_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
sys.path.append(PARENT_DIR_PATH)

second_level_cat_names=\
            ["Beauty Products and Personal Care",
                  "Camera and Photos",
                  "Mobile Phone, Tablets and Accesories",
                  "Apparel & Accessories",
                  "Watches, Eyewear and Jewellery",
                  "Electronics and Appliances",
                  "Home and Kitchen",
                  "Computers and Laptops",
                  "Shoes and Footwear"
                 ]

second_level_cat_names_set=set(second_level_cat_names)

#logging file path
#LOGGING_PATH='/var/log/cat_subcat_logs/cat_subcat.log'

app = Flask(__name__)
"""
handler = RotatingFileHandler(LOGGING_PATH,maxBytes=10000000,backupCount=5)
formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s")
handler.setFormatter(formatter)
app.logger.addHandler(handler)
app.logger.setLevel(logging.INFO)

app.logger.info("Loading Process Started")
"""
vectorizer=joblib.load(PARENT_DIR_PATH+'/Models/vectorizer.pkl')
clf_bayes=joblib.load(PARENT_DIR_PATH+'/Models/clf_bayes.pkl')
clf_chi=joblib.load(PARENT_DIR_PATH+'/Models/clf_chi.pkl')
clf_fpr=joblib.load(PARENT_DIR_PATH+'/Models/clf_fpr.pkl')

second_level_vectorizer={}
second_level_clf_bayes={}
second_level_clf_fpr={}
for cat_name in second_level_cat_names_set:
    second_level_vectorizer[cat_name]=joblib.load(PARENT_DIR_PATH+'/Models/SubModels/Vectorizer_'+cat_name)
    second_level_clf_bayes[cat_name]=joblib.load(PARENT_DIR_PATH+'/Models/SubModels/clf_bayes_'+cat_name)
    second_level_clf_fpr[cat_name]=joblib.load(PARENT_DIR_PATH+'/Models/SubModels/clf_fpr_'+cat_name)

#app.logger.info("Loading Process Complete")

def predict_category(product_name):
#    app.logger.info("Request received {}".format(str(product_name)))

    try:
        l_product_name = product_name.lower()
        class1 = clf_bayes.predict(vectorizer.transform([l_product_name]))[0]
        class2_prob_vector = clf_chi.predict_proba(vectorizer.transform([l_product_name]))[0]
        class3_prob_vector = clf_fpr.predict_proba(vectorizer.transform([l_product_name]))[0]

        if len(np.unique(class2_prob_vector))==1:
            class2 = "NA"
        else:
            class2 = clf_bayes.classes_[np.argmax(class2_prob_vector)]
        if len(np.unique(class3_prob_vector))==1:
            class3 = "NA"
        else:
            class3 = clf_bayes.classes_[np.argmax(class2_prob_vector)]

        if class3 == "Delhivery_Others":
            if class1 == class2:
                first_level = class1
            elif class1 == "Delhivery_Others":
                first_level = class2
            elif class2 == "Delhivery_Others":
                first_level = class1
            else:
                first_level = class2
        else:
            first_level = class3

        second_level = ""

        if first_level in second_level_cat_names_set:
            prob_vector = second_level_clf_fpr[first_level].predict_proba(
                second_level_vectorizer[first_level].transform([l_product_name]))[0]
            if len(np.unique(prob_vector))==1:
                second_level = second_level_clf_bayes[first_level].predict(
                    second_level_vectorizer[first_level].transform([l_product_name]))[0]
            else:
                second_level = second_level_clf_bayes[first_level].classes_[np.argmax(prob_vector)]

        # prob_vector= second_level_clf[class_name].predict_proba(
            #second_level_vectorizer[class_name].transform([product_name.lower()]))[0]

        #app.logger.info("Result produced {} --> {}".format(
            #str(first_level), str(second_level)))
    except Exception as err:
        print(
            'Traceback: {}'.format(traceback.format_exc()))
        print(
            'Exec Info: {}'.format(sys.exc_info())[0])
        print(
            'Exception {} occurred against product: {}'.format(
                err, product_name))
    return (first_level,second_level)

#@app.route('/get_category', methods = ['POST'])
def get_category(products):
    try:
        list_product_names = products
        #app.logger.info("Request Received {}".format (str(list_product_names)))
	    #print "Working in Process #%d" % (os.getpid())

        for product_name_dict in list_product_names:
            response = {}
            response['category'], response['sub_category'] = predict_category(
                    product_name_dict.get('product_name',"").encode('ascii','ignore'))

            if response['sub_category']:
                try:
                    subcat = response['sub_category'].split('->')[1]
                except IndexError:
                    subcat= response['sub_category'].split('->')[0]
            else:
                subcat = None

            if 'product_url' not in product_name_dict:
                product_name_dict['product_url'] = None
            if 'retail_price' in product_name_dict:
                product_name_dict['retail_price'] = re.findall(r'\d+',product_name_dict['retail_price'])
                product_name_dict['retail_price'] = int(product_name_dict['retail_price'][0])
            else:
                product_name_dict['retail_price'] = None

            db.products_new.insert({"product_name": product_name_dict['product_name'],
                                "vendor": 'HQ-Data',
                                "category": response['category'],
                                "vendor_cat": response['category'],
                                "sub_category": subcat,
                                "vendor_subcat": subcat,
                                "price": product_name_dict['retail_price'],
                                "product_url": product_name_dict['product_url']})

    except Exception as err:
        print(
            'Exception {} occurred against payload: {}'.format(
                err, list_product_names))
        print(str(err))

def create_pool(records, start, end, diff):
    p = Pool(processes=16)
    try:
        while start < end:
            #print records[start:start+diff]
            print(start, diff, end)
            p.apply_async(get_category, args=(records[start:start+diff], ))
            start = start + diff
        p.close()
        p.join()
    except (KeyboardInterrupt, SystemExit):
        print("Exiting...")
        p.terminate()
        p.join()

if __name__ == "__main__":
    path_to_json = os.path.join(os.path.dirname(__file__), 'data/outsource_data/snapdeal/sep/')
    json_files = [pos_json for pos_json in os.listdir(path_to_json) if pos_json.endswith('.json')]
    print(json_files, len(json_files))

    data = []
    records = []
    for js in json_files:
        with open(os.path.join(path_to_json, js)) as json_file:
            for line in json_file:
                data.append(json.loads(line))

    # print(data)
    for i in data:
        for j in i:
            try:
                records.append({'product_name': j['record']['product_name'],
                                'product_url': j['record']['product_url'],
                                'retail_price': j['record']['retail_price']
                                })
            except Exception as e:
                pass
    start = 0
    end = len(records)
    diff = int(end/16)
    create_pool(records, start, end, diff)


def apresentarGraficoDeLinha():
    # importing modules
    import csv
    import matplotlib.pyplot as plt
    # data import csv --> Upload the data in GitHub
    x = []
    y = []

    with open('sample-line.csv', 'r') as csvfile:
        plots = csv.reader(csvfile, delimiter=',')
        for row in plots:
            x.append(int(row[0]))
            y.append(int(row[1]))
    #definindo tamando do grafico
    plt.figure(figsize=(6, 5))
    plt.plot(x, y, label='Loaded from file!')
    #axis X
    plt.xlabel('Game Number')
    #axis y
    plt.ylabel('Game Length')
    #titulo do gráfico
    plt.title('Relação entre Game Number e Game Length')
    plt.legend()
    #salvar em png
    plt.savefig('graph_linha.png')
    plt.show()

#modificar plt.show() para plt.close() para somente salvar o arquivo do gráfico.
    #95%