#!/usr/bin/env python
import essential_module
import pprint
import requests
from bs4 import BeautifulSoup
from lxml import etree
import re
import sqlite3
var = {
    'base': {
        "name": '信广财行',
        "db": r'C:\Users\Administrator\Desktop\spider\db_file\XGCH.db3',
        "headers": '''''',
        'partten': 2  # 1 Synchronization acquisition ;2 extract the uri and save db first and then collect detailed imformathon
    },
    'url': {
        'sourceUri': 'http://www.xgcaihang.com/investinfo?p={}',
        'varS': 1,
        'varE': 1,
        'varStep': 1,
        'requestMethod': 'get',
        'headers?': False,
        'autoExt': True, 'mustContain': '/invest/', 'noContain': '',
        # 'manualExt': True, 'extRegex': '', 'combine': '',
        'test': False
    },
    'detali': {
        'test': {
            'state': False,
            'uri': 'http://www.xgcaihang.com/invest/3262.html'
        },
        'content': {
            'requestMethod': 'get',
            'pageAttribute': 'html'
        },
        'associate': {
            'requestMethod': 'post',
            'posturi': 'http://www.xgcaihang.com/home/investinfo/investlog',
            'rule': 'http://www.xgcaihang.com/invest/(\w+).html',
            'postval': 'borrow_id=variable_a',
            'associated': []
        },
    },
    'tag': {
        "标题": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'xpath', 'selector': "//div[@class='touzwr']/h3",
            'dataHandle': {'htmlTagClear': 'yes'},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "完成度": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': 'body > div.qxWrapper > div.qxBox > div.zwj_xgch.content > div.touzye > div.toyelf > div:nth-of-type(5) > div > div > font',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '100%',
                'noContain': ''
            }
        },
        "借款金额": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "借款期限": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "年利率": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "作者": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "还款方式": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "回复内容": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "发标时间": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 3, 'values': None, 'tag': '回复内容',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "时间": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 3, 'values': None, 'tag': '回复内容',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "类型图片": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "网站编号": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "网址": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "编号": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 3, 'values': None, 'tag': '网址',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "年龄": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "性别": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
    }
}

def mydef(para):
    print("start extract ", para['base']['name'])
    essential_module.initialize_db(para['base']['db'])
    # initizlize databases,create table Content
    essential_module.update_db_field(para['base']['db'], para['tag'])
    # update table Content fields
    headers = essential_module.format_headers(para['base']['headers'])
    # format request headers info.
    for i in range(para['url']['varS'], para['url']['varE'] + 1, para['url']['varStep']):
        url = var['url']['sourceUri'].format(str(i))
        print('start extract list page:', url)
        listUrl = []
        num = 0
        dup = 0
        if para['url']['requestMethod'] == 'get':
            rep = requests.get(url, headers=headers)
            if para['url']['autoExt']:
                soup = BeautifulSoup(rep.text, "lxml")
                for linkEle in soup.find_all('a'):
                    link = str(linkEle.get('href'))
                    if re.match(para['url']['mustContain'], link):
                        if para['url']['noContain']:
                            if not re.match(para['url']['noContain'], link):
                                if re.match('/', link):
                                    g = re.match(
                                        '(^.*com)', para['url']['sourceUri'])
                                    link = g[0] + link
                                    listUrl.append(link)
                                    num += 1
                        else:
                            if re.match('/', link):
                                g = re.match(
                                    '(^.*com)', para['url']['sourceUri'])
                                link = g[0] + link
                                listUrl.append(link)
                                num += 1
        else:
            pass
        if listUrl:
            conn = sqlite3.connect(para['base']['db'])
            db = conn.cursor()
            for i in listUrl:
                s = 'select * from Content where 网址="%s"' % i
                db.execute('select * from Content where 网址="%s"' % i)
                if db.fetchall():
                    dup += 1
                else:
                    if not para['url']['test']:
                        db.execute(
                            'insert into Content (网址) values ("%s")' % i)
            conn.commit()
            db.close()
            conn.close()
        pprint.pprint(listUrl)
        print('extract links num is', num, ',duplicate is ',
              dup, 'new links is', num - dup)
    if para['url']['test']:
        exit()
    if para['base']['partten'] == 2:
        conn = sqlite3.connect(para['base']['db'])
        db = conn.cursor()
        db.execute("select 网址 from Content where 已采 is null")
        l = db.fetchall()
        targetUrl = []
        if l:
            for x in l:
                targetUrl.append(x[0])
            print('need collected data is', len(targetUrl))
            for x in targetUrl:
                if para['detali']['test']['state']:
                    requestUrl = para['detali']['test']['uri']
                else:
                    requestUrl = x
                print('start requests Uri:', requestUrl)
                if para['detali']['content']['requestMethod'] == 'get':
                    page = requests.get(requestUrl, headers=headers)
                    print(page.status_code)
                    if para['detali']['associate']['requestMethod'] == 'post':
                        s = re.search(
                            para['detali']['associate']['rule'], requestUrl)
                        source = re.sub(
                            'variable_a', s[1], para['detali']['associate']['postval'])
                        data = essential_module.format_postval(source)
                        print('start requests associated uri: ',
                              para['detali']['associate']['posturi'], data)
                        associatedPage = requests.post(
                            para['detali']['associate']['posturi'], data=data, headers=headers)
                        print(associatedPage.status_code)
                    else:
                        # get method ----------------------------------------------
                        pass
                    entry = {}
                    if para['detali']['content']['pageAttribute'] == 'html':
                        html = etree.HTML(page.text)
                        soup = BeautifulSoup(page.text, 'lxml')
                    else:
                        # para['detali']['content']['pageAttribute'] == 'json':
                        pass
                    for tag_name, tag_rule in para['tag'].items():
                        if tag_rule['accessMethod'] == 1:
                            if tag_rule['dataSource'] == 1:
                                if tag_rule['extractMethod'] == 'css':
                                    try:
                                        if para['detali']['test']['state']:
                                            if not tag_rule['selector']:
                                                entry[tag_name] = None
                                            else:
                                                if soup.select(tag_rule['selector']):
                                                    temporary_a = ''
                                                    for i in soup.select(tag_rule['selector']):
                                                        temporary_a += i.text
                                                    entry[tag_name] = temporary_a
                                        else:
                                            if soup.select(tag_rule['selector']):
                                                temporary_a = ''
                                                for i in soup.select(tag_rule['selector']):
                                                    temporary_a += i.text
                                                entry[tag_name] = temporary_a
                                            else:
                                                entry[tag_name] = None
                                    except NotImplementedError:
                                        print(
                                            tag_name, ' css selector is error, please change "nth-child" to "nth-of-type"')
                                        exit()
                                else:
                                    # tag_rule['extractMethod'] == 'xpath':
                                    if html.xpath(tag_rule['selector']):
                                        result = ''
                                        for ele in html.xpath(tag_rule['selector']):
                                            result += ele.text
                                        if tag_rule['dataHandle']['htmlTagClear'] == 'yes':
                                            result = re.subn(
                                                r'[\s\r\n]', '', result)[0]
                                        entry[tag_name] = result
                                    else:
                                        entry[tag_name] = None
                            elif tag_rule['dataSource'] == 2:
                                pass
                            else:
                                # tag_rule['dataSource'] == 3:
                                pass
                            if tag_rule['contentFilter']['noNone']:
                                if not entry[tag_name]:
                                    if para['detali']['test']['state']:
                                        entry[tag_name] = 'do not satisfy the necessary conditions--noNone'
                                        print(requestUrl, '\n', tag_name,
                                              ' extract result is None')
                                    else:
                                        print(requestUrl, '\n', tag_name,
                                              ' extract result is None')
                                        next
                            if tag_rule['contentFilter']['mustContent']:
                                f = re.match(
                                    tag_rule['contentFilter']['mustContent'], entry[tag_name])
                                if not f:
                                    print(
                                        requestUrl, '\n', tag_name, 'do not satisfy the necessary conditions--mustContent')
                                    if para['detali']['test']['state']:
                                        entry[tag_name] = None
                                    else:
                                        next
                            if tag_rule['contentFilter']['noContain']:
                                pass
                            # print(entry[tag_name])
                        elif tag_rule['accessMethod'] == 2:
                            pass
                        else:
                            # tag_rule['accessMethod']==3
                            pass
                    # ----------------------------------------------------
                    print(entry)
                else:
                    # post method------------------------------------------------------
                    pass
                if para['detali']['test']['state']:
                    exit()
        else:
            print('need collected data is 0')
        conn.commit()
        db.close()
        conn.close()

mydef(var)


import requests
from PIL import Image
from io import BytesIO
import os
import json
import uuid

counter = 1
for i in range(6,11):
    filename = "file" + str(i) + ".json"
    with open(filename) as json_file:
        data = json.load(json_file)
        for hit in data["hits"]:
            url = hit["previewURL"]
            r = requests.get(url)
            try:
                i = Image.open(BytesIO(r.content))
                u = uuid.uuid4().hex
                i.save("output/" + str(u) + ".jpg")
            except:
                #woop
                continue


from flask import *
from flask_login import *
from mysql_db import MySQL
import mysql.connector
from math import ceil
PERMITED_PARAMS = ['title', 'description', 'year', 'publisher', 'author', 'page_count', 'cover_id']
EDIT_PARAMS = ['last_name', 'first_name', 'middle_name', 'role_id']
PER_PAGE = 3

app = Flask(__name__)
application = app

app.config.from_pyfile('config.py')

db = MySQL(app)

from auth import bp as auth_bp
from auth import init_login_manager, permission_check

from visits import bp as visits_bp

app.register_blueprint(auth_bp)
init_login_manager(app)

app.register_blueprint(visits_bp)

@app.before_request
def loger():
    if request.endpoint == 'static':
        return
    path = request.path
    user_id = getattr(current_user, 'id', None)
    query = 'INSERT INTO visit_logs(user_id, path) VALUES (%s, %s);'
    try:
        with db.connection().cursor(named_tuple=True) as cursor:
            cursor.execute(query, (user_id, path))
            db.connection().commit()
    except mysql.connector.errors.DatabaseError:
        db.connection().rollback()

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/books')
def books():
    page = request.args.get('page', 1, type = int)
    query = ('SELECT books.title, books.description, books.year, books.publisher, books.author, books.page_count, covers.filename FROM books JOIN covers ON books.cover_id = covers.id LIMIT %s OFFSET %s')
    with db.connection().cursor(named_tuple=True) as cursor:
        cursor.execute(query,(PER_PAGE, (page-1)*PER_PAGE))
        books_list = cursor.fetchall()

    query = 'SELECT COUNT(*) AS count FROM (SELECT books.title, books.description, books.year, books.publisher, books.author, books.page_count, covers.filename FROM books JOIN covers ON books.cover_id = covers.id) as result'
    with db.connection().cursor(named_tuple=True) as cursor:
        cursor.execute(query)
        count = cursor.fetchone().count

    last_page = ceil(count/PER_PAGE)

    return render_template('books.html', books_list=books_list, last_page = last_page, current_page = page)

@app.route('/books/new')
@login_required
@permission_check('create')
def users_new():
    roles_list = load_roles()
    return render_template('books_new.html', roles_list=roles_list, book={})

def load_roles():
    query = 'SELECT * FROM roles;'
    cursor = db.connection().cursor(named_tuple=True)
    cursor.execute(query)
    roles = cursor.fetchall()
    cursor.close()
    return roles

def extract_params(params_list):
    params_dict = {}
    for param in params_list:
        params_dict[param] = request.form[param] or None
    return params_dict

@app.route('/books/create', methods=['POST'])
@login_required
@permission_check('create')
def create_user():
    params = extract_params(PERMITED_PARAMS)
    query = 'INSERT INTO books (title, description, year, publisher, author, page_count, cover_id) VALUES (%(title)s, %(description)s, %(year)s, %(publisher)s, %(author)s, %(page_count)s, %(cover_id)s'
    try:
        with db.connection().cursor(named_tuple=True) as cursor:
            cursor.execute(query, params)
            db.connection().commit()
            flash('Успешно!', 'success')
    except mysql.connector.errors.DatabaseError:
        db.connection().rollback()
        flash('При сохранении данных возникла ошибка.', 'danger')
        return render_template('books_new.html', book = params, roles_list = load_roles())

    return redirect(url_for('books'))

# create_user = login_required(create_user)

@app.route('/users/<int:book_id>/update', methods=['POST'])
@login_required
@permission_check('edit')
def update_book(book_id):
    params = extract_params(EDIT_PARAMS)
    params['id'] = book_id
    if current_user.can('change_role'):
        query = ('UPDATE users SET last_name=%(last_name)s, first_name=%(first_name)s, '
                'middle_name=%(middle_name)s, role_id=%(role_id)s WHERE id=%(id)s;')
    else:
        del params['role_id']
        query = ('UPDATE users SET last_name=%(last_name)s, first_name=%(first_name)s, '
                'middle_name=%(middle_name)s WHERE id=%(id)s;')
    try:
        with db.connection().cursor(named_tuple=True) as cursor:
            cursor.execute(query, params)
            db.connection().commit()
            flash('Успешно!', 'success')
    except mysql.connector.errors.DatabaseError:
        db.connection().rollback()
        flash('При сохранении данных возникла ошибка.', 'danger')
        return render_template('users_edit.html', user = params, roles_list = load_roles())

    return redirect(url_for('users'))

@app.route('/users/<int:user_id>/edit')
@login_required
@permission_check('edit')
def edit_user(user_id):
    query = 'SELECT * FROM users WHERE users.id = %s;'
    cursor = db.connection().cursor(named_tuple=True)
    cursor.execute(query, (user_id,))
    user = cursor.fetchone()
    cursor.close()
    return render_template('users_edit.html', user=user, roles_list = load_roles())

@app.route('/users/<int:user_id>/delete', methods=['POST'])
@login_required
@permission_check('delete')
def delete_user(user_id):
    query = 'DELETE FROM users WHERE users.id=%s;'
    try:
        cursor = db.connection().cursor(named_tuple=True)
        cursor.execute(query, (user_id,))
        db.connection().commit()
        cursor.close()
        flash('Пользователь успешно удален', 'success')
    except mysql.connector.errors.DatabaseError:
        db.connection().rollback()
        flash('При удалении пользователя возникла ошибка.', 'danger')
    return redirect(url_for('users'))

@app.route('/user/<int:user_id>')
@login_required
@permission_check('show')
def show_user(user_id):
    query = 'SELECT * FROM users WHERE users.id = %s;'
    cursor = db.connection().cursor(named_tuple=True)
    cursor.execute(query, (user_id,))
    user = cursor.fetchone()
    cursor.close()

    if user is not None:
        return render_template('users_show.html', user=user)
    else:
        flash('Недостаточно прав', 'warning')
        return redirect(url_for('users'))


import pyautogui
from time import sleep
import logging

x=10
y=12
## Logging configuration
logging.basicConfig(format='%(asctime)s %(message)s')

def load_animation(loading_time=1):
    anim_symbols = ['⣾', '⣷', '⣯', '⣟', '⡿', '⢿', '⣻', '⣽']
    i = 0
    load_count_for_second = len()
    for anim_count in range(64 * loading_time):
        i = (i + 1) % len(symbols)
        print('\r\033[K%s loading...' % anim_symbols[i], flush=True, end='')
        sleep(0.1)

def main():
    while True:
        # Get the current poistion
        mouse_cord = pyautogui.position()
        # Move the mouse cursor to the desired position
        pyautogui.moveTo(x, y)
        # Perform a right-click
        pyautogui.click(button="left")
        logging.warning(f"Mouse left Pos x:{x} y:{y}")
        pyautogui.moveTo(mouse_cord)
        logging.warning(f"Mouse left Pos x:{mouse_cord.x} y:{mouse_cord.y}")
        sleep(200)

try:
    if __name__ == "__main__":
        main()
except KeyboardInterrupt:
    my_choice = input('Do you want to exit (Y/N):')
    if my_choice in ('y', 'Y'):
        print("exiting ......")
        exit()
    main()
finally:
    print("exited from loop")


from langchain.chat_models import ChatOpenAI
from langchain.chains import ConversationChain
from langchain.chains.conversation.memory import ConversationBufferWindowMemory
from langchain.prompts import (
    SystemMessagePromptTemplate,
    HumanMessagePromptTemplate,
    ChatPromptTemplate,
    MessagesPlaceholder
)
import streamlit as st
from streamlit_chat import message
from utils import *
import os
import pyttsx3
import speech_recognition as sr
import threading
import time
from gtts import gTTS
import playsound
from streamlit_card import card
import base64
from streamlit_option_menu import option_menu
import requests
from PIL import Image
from io import BytesIO

from streamlit_card import card

import nltk
from nltk.stem.lancaster import LancasterStemmer
import numpy as np
import tflearn
import tensorflow as tf
import random

# restore all of our data structures
import pickle
import requests
from tensorflow.keras.models import load_model

api_url="http://127.0.0.1:8000/fetchanswer/"
stemmer = LancasterStemmer()

data = pickle.load( open( "training_data", "rb" ) )
words = data['words']
classes = data['classes']
train_x = data['train_x']
train_y = data['train_y']

# import our chat-bot intents file
import json
with open('IntentBotData1.json') as json_data:
    intents = json.load(json_data)

# load our saved model
model = load_model("Intent_Model.h5")

def clean_up_sentence(sentence):
    # tokenize the pattern
    sentence_words = nltk.word_tokenize(sentence)
    # stem each word
    sentence_words = [stemmer.stem(word.lower()) for word in sentence_words]
    return sentence_words

# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence
def bow(sentence, words, show_details=False):
    # tokenize the pattern
    sentence_words = clean_up_sentence(sentence)
    # bag of words
    bag = [0]*len(words)
    for s in sentence_words:
        for i,w in enumerate(words):
            if w == s:
                bag[i] = 1
                if show_details:
                    print ("found in bag: %s" % w)

    return(np.array(bag))

ERROR_THRESHOLD = 0.55
def classify(sentence):
    # generate probabilities from the model
    results = model.predict(np.array([bow(sentence, words)]))[0]
    print(results)
    print(len(results))
    # filter out predictions below a threshold
    results = [[i, r] for i, r in enumerate(results) if r > ERROR_THRESHOLD]
    print(results)
    # sort by strength of probability
    results.sort(key=lambda x: x[1], reverse=True)
    print(results)
    return_list = []
    for r in results:
        return_list.append((classes[r[0]], r[1]))
    # return tuple of intent and probability
    print(return_list)
    return return_list

def response(sentence, userID='123', show_details=False):
    results = classify(sentence)
    # if we have a classification then find the matching intent tag
    if results:
        # loop as long as there are matches to process
        while results:
            for i in intents['intents']:
                # find a tag matching the first result
                if i['tag'] == results[0][0]:
                    # a random response from the intent
                    return random.choice(i['responses'])
            results.pop(0)
    else:
        data = None  # Initialize data with None

        try:
            params = {"question": sentence}
            response = requests.get(api_url, params=params)
            print(response.text)
            data = response.json()
        except Exception as e:
            print(e)

        # Now you can check if data is not None before trying to access it
        if data is not None:
            return data['question']
        else:
            return "An error occurred"

PAGE_TITLE: str = "AI Talks"
PAGE_ICON: str = "ðŸ¤–"
LANG_EN: str = "En"

location_map=['washroom']

st.set_page_config(page_title=PAGE_TITLE, page_icon=PAGE_ICON)
st.title('ðŸ¤– Parklink System Chatbot')

selected_lang = option_menu(
    menu_title="",
    options=[LANG_EN],
    icons=["globe2", "translate"],
    menu_icon="cast",
    default_index=0,
    orientation="horizontal",
)
# Store the image URL in session state
if 'image_url' not in st.session_state:
    st.session_state['image_url'] = "url"

if 'responses' not in st.session_state:
    st.session_state['responses'] = ["How can I assist you?"]

if 'requests' not in st.session_state:
    st.session_state['requests'] = []

# Create a pyttsx3 object for text to speech
engine = pyttsx3.init()

# Create a speech recognition object for speech to text
r = sr.Recognizer()

# Define a function to listen to the user's voice and return the text
def listen():
    # Use the default microphone as the audio source
    with sr.Microphone() as source:
        # Listen for the user's voice
        print("Listening...")
        audio = r.listen(source)
    try:
        # Recognize the user's voice using Google Speech Recognition
        text = r.recognize_google(audio)
        # Print the user's input
        print(f"You said: {text}")
        # Return the text
        return text
    except Exception as e:
        # If there is an error, return None
        print(str(e))
        print("Sorry, I could not understand you.")
        return None

# Define a function to speak the chatbot's response
def speak(text):
    try:
        print(f"AI said: {text}")
        tts = gTTS(text=text, lang='en',tld='ca')
        tts.save("good.mp3")
        playsound.playsound('good.mp3')
        os.remove('good.mp3')
    except Exception as e:
        print("Exception occured:",e)

# container for chat history
response_container = st.container()
# container for text box
textcontainer = st.container()

if st.button("Voice-Search ðŸŽ¤"):
# Listen to the user's input and store it in a variable
    query = listen()
    if query:
        with st.spinner("processing..."):
            result=response(query)
            print("Response------>",result)
            # conversation_string = get_conversation_string()
            # # st.code(conversation_string)
            # refined_query = query_refiner(conversation_string, query)
            # st.subheader("Refined Query:")
            # st.write(result)
            # context = find_match(refined_query)
            # print(context)
            # response = conversation.predict(input=f"Context:\n {context} \n\n Query:\n{query}")
            # if "content/images" in response and "jpg" in response:
            #     speak("Here is the map for the desired location:")
            # else:
            response=result
            speak(result)
            # print("hello")

        st.session_state.requests.append(query)
        st.session_state.responses.append(response)

with textcontainer:
    with st.form("Query Form", clear_on_submit=True):
        query = st.text_input("Query: ", key="input")
        submit_button = st.form_submit_button("Submit")

        if submit_button and query:
            with st.spinner("typing..."):
                result = response(query)
                print("Response------>", result)
                response = result
                st.session_state.requests.append(query)
                st.session_state.responses.append(response)

with response_container:
    print(st.session_state['responses'])
    print(response_container)
    if st.session_state['responses']:
        for i in range(len(st.session_state['responses'])):
            if "images/" in st.session_state['responses'][i] and st.session_state['image_url']!="url":
                st.image(st.session_state['responses'][i], caption=None, width=None, use_column_width=None, clamp=False, channels="RGB", output_format="auto")
            message(st.session_state['responses'][i],key=str(i))
            if i < len(st.session_state['requests']):
                message(st.session_state["requests"][i], is_user=True,key=str(i)+ '_user')

        try:
            print(st.session_state['responses'])
            last_element=st.session_state['responses'][-1]
            if 'images/' in last_element and '.jpeg' in last_element and st.session_state['image_url']=="url":
                print("True")
                with st.chat_message("Response:"):
                    contentIndex=st.session_state['responses'][-1].index('images')
                    jpgIndex=st.session_state['responses'][-1].index('.jpeg')
                    url=st.session_state['responses'][-1][contentIndex:jpgIndex+5]
                    print(st.session_state['responses'][-1][contentIndex:jpgIndex+5])
                    print(st.session_state['image_url'])
                    st.session_state['image_url'] = url
                    print(st.session_state['image_url'])
                    st.image(st.session_state['image_url'], caption=None, width=None, use_column_width=None, clamp=False, channels="RGB", output_format="auto")

        except Exception as e:

            print("Exception Occured :", e)


from django.urls import path
from . import views

urlpatterns = [

    path('', views.home, name="dashboard"),
    path('product/', views.ProductDetails, name='product_details'),
    path('create_product/', views.CreateProduct, name='product_create'),
    path('update_product/<str:pk>/', views.UpdateProduct, name='update_product'),
    path('delete_product/<str:pk>/', views.DeleteProduct, name='delete_product'),
    path('tag_details/', views.TagsDetails, name="tag_details"),
    path('create_tag/', views.CreateTags, name='create_tag'),
    path('update_tag/<str:pk>/', views.UpdateTags, name='update_tag'),
    path('delete_tag/<str:pk>/', views.DeleteTags, name='delete_tag'),
    path('customer_details/<str:pk>/', views.CustomerDetails, name='customer_details'),
    path('create_customer/', views.CreateCustomer, name='Create_customer'),
    path('update_customer/<str:pk>/', views.UpdateCustomer, name='update_customer'),
    path('delete_customer/<str:pk>/', views.DeleteCustomer, name='delete_customer'),
]


import pytest

from moona.http import HTTPContext, end
from moona.http.response_headers import (
    auto_content_length,
    content_length,
    content_type,
    content_type_application_json,
    content_type_text_plain,
    header,
)

@pytest.mark.asyncio
@pytest.mark.parametrize(
    "s_name, s_value, b_name, b_value",
    [
        ("Content-Type", "application/json", b"content-type", b"application/json"),
        ("Content-type", "application/json", b"content-type", b"application/json"),
        ("content-type", "application/json", b"content-type", b"application/json"),
    ],
)
async def test_header(ctx: HTTPContext, s_name, s_value, b_name, b_value):
    _ctx = await header(s_name, s_value)(end, ctx)
    assert _ctx.response_headers[b_name] == b_value

@pytest.mark.asyncio
@pytest.mark.parametrize(
    "s_value, b_value",
    [
        ("application/json", b"application/json"),
        ("text/plain", b"text/plain"),
        ("form/multipart", b"form/multipart"),
    ],
)
async def test_content_type(ctx: HTTPContext, s_value, b_value):
    _ctx = await content_type(s_value)(end, ctx)
    assert _ctx.response_headers[b"content-type"] == b_value

@pytest.mark.asyncio
async def test_content_type_application_json(ctx: HTTPContext):
    _ctx = await content_type_application_json(end, ctx)
    assert _ctx.response_headers[b"content-type"] == b"application/json"

@pytest.mark.asyncio
async def test_content_type_text_plain(ctx: HTTPContext):
    _ctx = await content_type_text_plain(end, ctx)
    assert _ctx.response_headers[b"content-type"] == b"text/plain"

@pytest.mark.asyncio
@pytest.mark.parametrize(
    "s_value, b_value",
    [
        (0, b"0"),
        (12, b"12"),
        (2355, b"2355"),
    ],
)
async def test_content_length(ctx: HTTPContext, s_value, b_value):
    _ctx = await content_length(s_value)(end, ctx)
    assert _ctx.response_headers[b"content-length"] == b_value

@pytest.mark.asyncio
@pytest.mark.parametrize(
    "body, length",
    [
        (None, b"0"),
        (b"", b"0"),
        (b"body", b"4"),
    ],
)
async def test_auto_content_length(ctx: HTTPContext, body, length):
    ctx.response_body = body
    _ctx = await auto_content_length(end, ctx)
    assert _ctx.response_headers[b"content-length"] == length


#8
from imageio import imread, imwrite
import sys
import numpy as np
import math

def draw(a,b):
    global im, h, w
    if a == b:
        return
    if abs(b[1] - a[1]) > abs(b[0] - a[0]): #x > y
        if a[1] > b[1]:
            a,b = b,a
        k = (b[0] - a[0])/(b[1] - a[1])
        for i in range(b[1] - a[1]):
            x = a[1] + i
            y = round(k*i + a[0])
            if x < 0 or x >= w or y < 0 or y >= h:
                continue
            im[y][x] = (0,0,0)
    else:
        if a[0] > b[0]:
            a,b = b,a
        k = (b[1] - a[1])/(b[0] - a[0])
        for i in range(b[0] - a[0]):
            y = a[0] + i
            x = round(k*i + a[1])
            if x < 0 or x >= w or y < 0 or y >= h:
                continue
            im[y][x] = (0,0,0)

if len(sys.argv) > 2:
    im = imread(sys.argv[1])
else:
    print("err")
    exit()

h = len(im)
w = len(im[0])
print('width = ', w, 'height = ', h)

L = []
for i in range(w):
    for j in range(h):
        if list(im[j][i]) == [0,0,0]:
            L.append((j,i))
center = (h//2,w//2)

for p1 in range(len(L)):
    for p2 in range(len(L)):
        if p1 != p2 and (L[p1][1]-center[1]) == -(L[p2][1]-center[1]) and (L[p1][0]-center[0]) == -(L[p2][0]-center[0]):
            draw(L[p1], L[p2])
im = imwrite(sys.argv[2], im)


"""
Build: parse and add user-supplied files to store
"""
import json
import os
from shutil import copyfile, move, rmtree
from stat import S_IRUSR, S_IRGRP, S_IROTH, S_IWUSR
import uuid

from enum import Enum
import numpy as np
import pandas as pd

from .const import (DEFAULT_TEAM, PACKAGE_DIR_NAME, QuiltException, SYSTEM_METADATA,
                    TargetType)
from .core import (GroupNode, RootNode, FileNode, decode_node, encode_node,
                   find_object_hashes, hash_contents)
from .hashing import digest_file
from .util import BASE_DIR, sub_dirs, sub_files, is_nodename

CHUNK_SIZE = 4096

# Helper function to return the default package store path
def default_store_location():
    package_dir = os.path.join(BASE_DIR, PACKAGE_DIR_NAME)
    return os.getenv('QUILT_PRIMARY_PACKAGE_DIR', package_dir)

class ParquetLib(Enum):
    SPARK = 'pyspark'
    ARROW = 'pyarrow'

class PackageException(QuiltException):
    """
    Exception class for Package handling
    """
    pass

class StoreException(QuiltException):
    """
    Exception class for store I/O
    """
    pass

class PackageStore(object):
    """
    Base class for managing Quilt data package repositories. This
    class and its subclasses abstract file formats, file naming and
    reading and writing to/from data files.
    """
    BUILD_DIR = 'build'
    OBJ_DIR = 'objs'
    TMP_OBJ_DIR = 'tmp'
    PKG_DIR = 'pkgs'
    CACHE_DIR = 'cache'
    VERSION = '1.3'
    CONTENTS_DIR = 'contents'
    TAGS_DIR = 'tags'
    VERSIONS_DIR = 'versions'
    LATEST = 'latest'

    __parquet_lib = None

    @classmethod
    def get_parquet_lib(cls):
        """
        Find/choose a library to read and write Parquet files
        based on installed options.
        """
        if cls.__parquet_lib is None:
            parq_env = os.environ.get('QUILT_PARQUET_LIBRARY', ParquetLib.ARROW.value)
            cls.__parquet_lib = ParquetLib(parq_env)
        return cls.__parquet_lib

    @classmethod
    def reset_parquet_lib(cls):
        cls.__parquet_lib = None

    @classmethod
    def set_parquet_lib(cls, parqlib):
        cls.__parquet_lib = ParquetLib(parqlib)

    def __init__(self, location=None):
        if location is None:
            location = default_store_location()

        assert os.path.basename(os.path.abspath(location)) == PACKAGE_DIR_NAME, \
            "Unexpected package directory: %s" % location
        self._path = location

        version = self._read_format_version()

        if version == '1.2':
            # Migrate to the teams format.
            pkgdir = os.path.join(self._path, self.PKG_DIR)
            old_dirs = sub_dirs(pkgdir)
            os.mkdir(os.path.join(pkgdir, DEFAULT_TEAM))
            for old_dir in old_dirs:
                os.rename(os.path.join(pkgdir, old_dir), os.path.join(pkgdir, DEFAULT_TEAM, old_dir))
            self._write_format_version()
        elif version not in (None, self.VERSION):
            msg = (
                "The package repository at {0} is not compatible"
                " with this version of quilt. Revert to an"
                " earlier version of quilt or remove the existing"
                " package repository."
            )
            raise StoreException(msg.format(self._path))

    def __eq__(self, rhs):
        return isinstance(rhs, PackageStore) and self._path == rhs._path # pylint:disable=W0212

    def __ne__(self, rhs):
        return not (self == rhs)

    def __hash__(self):
        return hash(self._path)

    def create_dirs(self):
        """
        Creates the store directory and its subdirectories.
        """
        if not os.path.isdir(self._path):
            os.makedirs(self._path)
        for dir_name in [self.OBJ_DIR, self.TMP_OBJ_DIR, self.PKG_DIR, self.CACHE_DIR]:
            path = os.path.join(self._path, dir_name)
            if not os.path.isdir(path):
                os.mkdir(path)
        if not os.path.exists(self._version_path()):
            self._write_format_version()

    @classmethod
    def find_store_dirs(cls):
        """
        Returns the primary package directory and any additional ones from QUILT_PACKAGE_DIRS.
        """
        store_dirs = [default_store_location()]
        extra_dirs_str = os.getenv('QUILT_PACKAGE_DIRS')
        if extra_dirs_str:
            store_dirs.extend(extra_dirs_str.split(':'))
        return store_dirs

    @classmethod
    def find_package(cls, team, user, package, pkghash=None, store_dir=None):
        """
        Finds an existing package in one of the package directories.
        """
        cls.check_name(team, user, package)
        dirs = cls.find_store_dirs()
        for store_dir in dirs:
            store = PackageStore(store_dir)
            pkg = store.get_package(team, user, package, pkghash=pkghash)
            if pkg is not None:
                return store, pkg
        return None, None

    @classmethod
    def check_name(cls, team, user, package, subpath=None):
        if team is not None and not is_nodename(team):
            raise StoreException("Invalid team name: %r" % team)
        if not is_nodename(user):
            raise StoreException("Invalid user name: %r" % user)
        if not is_nodename(package):
            raise StoreException("Invalid package name: %r" % package)
        if subpath:
            for element in subpath:
                if not is_nodename(element):
                    raise StoreException("Invalid element in subpath: %r" % element)

    def _version_path(self):
        return os.path.join(self._path, '.format')

    def _read_format_version(self):
        if os.path.exists(self._version_path()):
            with open(self._version_path(), 'r') as versionfile:
                version = versionfile.read()
                return version
        else:
            return None

    def _write_format_version(self):
        with open(self._version_path(), 'w') as versionfile:
            versionfile.write(self.VERSION)

    # TODO: find a package instance other than 'latest', e.g. by
    # looking-up by hash, tag or version in the local store.
    def get_package(self, team, user, package, pkghash=None):
        """
        Gets a package from this store.
        """
        self.check_name(team, user, package)
        path = self.package_path(team, user, package)
        if not os.path.isdir(path):
            return None

        if pkghash is None:
            latest_tag = os.path.join(path, self.TAGS_DIR, self.LATEST)
            if not os.path.exists(latest_tag):
                return None

            with open (latest_tag, 'r') as tagfile:
                pkghash = tagfile.read()

        assert pkghash is not None
        contents_path = os.path.join(path, self.CONTENTS_DIR, pkghash)
        if not os.path.isfile(contents_path):
            return None

        with open(contents_path, 'r') as contents_file:
            return json.load(contents_file, object_hook=decode_node)

    def install_package(self, team, user, package, contents):
        """
        Creates a new package in the default package store
        and allocates a per-user directory if needed.
        """
        self.check_name(team, user, package)
        assert contents is not None

        self.create_dirs()
        path = self.package_path(team, user, package)

        # Delete any existing data.
        try:
            os.remove(path)
        except OSError:
            pass

    def create_package_node(self, team, user, package, dry_run=False):
        """
        Creates a new package and initializes its contents. See `install_package`.
        """
        contents = RootNode(dict())
        if dry_run:
            return contents

        self.check_name(team, user, package)
        assert contents is not None
        self.create_dirs()

        # Delete any existing data.
        path = self.package_path(team, user, package)
        try:
            os.remove(path)
        except OSError:
            pass

        return contents

    def remove_package(self, team, user, package):
        """
        Removes a package (all instances) from this store.
        """
        self.check_name(team, user, package)

        path = self.package_path(team, user, package)
        remove_objs = set()
        # TODO: do we really want to delete invisible dirs?
        if os.path.isdir(path):
            # Collect objects from all instances for potential cleanup
            contents_path = os.path.join(path, PackageStore.CONTENTS_DIR)
            for instance in os.listdir(contents_path):
                pkg = self.get_package(team, user, package, pkghash=instance)
                remove_objs.update(find_object_hashes(pkg))
            # Remove package manifests
            rmtree(path)

        return self.prune(remove_objs)

    def iterpackages(self):
        """
        Return an iterator over all the packages in the PackageStore.
        """
        pkgdir = os.path.join(self._path, self.PKG_DIR)
        if not os.path.isdir(pkgdir):
            return
        for team in sub_dirs(pkgdir):
            for user in sub_dirs(self.team_path(team)):
                for pkg in sub_dirs(self.user_path(team, user)):
                    pkgpath = self.package_path(team, user, pkg)
                    for hsh in sub_files(os.path.join(pkgpath, PackageStore.CONTENTS_DIR)):
                        yield self.get_package(team, user, pkg, pkghash=hsh)

    def ls_packages(self):
        """
        List packages in this store.
        """
        packages = []
        pkgdir = os.path.join(self._path, self.PKG_DIR)
        if not os.path.isdir(pkgdir):
            return []
        for team in sub_dirs(pkgdir):
            for user in sub_dirs(self.team_path(team)):
                for pkg in sub_dirs(self.user_path(team, user)):
                    pkgpath = self.package_path(team, user, pkg)
                    pkgmap = {h : [] for h in sub_files(os.path.join(pkgpath, PackageStore.CONTENTS_DIR))}
                    for tag in sub_files(os.path.join(pkgpath, PackageStore.TAGS_DIR)):
                        with open(os.path.join(pkgpath, PackageStore.TAGS_DIR, tag), 'r') as tagfile:
                            pkghash = tagfile.read()
                            pkgmap[pkghash].append(tag)
                    for pkghash, tags in pkgmap.items():
                        # add teams here if any other than DEFAULT_TEAM should be hidden.
                        team_token = '' if team in (DEFAULT_TEAM,) else team + ':'
                        fullpkg = "{team}{owner}/{pkg}".format(team=team_token, owner=user, pkg=pkg)
                        # Add an empty string tag for untagged hashes
                        displaytags = tags if tags else [""]
                        # Display a separate full line per tag like Docker
                        for tag in displaytags:
                            packages.append((fullpkg, str(tag), pkghash))

        return packages

    def team_path(self, team=None):
        """
        Returns the path to directory with the team's users' package repositories.
        """
        if team is None:
            team = DEFAULT_TEAM
        return os.path.join(self._path, self.PKG_DIR, team)

    def user_path(self, team, user):
        """
        Returns the path to directory with the user's package repositories.
        """
        return os.path.join(self.team_path(team), user)

    def package_path(self, team, user, package):
        """
        Returns the path to a package repository.
        """
        return os.path.join(self.user_path(team, user), package)

    def object_path(self, objhash):
        """
        Returns the path to an object file based on its hash.
        """
        return os.path.join(self._path, self.OBJ_DIR, objhash)

    def temporary_object_path(self, name):
        """
        Returns the path to a temporary object, before we know its hash.
        """
        return os.path.join(self._path, self.TMP_OBJ_DIR, name)

    def cache_path(self, name):
        """
        Returns the path to a temporary object, before we know its hash.
        """
        return os.path.join(self._path, self.CACHE_DIR, name)

    def prune(self, objs=None):
        """
        Clean up objects not referenced by any packages. Try to prune all
        objects by default.
        """
        if objs is None:
            objdir = os.path.join(self._path, self.OBJ_DIR)
            objs = os.listdir(objdir)
        remove_objs = set(objs)

        for pkg in self.iterpackages():
            remove_objs.difference_update(find_object_hashes(pkg))

        for obj in remove_objs:
            path = self.object_path(obj)
            if os.path.exists(path):
                os.chmod(path, S_IWUSR)
                os.remove(path)
        return remove_objs

    def _read_parquet_arrow(self, hash_list):
        from pyarrow.parquet import ParquetDataset

        objfiles = [self.object_path(h) for h in hash_list]
        dataset = ParquetDataset(objfiles)
        table = dataset.read(nthreads=4)
        try:
            dataframe = table.to_pandas()
        except Exception:
            # Try again to convert the table after removing
            # the possibly buggy Pandas-specific metadata.
            meta = table.schema.metadata.copy()
            meta.pop(b'pandas')
            newtable = table.replace_schema_metadata(meta)
            dataframe = newtable.to_pandas()

        return dataframe

    def _read_parquet_spark(self, hash_list):
        from pyspark import sql as sparksql

        spark = sparksql.SparkSession.builder.getOrCreate()
        objfiles = [self.object_path(h) for h in hash_list]
        dataframe = spark.read.parquet(*objfiles)
        return dataframe

    def _check_hashes(self, hash_list):
        for objhash in hash_list:
            path = self.object_path(objhash)
            if not os.path.exists(path):
                raise StoreException("Missing object fragments; re-install the package")

    def load_dataframe(self, hash_list):
        """
        Creates a DataFrame from a set of objects (identified by hashes).
        """
        self._check_hashes(hash_list)
        parqlib = self.get_parquet_lib()
        if parqlib is ParquetLib.SPARK:
            return self._read_parquet_spark(hash_list)
        elif parqlib is ParquetLib.ARROW:
            try:
                return self._read_parquet_arrow(hash_list)
            except ValueError as err:
                raise StoreException(str(err))
        else:
            assert False, "Unimplemented Parquet Library %s" % parqlib

    def save_dataframe(self, dataframe):
        """
        Save a DataFrame to the store.
        """
        storepath = self.temporary_object_path(str(uuid.uuid4()))

        # switch parquet lib
        parqlib = self.get_parquet_lib()
        if isinstance(dataframe, pd.DataFrame):
            #parqlib is ParquetLib.ARROW: # other parquet libs are deprecated, remove?
            import pyarrow as pa
            from pyarrow import parquet
            table = pa.Table.from_pandas(dataframe)
            parquet.write_table(table, storepath)
        elif parqlib is ParquetLib.SPARK:
            from pyspark import sql as sparksql
            assert isinstance(dataframe, sparksql.DataFrame)
            dataframe.write.parquet(storepath)
        else:
            assert False, "Unimplemented ParquetLib %s" % parqlib

        # Move serialized DataFrame to object store
        if os.path.isdir(storepath): # Pyspark
            hashes = []
            files = [ofile for ofile in os.listdir(storepath) if ofile.endswith(".parquet")]
            for obj in files:
                path = os.path.join(storepath, obj)
                objhash = digest_file(path)
                self._move_to_store(path, objhash)
                hashes.append(objhash)
            rmtree(storepath)
        else:
            filehash = digest_file(storepath)
            self._move_to_store(storepath, filehash)
            hashes = [filehash]

        return hashes

    def load_numpy(self, hash_list):
        """
        Loads a numpy array.
        """
        assert len(hash_list) == 1
        self._check_hashes(hash_list)
        with open(self.object_path(hash_list[0]), 'rb') as fd:
            return np.load(fd, allow_pickle=False)

    def save_numpy(self, nparray):
        storepath = self.temporary_object_path(str(uuid.uuid4()))
        with open(storepath, 'wb') as fd:
            np.save(fd, nparray, allow_pickle=False)

        filehash = digest_file(storepath)
        self._move_to_store(storepath, filehash)

        return filehash

    def get_file(self, hash_list):
        """
        Returns the path of the file - but verifies that the hash is actually present.
        """
        assert len(hash_list) == 1
        self._check_hashes(hash_list)
        return self.object_path(hash_list[0])

    def save_file(self, srcfile):
        """
        Save a (raw) file to the store.
        """
        filehash = digest_file(srcfile)
        if not os.path.exists(self.object_path(filehash)):
            # Copy the file to a temporary location first, then move, to make sure we don't end up with
            # truncated contents if the build gets interrupted.
            tmppath = self.temporary_object_path(filehash)
            copyfile(srcfile, tmppath)
            self._move_to_store(tmppath, filehash)

        return filehash

    def load_metadata(self, metahash):
        self._check_hashes([metahash])
        path = self.object_path(metahash)
        with open(path) as fd:
            return json.load(fd)

    def save_metadata(self, metadata):
        """
        Save metadata to the store.
        """
        if metadata in (None, {}):
            return None

        if SYSTEM_METADATA in metadata:
            raise StoreException("Not allowed to store %r in metadata" % SYSTEM_METADATA)

        path = self.temporary_object_path(str(uuid.uuid4()))

        with open(path, 'w') as fd:
            try:
                # IMPORTANT: JSON format affects the hash of the package.
                # In particular, it cannot contain line breaks because of Windows (LF vs CRLF).
                # To be safe, we use the most compact encoding.
                json.dump(metadata, fd, sort_keys=True, separators=(',', ':'))
            except (TypeError, ValueError):
                raise StoreException("Metadata is not serializable")

        metahash = digest_file(path)
        self._move_to_store(path, metahash)
        return metahash

    def save_package_contents(self, root, team, owner, pkgname):
        """
        Saves the in-memory contents to a file in the local
        package repository.
        """
        assert isinstance(root, RootNode)
        instance_hash = hash_contents(root)
        pkg_path = self.package_path(team, owner, pkgname)
        if not os.path.isdir(pkg_path):
            os.makedirs(pkg_path)
            os.mkdir(os.path.join(pkg_path, self.CONTENTS_DIR))
            os.mkdir(os.path.join(pkg_path, self.TAGS_DIR))
            os.mkdir(os.path.join(pkg_path, self.VERSIONS_DIR))

        dest = os.path.join(pkg_path, self.CONTENTS_DIR, instance_hash)
        with open(dest, 'w') as contents_file:
            json.dump(root, contents_file, default=encode_node, indent=2, sort_keys=True)

        tag_dir = os.path.join(pkg_path, self.TAGS_DIR)
        if not os.path.isdir(tag_dir):
            os.mkdir(tag_dir)

        latest_tag = os.path.join(pkg_path, self.TAGS_DIR, self.LATEST)
        with open (latest_tag, 'w') as tagfile:
            tagfile.write("{hsh}".format(hsh=instance_hash))

    def _move_to_store(self, srcpath, objhash):
        """
        Make the object read-only and move it to the store.
        """
        destpath = self.object_path(objhash)
        if os.path.exists(destpath):
            # Windows: delete any existing object at the destination.
            os.chmod(destpath, S_IWUSR)
            os.remove(destpath)
        os.chmod(srcpath, S_IRUSR | S_IRGRP | S_IROTH)  # Make read-only
        move(srcpath, destpath)

    def _add_to_package_contents(self, pkgroot, node_path, hashes, target,
                                 source_path, transform, user_meta_hash):
        """
        Adds an object (name-hash mapping) or group to package contents.
        """
        assert isinstance(node_path, list)
        assert user_meta_hash is None or isinstance(user_meta_hash, str)

        contents = pkgroot

        if not node_path:
            # Allow setting metadata on the root node, but that's it.
            assert target is TargetType.GROUP
            contents.metadata_hash = user_meta_hash
            return

        ptr = contents
        for node in node_path[:-1]:
            ptr = ptr.children[node]

        if target is TargetType.GROUP:
            node = GroupNode(
                children=dict(),
                metadata_hash=user_meta_hash
            )
        else:
            node = FileNode( # pylint:disable=R0204
                hashes=hashes,
                metadata=dict(
                    q_ext=transform,
                    q_path=source_path,
                    q_target=target.value
                ),
                metadata_hash=user_meta_hash
            )

        ptr.children[node_path[-1]] = node

########################################
# Methods ported from save_<xyz>
# in Package class
########################################

    def add_to_package_df(self, root, dataframe, node_path, target, source_path, transform, custom_meta):
        hashes = self.save_dataframe(dataframe)
        metahash = self.save_metadata(custom_meta)
        self._add_to_package_contents(root, node_path, hashes, target, source_path, transform, metahash)
        return hashes

    def add_to_package_cached_df(self, root, hashes, node_path, target, source_path, transform, custom_meta):
        metahash = self.save_metadata(custom_meta)
        self._add_to_package_contents(root, node_path, hashes, target, source_path, transform, metahash)

    def add_to_package_numpy(self, root, ndarray, node_path, target, source_path, transform, custom_meta):
        """
        Save a Numpy array to the store.
        """
        filehash = self.save_numpy(ndarray)
        metahash = self.save_metadata(custom_meta)
        self._add_to_package_contents(root, node_path, [filehash], target, source_path, transform, metahash)

    def add_to_package_group(self, root, node_path, custom_meta):
        metahash = self.save_metadata(custom_meta)
        self._add_to_package_contents(root, node_path, None, TargetType.GROUP, None, None, metahash)

    def add_to_package_file(self, root, srcfile, node_path, target, source_path, transform, custom_meta):
        """
        Save a (raw) file to the store.
        """
        filehash = self.save_file(srcfile)
        metahash = self.save_metadata(custom_meta)
        self._add_to_package_contents(root, node_path, [filehash], target, source_path, transform, metahash)

    def add_to_package_package_tree(self, root, node_path, pkgnode):
        """
        Adds a package or sub-package tree from an existing package to this package's
        contents.
        """
        if node_path:
            ptr = root
            for node in node_path[:-1]:
                ptr = ptr.children.setdefault(node, GroupNode(dict()))
            ptr.children[node_path[-1]] = pkgnode
        else:
            if root.children:
                raise PackageException("Attempting to overwrite root node of a non-empty package.")
            root.children = pkgnode.children.copy()


from haversine import haversine
from airports import airport, yourCordinates
from localeparser import *
from colorama import Fore, Back, Style

yc = yourCordinates()

# should be EWR or LGA
def fastlyDistance():
    fastlyLocation = fastly(requests.get("https://tile.openstreetmap.org").headers["X-served-by"], True)

    # lat, long
    air = airport(fastlyLocation[0])

    return {"provider": "Fastly \t \t \t \t", "airport": fastlyLocation[0], "distance": haversine(air[0], air[1], yc[0], yc[1])}

def cloudflareDistance():
    cloudflareLocation = cloudflare(requests.get("https://www.cloudflare.com").headers["Cf-ray"])

    air = airport(cloudflareLocation[0])

    return {"provider": "Cloudflare \t \t \t", "airport": cloudflareLocation[0], "distance": haversine(air[0], air[1], yc[0], yc[1])}

def cloudflareFreeDistance():
    cloudflareLocation = cloudflare(requests.get("https://check-host.net").headers["Cf-ray"])

    air = airport(cloudflareLocation[0])

    return {"provider": "Cloudflare (free) \t \t", "airport": cloudflareLocation[0], "distance": haversine(air[0], air[1], yc[0], yc[1])}

def cloudfrontDistance():
    cloudfrontLocation = cloudfront(requests.get("https://desmos.com").headers["x-amz-cf-pop"])
    air = airport(cloudfrontLocation[0])

    return {"provider": "CloudFront \t \t \t", "airport": cloudfrontLocation[0], "distance": haversine(air[0], air[1], yc[0], yc[1])}

def cacheFlyDistance():
    cacheFlyLocation = cacheFly(requests.get("https://www.cachefly.com").headers["x-cf1"])
    air = airport(cacheFlyLocation[0])

    return {"provider": "CacheFly \t \t \t", "airport": cacheFlyLocation[0], "distance": haversine(air[0], air[1], yc[0], yc[1])}

def fancyPrintResults():
    finalArray = []
    finalArray.append(fastlyDistance())
    finalArray.append(cloudflareDistance())
    finalArray.append(cloudflareFreeDistance())
    finalArray.append(cloudfrontDistance())
    finalArray.append(cacheFlyDistance())

    for x in finalArray:
        if x["distance"] >= 0 and x["distance"] < 50:
            print(Fore.CYAN + x["provider"] + " " + str(round(x["distance"])) + "km from you (via " + x["airport"] + ")")
            print(Style.RESET_ALL)
        elif x["distance"] >= 50 and x["distance"] < 100:
            print(Fore.BLUE + x["provider"] + " " + str(round(x["distance"])) + "km from you (via " + x["airport"] + ")")
            print(Style.RESET_ALL)
        elif x["distance"] >= 100 and x["distance"] < 300:
            print(Fore.GREEN + x["provider"] + " " + str(round(x["distance"])) + "km from you (via " + x["airport"] + ")")
            print(Style.RESET_ALL)
        else:
            print(Fore.RED + x["provider"] + " " + str(round(x["distance"])) + "km from you (via " + x["airport"] + ")")
            print(Style.RESET_ALL)


#!/usr/bin/env python
# coding: utf-8

# In[54]:

get_ipython().system('pip install nltk')
get_ipython().system('pip install gensim')

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import nltk
nltk.download('stopwords')
nltk.download('punkt')
from nltk.corpus import stopwords
from nltk.util import ngrams
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from collections import defaultdict
from collections import Counter
plt.style.use('ggplot')
stop_words = set(stopwords.words('english'))
import re
from nltk.tokenize import word_tokenize
import gensim
import string
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import Tokenizer
from tqdm import tqdm
from keras.models import Sequential
from keras.preprocessing.text import Tokenizer
from keras.layers import Embedding, LSTM, Dense, SpatialDropout1D
from keras.initializers import Constant
from keras.optimizers import Adam
from keras.layers import Bidirectional
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense, SpatialDropout1D
from tensorflow.keras.optimizers import Adam

# In[29]:

get_ipython().system('wget https://github.com/stanfordnlp/GloVe/blob/master/glove.twitter.27B.25d.txt')
embedding_dict = {}
with open("glove.twitter.27B.25d.txt", 'r', encoding='utf-8') as file:
    for line in file:
        values = line.split()
        word = values[0]
        vector = np.asarray(values[1:], dtype='float32')
        embedding_dict[word] = vector

# In[32]:

train_data = pd.read_csv('train.csv')
test_data = pd.read_csv('test.csv')

train_data.head(3)
print('There are {} rows and {} columns in the training data'.format(train_data.shape[0], train_data.shape[1]))
print('There are {} rows and {} columns in the test data'.format(test_data.shape[0], test_data.shape[1]))

# In[33]:

x = train_data.target.value_counts()
sns.barplot(x=x.index, y=x)
plt.gca().set_ylabel('samples')

# In[34]:

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))
tweet_len = train_data[train_data['target'] == 1]['text'].str.split().map(lambda x: len(x))
ax1.hist(tweet_len, color='red')
ax1.set_title('Disaster tweets')
tweet_len = train_data[train_data['target'] == 0]['text'].str.split().map(lambda x: len(x))
ax2.hist(tweet_len, color='blue')
ax2.set_title('Non-disaster tweets')
fig.suptitle('Word count in tweets')
plt.show()

# In[35]:

plt.style.use('ggplot')
stop = set(stopwords.words('english'))

# In[39]:

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))
tweet_len = train_data[train_data['target'] == 1]['text'].str.split().map(lambda x: len(x))
sns.histplot(tweet_len, ax=ax1, color='red')
ax1.set_title('disaster tweets')
tweet_len = train_data[train_data['target'] == 0]['text'].str.split().map(lambda x: len(x))
sns.histplot(tweet_len, ax=ax2, color='blue')
ax2.set_title('Not disaster tweets')
fig.suptitle('Words in a tweet')
plt.show()

# In[41]:

def create_corpus(target_data, target):
    corpus = []
    for x in target_data[target_data['target'] == target]['text'].str.split():
        for i in x:
            corpus.append(i)
    return corpus

corpus = create_corpus(train_data, target=1)

counter = Counter(corpus)
most = counter.most_common()
x, y = [], []
for word, count in most[:40]:
    if word not in stop:
        x.append(word)
        y.append(count)
sns.barplot(x=y, y=x)
plt.title('Most common words in disaster tweets')
plt.show()

# In[42]:

def get_top_tweet_bigrams(corpus, n=None):
    vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
    bag_of_words = vec.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]
    words_freq = sorted(words_freq, key=lambda x: x[1], reverse=True)
    return words_freq[:n]

# In[43]:

df = pd.concat([train_data, test_data])
df.shape

# In[44]:

def remove_url(text):
    url_pattern = re.compile(r'https?://\S+|www\.\S+')
    return url_pattern.sub(r'', text)

# In[45]:

example = "Click on this link: https://www.example.com"
print(remove_url(example))
df['text'] = df['text'].apply(lambda x: remove_url(x))

# In[46]:

def remove_html_tags(text):
    html_pattern = re.compile(r'<.*?>')
    return html_pattern.sub(r'', text)

# In[47]:

example = """

Real or Fake

Kaggle

getting started

"""

print(remove_html_tags(example))
df['text'] = df['text'].apply(lambda x: remove_html_tags(x))

# In[48]:

def remove_emojis(text):
    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  # emoticons
                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               u"\U00002702-\U000027B0"
                               u"\U000024C2-\U0001F251"
                               "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

# In[49]:

example = remove_emojis("second Earthquake ðŸ˜”ðŸ˜”")
print(remove_emojis(example))
df['text'] = df['text'].apply(lambda x: remove_emojis(x))

# In[50]:

def remove_punctuation(text):
    table = str.maketrans('', '', string.punctuation)
    return text.translate(table)

# In[51]:

example = "I @am the #best"
print(remove_punctuation(example))
df['text'] = df['text'].apply(lambda x: remove_punctuation(x))

# In[52]:

def remove_stopwords(text):
    words = [word.lower() for word in word_tokenize(text) if word.lower() not in stop_words]
    return " ".join(words)

# In[55]:

example = "I am a very good boy, but I don't like to study"
print(remove_stopwords(example))
df['text'] = df['text'].apply(lambda x: remove_stopwords(x))

# In[56]:

corpus = df['text'].apply(lambda x: word_tokenize(x))
num_words = len(corpus)
embedding_dict = gensim.models.Word2Vec(corpus, vector_size=100, window=5, min_count=2)
max_seq_len = 100
embed_num_dims = 25

# In[57]:

X = train_data['text']
y = train_data['target']

tokenizer = Tokenizer(num_words=10000)
tokenizer.fit_on_texts(X)
X = tokenizer.texts_to_sequences(X)
X = pad_sequences(X, maxlen=max_seq_len)

# In[58]:

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=42)
lstm_out = 128
batch_size = 32
num_epochs = 1

# In[59]:

model = Sequential()
model.add(Embedding(10000, embed_num_dims, input_length=max_seq_len))
model.add(Bidirectional(LSTM(lstm_out, dropout=0.2, recurrent_dropout=0.2)))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit(X_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_data=(X_test, y_test))

# In[60]:

embed_num_dims = 100
max_seq_len = X.shape[1]
word_index = tokenizer.word_index
num_words = min(10000, len(word_index) + 1)

# In[61]:

def create_embedding_matrix(word_index, embedding_dict, embed_num_dims):
    embedding_matrix = np.zeros((len(word_index) + 1, embed_num_dims))
    for word, index in word_index.items():
        if word in embedding_dict.wv.key_to_index:
            embedding_matrix[index] = embedding_dict.wv[word]

        return embedding_matrix

# In[62]:

embedding_matrix = create_embedding_matrix(word_index, embedding_dict, embed_num_dims)

# In[63]:

model = Sequential()
model.add(Embedding(len(word_index) + 1,
                    embed_num_dims,
                    embeddings_initializer=Constant(embedding_matrix),
                    input_length=max_seq_len,
                    trainable=False))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(64, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))

optimizer = Adam(learning_rate=1e-5)

model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])

# In[64]:

y = train_data['target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15)

history = model.fit(X_train, y_train, batch_size=4, epochs=5, validation_data=(X_test, y_test))

# In[76]:

epochs = [i for i in range(5)]

plt.figure(figsize=(12, 6))
plt.plot(epochs, history.history['accuracy'], 'go-', label='Training Accuracy')
plt.plot(epochs, history.history['val_accuracy'], 'ro-', label='Validation Accuracy')
plt.title('Training & Validation Accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()

# In[66]:

loss, accuracy = model.evaluate(X_test, y_test)

# In[68]:

sample_sub=pd.read_csv('train.csv')
y_pre=model.predict(X_test)
y_pre=np.round(y_pre).astype(int).reshape(1, -1)[0]
sub = pd.concat([sample_sub['id'], pd.Series(y_pre, name='target')], axis=1)
sub.to_csv('submission.csv', index=False)
sub.head()

# In[70]:

dataset = pd.read_csv('train.csv')
embedding_dim=25
embedding_dict = {}
with open('glove.twitter.27B.25d.txt', 'r', encoding='utf-8') as f:
    for line in f:
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embedding_dict[word] = coefs

# In[71]:

word_index = tokenizer.word_index
num_words = min(10000, len(word_index) + 1)
embedding_matrix = np.zeros((num_words, embedding_dim))
for word, i in word_index.items():
    if i >= num_words:
        continue
    embedding_vector = embedding_dict.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector

# In[72]:

model = Sequential()
model.add(Embedding(num_words, embedding_dim, input_length=max_seq_len, embeddings_initializer=Constant(embedding_matrix), trainable=False))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(64, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))

optimizer = Adam(learning_rate=1e-5)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])

epochs = 10
batch_size = 32
model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(X_test, y_test))

# In[73]:

new_data = ['your text data goes here', 'more text data']
new_sequences = tokenizer.texts_to_sequences(new_data)
new_padded_sequences = pad_sequences(new_sequences, maxlen=max_seq_len)
predictions = model.predict(new_padded_sequences)

# In[74]:

y_pre=model.predict(X_test)
y_pre=np.round(y_pre).astype(int).reshape(1, -1)[0]
sub = pd.concat([sample_sub['id'], pd.Series(y_pre, name='target')], axis=1)
sub.to_csv('submission.csv', index=False)
sub.head()

# In[75]:

loss, accuracy = model.evaluate(X_test, y_test)

# In[ ]:


import uvicorn
from fastapi import FastAPI

from core.config import WEB_PORT, WEB_HOST
from db.base import database
from endpoints import author, books

app = FastAPI(title="MY_APP")
app.include_router(author.router, prefix="/author", tags=["author"])
app.include_router(books.router, prefix="/books", tags=["books"])

@app.on_event("startup")
async def startup():
    await database.connect()

@app.on_event("shutdown")
async def shutdown():
    await database.disconnect()

@app.get("/")
async def read_root():
    return {"Hello": "World"}

if __name__ == "__main__":
    uvicorn.run("main:app", port=int(WEB_PORT), host=WEB_HOST, reload=True)


import utils.displayStandards as dsp
from . import scattering_factors as scatf
import numpy as np
from math import pi

__all__=['structure_factor3D','plot_structure3D','get_miller3D',
        'get_pendulossung']

#2D for factors
ai = 1/np.array([0.1,0.25,0.26,0.27,1.5])**2
fj = lambda q,j,eps:eps*(np.pi/ai[j])*np.exp(-(np.pi*q)**2/ai[j])

def structure_factor3D(pattern,lat_vec,hkl=None,hklMax=10,sym=1,v=''):
    '''Computes structure factor in 3D from :
    - `pattern` : Nx4 array - N atoms with each row : fractional coordinates and Za
    - `lat_vec` : 3x3 array - reciprocal lattice vectors (2*pi/a convention)
    - `hkl`     : list of 3 miller indices h,k,l each as 3Ndarray
    - `hklMax`  : int - max miller index in each direction from -hklMax to hklMax
    '''
    #unpack
    if not hkl : hkl = get_miller3D(hklMax,sym)
    hx,ky,lz = hkl
    # print('rearranging h,k,l so h[(hi,k,l)]=hi')
    hx,ky,lz = np.transpose(hx,[1,0,2]),np.transpose(ky,[1,0,2]),np.transpose(lz,[1,0,2])
    hkl = [hx,ky,lz]
    ra,fa = pattern[:,:3],pattern[:,3]
    atoms = list(np.array(np.unique(fa),dtype=int));#print(atoms)
    # get scattering factor
    b1,b2,b3 = lat_vec
    k_x,k_y,k_z = hx*b1[0]+ky*b2[0]+lz*b3[0],hx*b1[1]+ky*b2[1]+lz*b3[1],hx*b1[2]+ky*b2[2]+lz*b3[2]
    q = np.sqrt(k_x**2+k_y**2+k_z**2)/(2*pi)
    q,fq = scatf.get_elec_atomic_factors(atoms,q)
    if 'q' in v:qmax=q.max();print('qmax=%.4f A^-1\nmax_res=%.4f A' %(qmax,1/qmax))
    #structure factor
    Fhkl,n_atoms = np.zeros(hx.shape,dtype=complex),len(atoms)
    for i,atom in zip(range(n_atoms),atoms):
        F_i = np.zeros(hx.shape,dtype=complex)
        idx = fa==atom
        for ri in ra[idx,:]:
            # print(ri)
            # print(ri,np.exp(-2*pi*1J*(ri[0]*hx+ri[1]*ky+ri[2]*lz)))
            F_i += np.exp(-2*pi*1J*(ri[0]*hx+ri[1]*ky+ri[2]*lz))
        Fhkl += F_i*fq[i]

    # cs=dsp.getCs('Spectral',n_atoms)
    # plts=[[q.flatten(),fq[i].flatten(),[cs[i],'+'],atom] for i,atom in enumerate(atoms)]
    # dsp.stddisp(plts,lw=2)
    return hkl,Fhkl

def structure_factor2D(pattern,lat_vec,hk=None,hkMax=10,sym=1,v=0,eps=1):
    '''get structure factor
    - pattern : atomic positions in real space (fractional coordinates)
    - lat_vec : reciprocal lattice vectors (2pi/a convention)
    '''
    #unpack
    if not hk : hl = get_miller2D(hkMax,sym)
    hx,ky = hl
    hx,ky = hx.T,ky.T
    hl = [hx,ky]
    ra,fa = pattern[:,:2],pattern[:,2]
    atoms = list(np.array(np.unique(fa),dtype=int));#print(atoms)
    #scattering factor
    b1,b2 = lat_vec
    k_x,k_y = hx*b1[0]+ky*b2[0],hx*b1[1]+ky*b2[1]
    q = np.sqrt(k_x**2+k_y**2)/(2*pi)
    fq = [fj(q,Za,eps) for Za in atoms]
    if v : qmax=q.max();print('qmax=%.4f A^-1\nmax_res=%.4f A' %(qmax,1/qmax))
    #compute structure factor
    Fhl,n_atoms = np.zeros(hx.shape,dtype=complex),len(atoms)
    for i,atom in zip(range(n_atoms),atoms):
        F_i = np.zeros(hx.shape,dtype=complex)
        idx = fa==atom
        for ri in ra[idx,:]:
            F_i += np.exp(2*pi*1J*(ri[0]*hx+ri[1]*ky))
        Fhl += F_i*fq[i]
    return hl,Fhl

###
def get_pendulossung(name='Si',miller=[0,0,0],keV=200,opt='p'):
    ''' give the theorertical 2-beam approximation Pendullosung thickness
    - `name`    : compound
    - `miller`  : [h,k,l]
    - `keV`     : wavelength (keV)
    RETURN
    - xi : Pendullosung thickness (A)
    '''
    # compute structure factor
    from crystals import Crystal
    crys = Crystal.from_database(name)
    lat_vec  = crys.reciprocal_vectors
    pattern  = np.array([list(a.coords_fractional)+[a.atomic_number] for a in crys.atoms])
    hkl,Fhkl = structure_factor3D(pattern,lat_vec,hklMax=max(miller),sym=0,v='')
    ax,by,cz = crys.lattice_parameters[:3]
    Vcell    = crys.volume
    # compute Pendullosung
    h,k,l = miller
    Ug,K  = np.abs(Fhkl[h,k,l])/Vcell,1/scatf.wavelength(keV)
    xi  = K/Ug
    if 'p' in opt:
        print(green+"\tPendullosung thickness "+name+'[%d%d%d]' %(h,k,l)+black)
        print('%-5s= %.2f\n%-5s= %.2f\n%-5s= %.2f'%('K',K,'Ug',Ug,'xi',xi))
    return xi

##########################################################################
###def : display
##########################################################################
def plot_structure3D(hkl,Fhkl,log=0,**kwargs):
    '''scatter plot of the intensity'''
    hx,ky,lz = hkl
    hx,ky,lz = hx.flatten(),ky.flatten(),lz.flatten()
    S        = np.abs(Fhkl.flatten())**2
    if log : S = np.log10(S+1)
    N = hx.max()
    lims = [0,N,0,N,0,N]
    dsp.stddisp(scat=[hx,ky,lz,S],rc='3d',xylims=lims,**kwargs)

def plot2Dcutplane(Shkl,n='100',title='auto',**kwargs):
    '''Not working for arbitrary cut yet '''
    N = Shkl.shape[0]
    if title=='auto' : title='Silicon([%s]) $S_{hk}$' %n
    if   n=='100' : Scut = Shkl[0,:,:]
    elif n=='110' : Scut = Shkl[np.identity(N,dtype=bool),:]
    dsp.stddisp(im=Scut,imOpt='c',legOpt=0,title=title,**kwargs)

def plot1Dcutline(Shkl,u='001',lat_vec=None,dopt='fq',**kwargs):
    '''
    u : str cutline
    dopt:f(fe(q)),q(q units)
    '''
    N,labx = Shkl.shape[0],'$index$'
    ql = np.arange(N)
    if u == '001' :
        Scut = Shkl[0,0,:]
    else :
        Scut = Shkl[0,0,:]
    plts = [[ql,Scut,'bs-','$S_{%s}$' %u]]
    if 'q' in dopt or 'f' in dopt :
        b1,b2,b3 = lat_vec
        ql *= b3[2]/(2*pi)
        labx = '$q(A^{-1})$'
    if 'f' in dopt:
        #k_x,k_y,k_z = hx*b1[0]+ky*b2[0]+lz*b3[0],hx*b1[1]+ky*b2[1]+lz*b3[1],hx*b1[2]+ky*b2[2]+lz*b3[2]
        #q = np.sqrt(k_x**2+k_y**2+k_z**2)/(2*pi)
        qz = np.linspace(0,N,1000)*b3[2]/(2*pi)
        qz,fq = scatf.get_elec_atomic_factors([14],qz)
        plts += [[qz,64*fq[0]**2,'g--','$64f_e^2$'],[qz,32*fq[0]**2,'c--','$32f_e^2$']]
    dsp.stddisp(plts,labs=[labx,'$S_q(A^2)$'],**kwargs)

##########################################################################
##### def: misc
def get_miller3D(hklMax=10,sym=1):
    '''hklMax : Number of Beams such that Fourier components = 2^(N+1) - 1'''
    Nhkl = range(-hklMax*sym,hklMax+1)
    hkl = np.meshgrid(Nhkl,Nhkl,Nhkl)
    return hkl
def get_miller2D(hkMax=10,sym=1):
    Nhk = range(-hkMax*sym,hkMax+1)
    hk = np.meshgrid(Nhk,Nhk)
    return hk

##########################################################################
#### def : test
##########################################################################
def _test_structure_factor2D(N=5):
    pptype,a,b,angle='p1',50,50,90      #A
    pattern = np.array([[0.5,0.5,1]])

    p1=pg.Wallpaper(pptype,a,b,angle,pattern=pattern,gen=False,pOpt='')
    b1,b2 = p1.get_reciprocal_lattice_2D()

    hl = np.arange(-2**N+1,2**N)
    h,l = np.meshgrid(hl,hl)
    Fhl = get_structure_factor2D(pattern,h,l,[b1,b2])

    qx,qy = (h*b1[0]+l*b2[0])/(2*pi),(h*b1[1]+l*b2[1])/(2*pi)
    fig,(axM,axP) = create_fig('12',pad=0.5,rc='12')
    axM.pcolor(np.abs(Fhl)**2);
    axP.pcolor(qx,qy,np.abs(Fhl)**2);
    #axP.pcolor(kx,ky,np.angle(Fhl));
    fig.show()
    return h,l,qx,qy,Fhl

def _test_structure_factor3D(N=5,sym=False,**kwargs):
    from crystals import Crystal
    Si = Crystal.from_database('Si'); #vol     = Si.volume
    lat_vec = Si.reciprocal_vectors
    rj      = np.array([a.coords_fractional for a in Si.atoms])
    pattern = np.concatenate((rj,14*np.ones((rj.shape[0],1))),axis=1)
    hkl,Fhkl  = structure_factor3D(pattern,lat_vec,hklMax=N,sym=sym)
    Shkl = np.abs(Fhkl)**2
    #plot_structure3D(hkl,Fhkl,name=figpath+'Si_Shkl.png',opt='s',cmap='jet',ms=50,title='Si $c=log_{10}(S+1)$',view=[9,-72])
    Shkl = np.log10(Shkl+1)
    plot2Dcutplane(Shkl,n='100',name=figpath+'Si_S100.png',**kwargs)
    plot2Dcutplane(Shkl,n='110',name=figpath+'Si_S110.png',**kwargs)
    #plot1Dcutline(Shkl,u='001',dopt='')
    return hkl,Fhkl

if __name__ == "__main__" :
    figpath=get_figpath(__file__,"/figures/")
    plt.close('all')
    #hk,Fhk = _test_structure_factor2D(N=8)
    #hkl,Fhkl = _test_structure_factor3D(N=4,opt='p',cmap='jet')


import sys
import argparse

from postcrypt import Postcrypt

def create_arg_parser():
    parser = argparse.ArgumentParser(description='Postcrypt HTTP scripts')

    parser.add_argument('crypt', type=str, help='Scrypt to run')
    parser.add_argument('mode', type=str, nargs='?', help='Optional scrypt mode')
    parser.add_argument('--verbose', action='store_true', help='Enable verbose mode')
    parser.add_argument('--save', action='store_true', help='Remember input values')

    return parser

if __name__ == '__main__':
    args = create_arg_parser().parse_args()
    postcrypt = Postcrypt(args.crypt, mode=args.mode, verbose=args.verbose, save=args.save)
    postcrypt.make_default()
    postcrypt.process()


# -*- encoding: utf-8 -*-
import os
import zmq
import yaml
import core.framer
import core.loader
import zmq.eventloop
import zmq.eventloop.zmqstream
import multiprocessing

'''
Create a basic Actor that can take requests
from a Publisher and spawn processes to perform
the actions
'''

CONFIG_LOCATION = '/home/mp/devel/eventdrivetalk/conf/demo_actor.yml'

class Actor(object):
    def __init__(self, opts=None):
        if opts is None:
            self.opts = self.process_config(CONFIG_LOCATION)
        else:
            self.opts = opts

        # Start setting up ZeroMQ
        self.ctx = zmq.Context()
        self.socket = self.ctx.socket(zmq.SUB)

        self.socket.connect('tcp://localhost:2000')

        self.loop = zmq.eventloop.IOLoop.instance()
        self.stream = zmq.eventloop.zmqstream.ZMQStream(self.socket, self.loop)
        self.stream.on_recv(act)

        # Load up actions
        self.actions = loader.load_actions(self.opts, '/home/mp/devel/eventdrivetalk/actions')

    def start(self):
        print('Starting Actor')
        try:
            self.loop.start()
        except KeyboardInterrupt:
            self.stream.close()
            self.loop.stop()
            print('Shut down!')

    def process_config(self, config_location):
        if not os.path.exists(config_location):
            print('WARNING: No config file was found at {0}'.format(config_location))
            return {}
        else:
            try:
                fh_ = open(config_location)
                config = yaml.load(fh_)
            finally:
                fh_close()
            return config

    def act(self, msg):
        '''
        Take a received event and act on it
        '''
        event = framer.unpack(msg)
        action = event['tag'].split().pop()

        # Obviously, we'd normally do some sort of host checking
        # and auth so this is ridiculous but this is just for fun!
        #
        # Blind trust ahoy! ;]
        mp = multiprocessing.Process(target=self._act_target, args=(action))

    def _act_target(self,  action):
        # Perform the action!
        self.actions[action]()