# Copyright 2011 Justin Santa Barbara
# Copyright 2012 Hewlett-Packard Development Company, L.P.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import errno
import glob
import os
import shutil
import tempfile
import testtools

import mock
from oslo_concurrency import processutils
from oslotest import base as test_base
import six

from ironic_python_agent import errors
from ironic_python_agent import utils

if six.PY2:
    OPEN_FUNCTION_NAME = '__builtin__.open'
else:
    OPEN_FUNCTION_NAME = 'builtins.open'

class ExecuteTestCase(testtools.TestCase):
    """This class is a copy of the same class in openstack/ironic."""

    def test_retry_on_failure(self):
        fd, tmpfilename = tempfile.mkstemp()
        _, tmpfilename2 = tempfile.mkstemp()
        try:
            fp = os.fdopen(fd, 'w+')
            fp.write('''#!/bin/sh
# If stdin fails to get passed during one of the runs, make a note.
if ! grep -q foo
then
    echo 'failure' > "$1"
fi
# If stdin has failed to get passed during this or a previous run, exit early.
if grep failure "$1"
then
    exit 1
fi
runs="$(cat $1)"
if [ -z "$runs" ]
then
    runs=0
fi
runs=$(($runs + 1))
echo $runs > "$1"
exit 1
''')
            fp.close()
            os.chmod(tmpfilename, 0o755)
            try:
                self.assertRaises(processutils.ProcessExecutionError,
                                  utils.execute,
                                  tmpfilename, tmpfilename2, attempts=10,
                                  process_input=b'foo',
                                  delay_on_retry=False)
            except OSError as e:
                if e.errno == errno.EACCES:
                    self.skipTest("Permissions error detected. "
                                  "Are you running with a noexec /tmp?")
                else:
                    raise
            fp = open(tmpfilename2, 'r')
            runs = fp.read()
            fp.close()
            self.assertNotEqual(runs.strip(), 'failure',
                                'stdin did not always get passed correctly')
            runs = int(runs.strip())
            self.assertEqual(10, runs,
                             'Ran %d times instead of 10.' % (runs,))
        finally:
            os.unlink(tmpfilename)
            os.unlink(tmpfilename2)

    def test_unknown_kwargs_raises_error(self):
        self.assertRaises(processutils.UnknownArgumentError,
                          utils.execute,
                          '/usr/bin/env', 'true',
                          this_is_not_a_valid_kwarg=True)

    def test_check_exit_code_boolean(self):
        utils.execute('/usr/bin/env', 'false', check_exit_code=False)
        self.assertRaises(processutils.ProcessExecutionError,
                          utils.execute,
                          '/usr/bin/env', 'false', check_exit_code=True)

    def test_no_retry_on_success(self):
        fd, tmpfilename = tempfile.mkstemp()
        _, tmpfilename2 = tempfile.mkstemp()
        try:
            fp = os.fdopen(fd, 'w+')
            fp.write('''#!/bin/sh
# If we've already run, bail out.
grep -q foo "$1" && exit 1
# Mark that we've run before.
echo foo > "$1"
# Check that stdin gets passed correctly.
grep foo
''')
            fp.close()
            os.chmod(tmpfilename, 0o755)
            try:
                utils.execute(tmpfilename,
                              tmpfilename2,
                              process_input=b'foo',
                              attempts=2)
            except OSError as e:
                if e.errno == errno.EACCES:
                    self.skipTest("Permissions error detected. "
                                  "Are you running with a noexec /tmp?")
                else:
                    raise
        finally:
            os.unlink(tmpfilename)
            os.unlink(tmpfilename2)

class GetAgentParamsTestCase(test_base.BaseTestCase):

    @mock.patch('oslo_log.log.getLogger')
    @mock.patch(OPEN_FUNCTION_NAME)
    def test__read_params_from_file_fail(self, logger_mock, open_mock):
        open_mock.side_effect = Exception
        params = utils._read_params_from_file('file-path')
        self.assertEqual(params, {})

    @mock.patch(OPEN_FUNCTION_NAME)
    def test__read_params_from_file(self, open_mock):
        kernel_line = 'api-url=http://localhost:9999 baz foo=bar\n'
        open_mock.return_value.__enter__ = lambda s: s
        open_mock.return_value.__exit__ = mock.Mock()
        read_mock = open_mock.return_value.read
        read_mock.return_value = kernel_line
        params = utils._read_params_from_file('file-path')
        open_mock.assert_called_once_with('file-path')
        read_mock.assert_called_once_with()
        self.assertEqual(params['api-url'], 'http://localhost:9999')
        self.assertEqual(params['foo'], 'bar')
        self.assertFalse('baz' in params)

    @mock.patch.object(utils, '_set_cached_params')
    @mock.patch.object(utils, '_read_params_from_file')
    @mock.patch.object(utils, '_get_cached_params')
    def test_get_agent_params_kernel_cmdline(self, get_cache_mock,
                                             read_params_mock,
                                             set_cache_mock):
        get_cache_mock.return_value = {}
        expected_params = {'a': 'b'}
        read_params_mock.return_value = expected_params
        returned_params = utils.get_agent_params()
        read_params_mock.assert_called_once_with('/proc/cmdline')
        self.assertEqual(expected_params, returned_params)
        set_cache_mock.assert_called_once_with(expected_params)

    @mock.patch.object(utils, '_set_cached_params')
    @mock.patch.object(utils, '_get_vmedia_params')
    @mock.patch.object(utils, '_read_params_from_file')
    @mock.patch.object(utils, '_get_cached_params')
    def test_get_agent_params_vmedia(self, get_cache_mock,
                                     read_params_mock,
                                     get_vmedia_params_mock,
                                     set_cache_mock):
        get_cache_mock.return_value = {}
        kernel_params = {'boot_method': 'vmedia'}
        vmedia_params = {'a': 'b'}
        expected_params = dict(list(kernel_params.items()) +
                               list(vmedia_params.items()))
        read_params_mock.return_value = kernel_params
        get_vmedia_params_mock.return_value = vmedia_params

        returned_params = utils.get_agent_params()
        read_params_mock.assert_called_once_with('/proc/cmdline')
        self.assertEqual(expected_params, returned_params)
        # Make sure information is cached
        set_cache_mock.assert_called_once_with(expected_params)

    @mock.patch.object(utils, '_set_cached_params')
    @mock.patch.object(utils, '_get_cached_params')
    def test_get_agent_params_from_cache(self, get_cache_mock,
                                         set_cache_mock):
        get_cache_mock.return_value = {'a': 'b'}
        returned_params = utils.get_agent_params()
        expected_params = {'a': 'b'}
        self.assertEqual(expected_params, returned_params)
        self.assertEqual(0, set_cache_mock.call_count)

    @mock.patch(OPEN_FUNCTION_NAME)
    @mock.patch.object(glob, 'glob')
    def test__get_vmedia_device(self, glob_mock, open_mock):

        glob_mock.return_value = ['/sys/class/block/sda/device/model',
                                  '/sys/class/block/sdb/device/model',
                                  '/sys/class/block/sdc/device/model']
        fobj_mock = mock.MagicMock()
        mock_file_handle = mock.MagicMock()
        mock_file_handle.__enter__.return_value = fobj_mock
        open_mock.return_value = mock_file_handle

        fobj_mock.read.side_effect = ['scsi disk', Exception, 'Virtual Media']
        vmedia_device_returned = utils._get_vmedia_device()
        self.assertEqual('sdc', vmedia_device_returned)

    @mock.patch.object(shutil, 'rmtree', autospec=True)
    @mock.patch.object(tempfile, 'mkdtemp', autospec=True)
    @mock.patch.object(utils, '_read_params_from_file')
    @mock.patch.object(os.path, 'exists')
    @mock.patch.object(os, 'mkdir')
    @mock.patch.object(utils, 'execute')
    def test__get_vmedia_params_by_label(self, execute_mock, mkdir_mock,
                                         exists_mock, read_params_mock,
                                         mkdtemp_mock, rmtree_mock):
        mkdtemp_mock.return_value = "/tempdir"

        null_output = ["", ""]
        expected_params = {'a': 'b'}
        read_params_mock.return_value = expected_params
        exists_mock.return_value = True
        execute_mock.side_effect = [null_output, null_output]

        returned_params = utils._get_vmedia_params()

        execute_mock.assert_any_call('mount', "/dev/disk/by-label/ir-vfd-dev",
                                     "/tempdir")
        read_params_mock.assert_called_once_with("/tempdir/parameters.txt")
        exists_mock.assert_called_once_with("/dev/disk/by-label/ir-vfd-dev")
        execute_mock.assert_any_call('umount', "/tempdir")
        self.assertEqual(expected_params, returned_params)
        mkdtemp_mock.assert_called_once_with()
        rmtree_mock.assert_called_once_with("/tempdir")

    @mock.patch.object(shutil, 'rmtree', autospec=True)
    @mock.patch.object(tempfile, 'mkdtemp', autospec=True)
    @mock.patch.object(utils, '_get_vmedia_device')
    @mock.patch.object(utils, '_read_params_from_file')
    @mock.patch.object(os.path, 'exists')
    @mock.patch.object(os, 'mkdir')
    @mock.patch.object(utils, 'execute')
    def test__get_vmedia_params_by_device(self, execute_mock, mkdir_mock,
                                          exists_mock, read_params_mock,
                                          get_device_mock, mkdtemp_mock,
                                          rmtree_mock):
        mkdtemp_mock.return_value = "/tempdir"

        null_output = ["", ""]
        expected_params = {'a': 'b'}
        read_params_mock.return_value = expected_params
        exists_mock.return_value = False
        execute_mock.side_effect = [null_output, null_output]
        get_device_mock.return_value = "sda"

        returned_params = utils._get_vmedia_params()

        execute_mock.assert_any_call('mount', "/dev/sda",
                                     "/tempdir")
        read_params_mock.assert_called_once_with("/tempdir/parameters.txt")
        execute_mock.assert_any_call('umount', "/tempdir")
        self.assertEqual(expected_params, returned_params)
        mkdtemp_mock.assert_called_once_with()
        rmtree_mock.assert_called_once_with("/tempdir")

    @mock.patch.object(utils, '_get_vmedia_device')
    @mock.patch.object(os.path, 'exists')
    def test__get_vmedia_params_cannot_find_dev(self, exists_mock,
                                                get_device_mock):
        get_device_mock.return_value = None
        exists_mock.return_value = False
        self.assertRaises(errors.VirtualMediaBootError,
                          utils._get_vmedia_params)

    @mock.patch.object(shutil, 'rmtree', autospec=True)
    @mock.patch.object(tempfile, 'mkdtemp', autospec=True)
    @mock.patch.object(utils, '_get_vmedia_device')
    @mock.patch.object(utils, '_read_params_from_file')
    @mock.patch.object(os.path, 'exists')
    @mock.patch.object(os, 'mkdir')
    @mock.patch.object(utils, 'execute')
    def test__get_vmedia_params_mount_fails(self, execute_mock,
                                            mkdir_mock, exists_mock,
                                            read_params_mock,
                                            get_device_mock, mkdtemp_mock,
                                            rmtree_mock):
        mkdtemp_mock.return_value = "/tempdir"

        expected_params = {'a': 'b'}
        exists_mock.return_value = True
        read_params_mock.return_value = expected_params
        get_device_mock.return_value = "sda"

        execute_mock.side_effect = processutils.ProcessExecutionError()

        self.assertRaises(errors.VirtualMediaBootError,
                          utils._get_vmedia_params)

        execute_mock.assert_any_call('mount', "/dev/disk/by-label/ir-vfd-dev",
                                     "/tempdir")
        mkdtemp_mock.assert_called_once_with()
        rmtree_mock.assert_called_once_with("/tempdir")

    @mock.patch.object(shutil, 'rmtree', autospec=True)
    @mock.patch.object(tempfile, 'mkdtemp', autospec=True)
    @mock.patch.object(utils, '_get_vmedia_device')
    @mock.patch.object(utils, '_read_params_from_file')
    @mock.patch.object(os.path, 'exists')
    @mock.patch.object(os, 'mkdir')
    @mock.patch.object(utils, 'execute')
    def test__get_vmedia_params_umount_fails(self, execute_mock, mkdir_mock,
                                             exists_mock, read_params_mock,
                                             get_device_mock, mkdtemp_mock,
                                             rmtree_mock):
        mkdtemp_mock.return_value = "/tempdir"

        null_output = ["", ""]
        expected_params = {'a': 'b'}
        exists_mock.return_value = True
        read_params_mock.return_value = expected_params
        get_device_mock.return_value = "sda"

        execute_mock.side_effect = [null_output,
                                    processutils.ProcessExecutionError()]

        returned_params = utils._get_vmedia_params()

        execute_mock.assert_any_call('mount', "/dev/disk/by-label/ir-vfd-dev",
                                     "/tempdir")
        read_params_mock.assert_called_once_with("/tempdir/parameters.txt")
        execute_mock.assert_any_call('umount', "/tempdir")
        self.assertEqual(expected_params, returned_params)
        mkdtemp_mock.assert_called_once_with()
        rmtree_mock.assert_called_once_with("/tempdir")

    @mock.patch.object(shutil, 'rmtree', autospec=True)
    @mock.patch.object(tempfile, 'mkdtemp', autospec=True)
    @mock.patch.object(utils, '_get_vmedia_device')
    @mock.patch.object(utils, '_read_params_from_file')
    @mock.patch.object(os.path, 'exists')
    @mock.patch.object(os, 'mkdir')
    @mock.patch.object(utils, 'execute')
    def test__get_vmedia_params_rmtree_fails(self, execute_mock, mkdir_mock,
                                             exists_mock, read_params_mock,
                                             get_device_mock, mkdtemp_mock,
                                             rmtree_mock):
        mkdtemp_mock.return_value = "/tempdir"
        rmtree_mock.side_effect = Exception

        null_output = ["", ""]
        expected_params = {'a': 'b'}
        exists_mock.return_value = True
        read_params_mock.return_value = expected_params
        get_device_mock.return_value = "sda"

        execute_mock.return_value = null_output

        returned_params = utils._get_vmedia_params()

        execute_mock.assert_any_call('mount', "/dev/disk/by-label/ir-vfd-dev",
                                     "/tempdir")
        read_params_mock.assert_called_once_with("/tempdir/parameters.txt")
        execute_mock.assert_any_call('umount', "/tempdir")
        self.assertEqual(expected_params, returned_params)
        mkdtemp_mock.assert_called_once_with()
        rmtree_mock.assert_called_once_with("/tempdir")

    @mock.patch.object(utils, 'get_agent_params')
    def test_parse_root_device_hints(self, mock_get_params):
        mock_get_params.return_value = {
            'root_device': 'vendor=SpongeBob,model=Square%20Pants',
            'ipa-api-url': 'http://1.2.3.4:1234'
        }
        expected = {'vendor': 'spongebob', 'model': 'square pants'}
        result = utils.parse_root_device_hints()
        self.assertEqual(expected, result)

    @mock.patch.object(utils, 'get_agent_params')
    def test_parse_root_device_hints_no_hints(self, mock_get_params):
        mock_get_params.return_value = {
            'ipa-api-url': 'http://1.2.3.4:1234'
        }
        result = utils.parse_root_device_hints()
        self.assertEqual({}, result)

    @mock.patch.object(utils, 'get_agent_params')
    def test_parse_root_device_size(self, mock_get_params):
        mock_get_params.return_value = {
            'root_device': 'size=12345',
            'ipa-api-url': 'http://1.2.3.4:1234'
        }
        result = utils.parse_root_device_hints()
        self.assertEqual(12345, result['size'])

    @mock.patch.object(utils, 'get_agent_params')
    def test_parse_root_device_not_supported(self, mock_get_params):
        mock_get_params.return_value = {
            'root_device': 'foo=bar,size=12345',
            'ipa-api-url': 'http://1.2.3.4:1234'
        }
        self.assertRaises(errors.DeviceNotFound,
                          utils.parse_root_device_hints)

class TestFailures(testtools.TestCase):
    def test_get_error(self):
        f = utils.AccumulatedFailures()
        self.assertFalse(f)
        self.assertIsNone(f.get_error())

        f.add('foo')
        f.add('%s', 'bar')
        f.add(RuntimeError('baz'))
        self.assertTrue(f)

        exp = ('The following errors were encountered:\n* foo\n* bar\n* baz')
        self.assertEqual(exp, f.get_error())

    def test_raise(self):
        class FakeException(Exception):
            pass

        f = utils.AccumulatedFailures(exc_class=FakeException)
        self.assertIsNone(f.raise_if_needed())
        f.add('foo')
        self.assertRaisesRegexp(FakeException, 'foo', f.raise_if_needed)


from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from six.moves import xrange

import numpy as np

from keras import backend as K
from . import backend_addons as K2

from ..hyp_defs import float_keras

log2pi=np.log(2*np.pi).astype(float_keras())
one=np.exp(0).astype(float_keras())

def bernoulli(y_true, y_pred):
    return K.sum(K.binary_crossentropy(y_pred, y_true), axis=-1)

def diag_normal(y_true, y_pred):
    y_mean, y_logvar = y_pred
    y_var = K.exp(y_logvar)
    dist2=K.square(y_true-y_mean)/y_var
    return 0.5*K.sum(log2pi+y_logvar+dist2, axis=-1)

def normal_1chol(y_true, y_pred):
    y_mean, y_logvar, y_chol = y_pred

    isigma = K.exp(-y_logvar/2)
    iy_chol = K2.matrix_inverse(y_chol)

    distn  = K.dot(y_true-y_mean, iy_chol) * isigma
    dist2 = K.square(distn)
    return 0.5*K.sum(log2pi+y_logvar+dist2, axis=-1)

def normal(y_true, y_pred):
    y_mean, y_logvar, y_chol = y_pred

    isigma = K.exp(-y_logvar/2)
    f = lambda x: K2.matrix_inverse(x)
    iy_chol = K.map_fn(f, y_chol)

    dist  = y_true-y_mean
    distn = K.batch_dot(dist, iy_chol, axes=(1, 1)) * isigma
    dist2 = K.square(distn)
    return 0.5*K.sum(log2pi+y_logvar+dist2, axis=-1)

def normal_3d(y_true, y_pred):
    y_mean, y_logvar, y_chol = y_pred

    x_dim = K.cast(K.shape(y_mean)[-1], 'int32')
    seq_length = K.cast(K.shape(y_mean)[-2], 'int32')

    y_chol = K.reshape(y_chol, (-1, x_dim, x_dim))

    isigma = K.exp(-y_logvar/2)
    f = lambda x: K2.matrix_inverse(x)
    iy_chol = K.map_fn(f, y_chol)

    dist = y_true-y_mean
    dist = K.reshape(dist, (-1, x_dim))
    distn = K.batch_dot(dist, iy_chol, axes=(1, 1))
    distn = K.reshape(distn, (-1, seq_length, x_dim)) * isigma
    dist2 = K.square(distn)
    return 0.5*K.sum(log2pi+y_logvar+dist2, axis=-1)

def normal_1chol_3d(y_true, y_pred):
    y_mean, y_logvar, y_chol = y_pred

    x_dim = K.cast(K.shape(y_mean)[-1], 'int32')
    seq_length = K.cast(K.shape(y_mean)[-2], 'int32')

    isigma = K.exp(-y_logvar/2)
    iy_chol = K2.matrix_inverse(y_chol)

    dist = y_true-y_mean
    #dist = K.reshape(dist, (-1, x_dim))
    distn  = K.dot(dist, iy_chol) * isigma
    #distn = K.reshape(distn, (-1, seq_length, x_dim))
    dist2 = K.square(distn)
    return 0.5*K.sum(log2pi+y_logvar+dist2, axis=-1)

def kl_normal_vs_std_normal(qy, beta=one):
    y_mean, y_logvar = qy[:2]
    T = one/beta
    return -0.5*K.sum((T-one) * log2pi + T + T * y_logvar
                      - K.square(y_mean) - K.exp(y_logvar), axis=-1)

def kl_normal_vs_diag_normal(qy1, qy2, beta=one):
    y1_mean, y1_logvar = qy1[:2]
    y2_mean, y2_logvar = qy2[:2]
    T = one/beta
    v1 = K.exp(y1_logvar)
    v2 = K.exp(y2_logvar)
    return -0.5*K.sum((T-one) * log2pi + T + T * y1_logvar - y2_logvar
                      - (K.square(y1_mean-y2_mean)+v1)/v2, axis=-1)

def kl_normal_commonvar2(qy1, qy2, beta=one):
    y1_mean, y1_logvar, y1_chol = qy1
    y2_mean, y2_logvar, y2_chol = qy2

    dim=K.shape(y1_mean)[-1]

    iy2_chol = K2.matrix_inverse(y2_chol)
    iy2_chol*= iy_chol * K.exp(-y2_logvar/2)

    dist  = y1_mean-y2_mean
    distn = K.batch_dot(dist, iy2_chol, axes=(2, 0))
    dist2 = K.square(distn)

    iy2_var = K.reshape(np.dot(iy2_chol, iy2_chol.T), shape=(-1))
    y1_chol *= K.exp(y1_logvar/2)
    f = lambda x: K.reshape(K.dot(x.T, x), shape=(-1, dim**2))
    y1_var = K.map_fh(f, y1_chol)

    tr = K.dot(y1_var, iy2_var)

    T = one/beta
    v1 = K.exp(y1_logvar)
    v2 = K.exp(y2_logvar)
    return -0.5*K.sum((T-1)*log2pi + T + T*y1_logvar - y2_logvar
                      - dist2 - tr, axis=-1)


import os
import json

__version__ = '0.1.0'
__all__ = ['fields', 'response', 'logger', 'validators', 'decorators']

# Common variables for file locations
TICKET_API_ROOT = '/var/www/html/ticketapi'
SETTINGS_FILE = os.path.join(TICKET_API_ROOT, 'settings.json')
LOG_FILE = '/var/log/ticket-api/ticket-api.log'

# Open the file and read it's json contents into SETTINGS
with open(SETTINGS_FILE, 'r') as settings:
    SETTINGS = json.loads(settings.read())


#!/usr/bin/python3.4
# -*- coding=utf-8 -*-
#本脚由亁颐堂现任明教教主编写，用于乾颐盾Python课程！
#教主QQ:605658506
#亁颐堂官网www.qytang.com
#乾颐盾是由亁颐堂现任明教教主开发的综合性安全课程
#包括传统网络安全（防火墙，IPS...）与Python语言和黑客渗透课程！
import sys
sys.path.append('/usr/local/lib/python3.4/dist-packages/PyQYT/ExtentionPackages')
sys.path.append('/usr/lib/python3.4/site-packages/PyQYT/ExtentionPackages')
sys.path.append('../../ExtentionPackages')

import urllib.request
from bs4 import BeautifulSoup

def qyt_browser_proxy(url, proxy):
	ProxyHandlered= urllib.request.ProxyHandler(proxy)
	opener = urllib.request.build_opener(ProxyHandlered)
	urllib.request.install_opener(opener)
	urlContent = urllib.request.urlopen(url).read()
	soup = BeautifulSoup(urlContent, "html.parser")
	print(soup)

if __name__ == '__main__':
	#Proxy网址
	proxy = {'http': '120.198.244.29:8080'}
	url = 'http://www.qytang.com/'
	qyt_browser_proxy(url, proxy)


import os
import time
import sys
import numpy as np
import time

from slm import SLM
from camera import Camera, ImageHandler
import holograms as hg
from arrays import ArrayGenerator

import logging
logging.basicConfig(filename=r'Z:\Tweezer\Code\Python 3.7\slm\run2.log',level=logging.WARNING,format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

slm = SLM(monitor=1)
cam = Camera(exposure=95,gain=40,roi=[300,331,1121,1022])
center = (288,227)
waist = 336
complex_amp_mod = hg.ComplexAmpMod()

exposure = 95
gain = 40
exposure = cam.update_exposure(exposure)
gain = cam.update_gain(gain)

phase_correction = hg.misc.load("Z:/Tweezer/Code/Python 3.7/slm/images/2021/May/25/zernike_measurements/I_opt.png")
blazing = hg.gratings.diag(-12.2)
holo = (blazing+hg.lenses.focal_plane_shift(0.614,center)+phase_correction)%1
holo = hg.apertures.circ(holo,center,210)

gratings = np.linspace(-12.3,-11.7,7)
np.random.shuffle(gratings)

for i,grating in enumerate(gratings):
    try:
        blazing = hg.gratings.diag(grating)
        holo = (blazing+hg.lenses.focal_plane_shift(0.614,center)+phase_correction)%1
        holo = hg.apertures.circ(holo,center,210)
        imager = ImageHandler(measure_params={'grating':grating})
        array_gen = ArrayGenerator(slm,cam,imager,holo,circ_aper_center=center,circ_aper_radius=210,calibration_blazing=hg.gratings.diag(-12)*0.5)
        array_gen.generate_input_intensity(waist,center)
        array_gen.load_traps_from_image(r"Z:\Tweezer\Code\Python 3.7\slm\image_processing\smiley.png",4)
        #array_gen.load_prev_trap_locs(r"Z:\Tweezer\Code\Python 3.7\slm\images\2021\May\26\Measure 96\trap_df.csv")
        array_gen.get_single_trap_coords()
        array_gen.generate_initial_hologram(iterations=110)
        array_gen.get_trap_depths(reps=1,plot=False)
        array_gen.generate_corrected_hologram(iterations=110)
        array_gen.get_trap_depths(reps=1,plot=False)
        array_gen.save_trap_df()
    except Exception as e:
       logging.error("Exception occured. Relative measure {}, aperture {}".format(i,grating),exc_info=True)
print("program completed")


import socket
from pathlib import Path
import os
import json
import struct
import time

HOST = 'localhost'
PORT = 10001

def getFileInfo(filePath):
	file = filePath.strip()
	if not Path.is_file(Path(file)):
		print("上传的文件路径错误")

	FileSize = os.path.getsize(file)
	FileName = 'new_' + str(Path(file).name)
	dirc = {
		'FileName': FileName,
		'FileSize': FileSize,
	}
	head_info = json.dumps(dirc)
	head_info_len = struct.pack('i', len(head_info))
	# head_info_len = len(head_info)
	return head_info_len,head_info,file

def test_server():
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	s.bind((HOST, PORT))
	s.listen(1)
	while True:
		conn, addr = s.accept()
		print(f"链接人信息 : {conn}")
		while True:
			time.sleep(5)
			if not conn:
				print("链接客户端失败")
				break
			FilePath = input("输入文件路径>>>")
			# 获取文件信息
			head_info_len,head_info,file_path = getFileInfo(FilePath)
			# 发送文件信息的信息长度，给客户端
			conn.sendall(head_info_len)
			# 发送文件信息给客户端
			conn.sendall(head_info.encode('utf-8'))
			try:
				with open(file_path, 'rb') as f:
					while 1:
						data = f.read(1024)
						if not data:
							print("读取的内容已经完成")
							break
						print(f"data.decode('utf-8') >>> {data}\n")
						conn.sendall(data)
					break
			except Exception as err:
				print(f"读取文件报错：{err}")
				break
		conn.close()
	s.close()

if __name__ == "__main__":
	# test_server()
	# print(getFileInfo(' /Users/qtt/Desktop/git-guozhijia/week02/pathlib_t.py '))
	test_server()


import random
import time
try:
    import urllib2  # python2
    # urllib.request = urllib2
except ImportError:
    import urllib.request  # python3 urllib2 = urllib.request
    urllib2 = urllib.request

import requests

class GetUrlParse(object):

    """docstring for GetParseUrl"""
    def __init__(self):

        self.sleep_time = [[0, 10], [0, 10], [0, 10], [0, 10], [0, 10], [20, 30]]
        self.request_url = []
        self.cookies = []
        self.ua_list = [
                'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
                'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
                'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
                'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
                'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
                'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
            ]

    def user_agent(self):
        user_agent_choice = {'user-agent': random.choice(self.ua_list)}
        return user_agent_choice

    def request_frequency(self):
        time_sleep_choice = random.sample(self.sleep_time, 1)[0]
        request_frequency_choice = random.randint(time_sleep_choice[0], time_sleep_choice[1])

        print('The program is sleeping:need {}s'.format(request_frequency_choice))
        time.sleep(request_frequency_choice)

    def url_parse(self, url):
        self.request_frequency()
        try:
            response = requests.get(url, headers=self.user_agent())
            if response.status_code is 200:
                print('Parse url:({}) successful!'.format(url))
                return response.text
            else:
                print("The url's status_code isn't 200! Please check!")
                return None
        except (ConnectionError):
            print('Error occurred')
            return None
        except Exception as result:
            print('捕获到其他异常:{}'.format(result))
            return None

    def get(self, urls):

        if isinstance(urls, list):

            return [self.url_parse(i) for i in urls]

        elif isinstance(urls, str):
            return self.url_parse(urls)

        else:
            print('Please input a valid value in function "get", for excaple: ["www.abc.com", "www.abc.com", "www.abc.com"] or "www.abc.com".')

class ResponseParse(GetUrlParse):
    def __init__(self):
        super(ResponseParse, self).__init__()

    def test(self):
        response = self.get('https://www.baidu.com/')
        print(response)

xx = ResponseParse()
xx.test()

爬虫/对爬虫请求的封装.py


import random
from django.shortcuts import render
from django.contrib.auth.models import User, Group
from django_filters.rest_framework import DjangoFilterBackend

from rest_framework import filters
from rest_framework import viewsets
from rest_framework.permissions import IsAuthenticatedOrReadOnly, IsAuthenticated

from . import models
from . import serializers
# from trivia.permissions import IsOwnerOrReadOnly
# from trivia.permissions import IsStaffOrTargetUser

"""
This viewset automatically provides `list`, `create`, `retrieve`,
`update` and `destroy` actions for the User model.
"""
class UserViewSet(viewsets.ModelViewSet):
    queryset = User.objects.all().order_by('-date_joined')
    serializer_class = serializers.User

    filter_backends = [filters.SearchFilter]
    search_fields = ['=username']

"""
This viewset automatically provides `list`, `create`, `retrieve`,
`update` and `destroy` actions for the Profile model.
"""
class ProfileViewSet(viewsets.ModelViewSet):
    queryset = models.Profile.objects.all()
    serializer_class = serializers.ProfileSerializer
    permission_classes = [IsAuthenticated]

    filter_backends = [DjangoFilterBackend, filters.SearchFilter]
#    ordering =
    filterset_fields = ['score', 'rank']
    search_fields = ['=user']

"""
This viewset automatically provides `list`, `create`, `retrieve`,
`update` and `destroy` actions for the Trivia Game model.
"""
class TriviaViewSet(viewsets.ModelViewSet):
    queryset = models.Trivia.objects.all()
    serializer_class = serializers.TriviaSerializer
    permission_classes = [IsAuthenticatedOrReadOnly]

    filter_backends = [DjangoFilterBackend, filters.SearchFilter]
    filterset_fields = ['type', 'category']
    search_fields = ['=id', '=title']

"""
This viewset automatically provides `list`, `create`, `retrieve`,
`update` and `destroy` actions for the Question model.
"""
class QuestionViewSet(viewsets.ModelViewSet):
    queryset = models.Question.objects.all()
    serializer_class = serializers.QuestionSerializer
    permission_classes = [IsAuthenticatedOrReadOnly]

    filter_backends = [DjangoFilterBackend, filters.SearchFilter]
    filterset_fields = ['type', 'category']
    search_fields = ['=id', '=content']


#!/usr/bin/env python
# encoding: utf-8

import argparse

import _init_paths
import dataset

def main():
    parser = argparse.ArgumentParser(description='collect data list from MPISintel')

    parser.add_argument('data_path', type=str,
                        help='the dataset path')
    parser.add_argument('list_path', type=str,
                        help='the output path')
    parser.add_argument('--test_size', type=float, default=None,
                        help='test size')

    args = parser.parse_args()
    mpi_sintel = dataset.MPISintel(args.data_path, args.test_size)
    train_list_file, test_list_file = mpi_sintel.write(args.list_path)
    return

if __name__ == '__main__':
    main()


#%% Imports
import serial
import time

#%% Initializations

Serial_Port = "COM8"
Baud_Rate = 115200

SER = serial.Serial( Serial_Port, Baud_Rate)

#%% Main Run
if __name__ == "__main__":
    SER.flushInput()

    # Clear LCD
    SER.write(b'\xff\xff\x02***\xc8\xc8')

    time.sleep(1)
    # LCD Goto (Row, Col)
    row, col = (2, 4)
    tmp_packet = '\xff\xff\x03{}{}*\xc8\xc8'.format(chr(row), chr(col))
    SER.write(bytes(tmp_packet, encoding='ANSI'))

    time.sleep(1)
    # Send a string to show on LCD
    for char_i in 'Parham Joon!':
        tmp_packet = '\xff\xff\x04{}\x06*\xc8\xc8'.format(char_i)
        SER.write(bytes(tmp_packet, encoding='ANSI'))
        time.sleep(2000/Baud_Rate)

    time.sleep(1)
    # LCD Shift Entire Display to the Left
    SER.write(b'\xff\xff\x05\x18**\xc8\xc8')

    SER.close()


##############################################################################
#
# Copyright (c) 2010 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""DottedName field tests
"""
from unittest import main, makeSuite

from six import b
from zope.schema import DottedName
from zope.schema.tests.test_field import FieldTestBase
from zope.schema.interfaces import InvalidDottedName, RequiredMissing

class DottedNameTest(FieldTestBase):
    """Test the DottedName Field."""

    _Field_Factory = DottedName

    def testValidate(self):
        field = self._Field_Factory(required=False)

        field.validate(None)
        field.validate('foo.bar')
        field.validate('foo.bar0')
        field.validate('foo0.bar')

        # We used to incorrectly allow ^: https://bugs.launchpad.net/zope.schema/+bug/191236
        self.assertRaises(InvalidDottedName, field.validate, 'foo.bar^foobar')
        self.assertRaises(InvalidDottedName, field.validate, 'foo^foobar.bar')
        # dotted names cannot start with digits
        self.assertRaises(InvalidDottedName, field.validate, 'foo.0bar')
        self.assertRaises(InvalidDottedName, field.validate, '0foo.bar')

    def testValidateRequired(self):
        field = self._Field_Factory(required=True)

        field.validate('foo.bar')

        self.assertRaises(RequiredMissing, field.validate, None)

def test_suite():
    suite = makeSuite(DottedNameTest)
    return suite


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Account',
            fields=[
                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
                ('password', models.CharField(max_length=128, verbose_name='password')),
                ('last_login', models.DateTimeField(verbose_name='last login', blank=True, null=True)),
                ('username', models.CharField(max_length=40, db_index=True, unique=True, verbose_name='Логин')),
                ('first_name', models.CharField(max_length=40, db_index=True, blank=True, default='', verbose_name='Имя')),
                ('last_name', models.CharField(max_length=40, db_index=True, blank=True, default='', verbose_name='Фамилия')),
                ('patronymic', models.CharField(max_length=40, db_index=True, blank=True, default='', verbose_name='Отчество')),
                ('is_active', models.BooleanField(verbose_name='Активен?', default=True)),
                ('is_admin', models.BooleanField(verbose_name='', default=False)),
                ('is_steward', models.BooleanField(verbose_name='Староста', default=False)),
                ('is_deanery', models.BooleanField(verbose_name='Деканат', default=False)),
                ('created_at', models.DateTimeField(verbose_name='Создан', auto_now_add=True)),
                ('updated_at', models.DateTimeField(verbose_name='Обновлен', auto_now=True)),
            ],
            options={
                'verbose_name_plural': 'пользователи',
                'verbose_name': 'пользователь',
                'get_latest_by': 'created_at',
                'ordering': ['-created_at'],
            },
        ),
        migrations.CreateModel(
            name='UniversityGroup',
            fields=[
                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
                ('name', models.CharField(max_length=15, default='', blank=True, unique=True, verbose_name='Название группы')),
                ('is_active', models.BooleanField(verbose_name='Активна?', default=True)),
                ('created_at', models.DateTimeField(verbose_name='Создана', auto_now_add=True)),
                ('updated_at', models.DateTimeField(verbose_name='Обновлена', auto_now=True)),
            ],
            options={
                'verbose_name_plural': 'группы студентов',
                'verbose_name': 'группа студентов',
                'get_latest_by': 'created_at',
                'ordering': ['-created_at'],
            },
        ),
        migrations.AddField(
            model_name='account',
            name='universityGroup',
            field=models.ForeignKey(blank=True, to='accounts.UniversityGroup', verbose_name='Группа студента', null=True),
        ),
    ]


#!/usr/bin/env python3
import sys

from tqdm import tqdm
import argparse
import numpy as np

def parse_arguments():
    def to_zero_base(x):
        x = int(x)
        if x < 1:
            raise argparse.ArgumentTypeError("Field starts from 1.")
        return x - 1

    parser = argparse.ArgumentParser(description='')

    parser.add_argument(
        '-i',
        '--input',
        type=argparse.FileType('r'),
        default=sys.stdin,
        help='')

    parser.add_argument(
        '-d', '--delimiter', default='\t', help='default to \\t')

    parser.add_argument(
        '--mode', choices=['key', 'value', 'both'], required=True)
    parser.add_argument('--key-field', type=to_zero_base, default=0)
    parser.add_argument('--key-num', type=int, default=1)
    parser.add_argument('--value-num', type=int, default=1)

    parser.add_argument(
        '-o',
        '--output',
        type=argparse.FileType('w'),
        default=sys.stdout,
        help='')

    return parser.parse_args()

def main():
    args = parse_arguments()
    data = {}
    for line in args.input:
        if not line.strip():
            continue
        splits = line.strip().split(args.delimiter)
        data.setdefault(splits[args.key_field], []).append(line)

    if args.mode in ['key', 'both']:
        keys = list(data.keys())
        keys = np.random.choice(
            keys, min(len(keys), args.key_num), replace=False)
        data = {k: data[k] for k in keys}

    if args.mode in ['value', 'both']:
        data = {
            k: np.random.choice(v, min(len(v), args.value_num), replace=False)
            for k, v in tqdm(data.items(), desc='Sampling values')
        }

    for values in data.values():
        args.output.writelines(list(values))

if __name__ == '__main__':
    main()


from django.urls import path
from . import views

app_name = 'docs_templates'

urlpatterns = [
	path('', views.List_of_documentsView.as_view(), name = 'list_of_documents'),
	path('<str:document_url>/', views.documents_context, name = 'document_template'),

]


import utils
import os

# https://sourceforge.net/p/mingw-w64/wiki2/Cross%20Win32%20and%20Win64%20compiler/
def build_toolchain(target_triple, package_version, num_jobs):
	print('Building toolchain for', target_triple)

	gcc_version = '6.1.0'
	binutils_version = '2.26'
	mingw_w64_version = '4.0.6'

	print('Downloading sources')

	utils.should_have('https://downloads.sourceforge.net/project/mingw-w64/mingw-w64/mingw-w64-release/mingw-w64-v{0}.tar.bz2'.format(mingw_w64_version), '0c407394b0d8635553f4fbca674cdfe446aac223e90b4010603d863e4bdd015c')
	utils.should_have('ftp://ftp.gnu.org/gnu/gcc/gcc-{0}/gcc-{0}.tar.bz2'.format(gcc_version), '09c4c85cabebb971b1de732a0219609f93fc0af5f86f6e437fd8d7f832f1a351')
	utils.should_have('ftp://ftp.gnu.org/gnu/binutils/binutils-{0}.tar.bz2'.format(binutils_version), 'c2ace41809542f5237afc7e3b8f32bb92bc7bc53c6232a84463c423b0714ecd9')

	utils.extract_tar('mingw-w64-v{0}.tar.bz2'.format(mingw_w64_version))
	utils.extract_tar('gcc-{0}.tar.bz2'.format(gcc_version))
	utils.extract_tar('binutils-{0}.tar.bz2'.format(binutils_version))

	log_dir = 'target/log/{0}'.format(target_triple)
	build_dir = 'target/build/{0}'.format(target_triple)
	sysroot = os.path.abspath('target/sysroots/{0}'.format(target_triple))

	utils.should_be_empty(log_dir)
	utils.should_be_empty(build_dir + '/binutils')
	utils.should_be_empty(build_dir + '/gcc')
	utils.should_be_empty(build_dir + '/headers')
	utils.should_be_empty(build_dir + '/crt')
	utils.should_be_empty(sysroot)

	binutils_gcc_options = [
		'--disable-lto', # We don't need LTO because it won't work with rustc which is based on LLVM.
		'--disable-nls', # English, motherfucker, do you speak it?
		'--disable-multilib',
		'--with-pkgversion={0}'.format(package_version),
		'--target={0}'.format(target_triple),
		'--with-sysroot={0}'.format(sysroot),
		'--prefix={0}'.format(sysroot),
	]

	env = {
		'PATH': os.environ['PATH'],
		'CFLAGS': '-O2',
		'CXXFLAGS': '-O2',
		'CFLAGS_FOR_TARGET': '-O2',
		'CXXFLAGS_FOR_TARGET': '-O2',
	}

	# binutils
	utils.execute(env, log_dir + '/01.binutils-configure.log', build_dir + '/binutils', [
		'../../../downloads/binutils-{0}/configure'.format(binutils_version), '--enable-deterministic-archives',
	] + binutils_gcc_options)

	utils.execute(env, log_dir + '/02.binutils-make.log', build_dir + '/binutils', [
		'make',	'--jobs={0}'.format(num_jobs)
	])

	utils.execute(env, log_dir + '/03.binutils-make-install.log', build_dir + '/binutils', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	# mingw-headers
	utils.execute(env, log_dir + '/04.headers-configure.log', build_dir + '/headers', [
		'../../../downloads/mingw-w64-v{0}/mingw-w64-headers/configure'.format(mingw_w64_version),
		'--host={0}'.format(target_triple),
		'--prefix={0}/{1}'.format(sysroot, target_triple),
	])

	utils.execute(env, log_dir + '/05.headers-make-install.log', build_dir + '/headers', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	os.symlink(sysroot + '/{0}'.format(target_triple), sysroot + '/mingw')

	# gcc (1 of 2)
	utils.execute(env, log_dir + '/06.gcc-configure.log', build_dir + '/gcc', [
		'../../../downloads/gcc-{0}/configure'.format(gcc_version), '--enable-languages=c',
	] + binutils_gcc_options)

	utils.execute(env, log_dir + '/07.gcc-make-all-gcc.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs), 'all-gcc'
	])

	utils.execute(env, log_dir + '/08.gcc-make-install-gcc.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs), 'install-gcc'
	])

	env['PATH'] = '{0}/bin:'.format(sysroot) + env['PATH']

	# crt
	utils.execute(env, log_dir + '/09.crt-configure.log', build_dir + '/crt', [
		'../../../downloads/mingw-w64-v{0}/configure'.format(mingw_w64_version),
		'--host={0}'.format(target_triple),
		'--prefix={0}/{1}'.format(sysroot, target_triple),
		'--with-sysroot={0}'.format(target_triple),
	])

	utils.execute(env, log_dir + '/10.crt-make.log', build_dir + '/crt', [
		'make', '--jobs={0}'.format(num_jobs),
	])

	utils.execute(env, log_dir + '/11.crt-make-install.log', build_dir + '/crt', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	# gcc (2 of 2)
	utils.execute(env, log_dir + '/12.gcc-make.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs)
	])

	utils.execute(env, log_dir + '/13.gcc-make-install.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	print('Toolchain for', target_triple, 'has been successfully built')


"""
    Owner: Nagadarshan N
    This program accepts a PDA which accepts by empty stack and converts it to PDA which accepts by final state

"""
#importing defaultdict for the internal use
from collections import defaultdict
from copy import copy
from copy import deepcopy
from prettytable import PrettyTable

#This is the class which defines PDA:
#This includes structure, transition functions and everything

class PDA:
    # The below Initializer __init__ performs the following:
    # Initializes the PDA with The number of states Q, Input alphabets Sigma, Stack alphabets Gamma,
    # Initial state q0 which belongs to Q,
    # Initial stack symbol Z0,
    # Set of Final states F
    def __init__(self, Q, S, G, q0, z0, M = None, F = None, _column_list = None):
        self.push_down_stack = list()
        self.Q = set(Q)
        self.S = set(S)
        self.G = set(G)
        self.q0 = q0
        self.z0 = z0
        self.push_down_stack.append(self.z0)

        if F != None:
            self.F = set(F)
        else:
            self.F = F

        #n is the total number of states
        self.n = len(self.Q)

        #_column_list is the helper list we use to index the columns
        self._column_list = defaultdict()

        #The _generate_list() function fills the _column_list dictionary
        if _column_list == None:
            self._generate_list()
        else:
            self._column_list = _column_list

        #m is the defines the number of columns
        #Let M be the global matrix of size m X n
        #checking the condition whether to accept matrix M explicitly or implicitly
        self.m = len(self._column_list)
        if M == None:
            self.M = [[None] * self.m for _ in range(self.n)]
        else:
            self.M = M

    #A helper function to help in indexing the columns
    def _generate_list(self):
        count = 0
        for i in self.S:
            for j in self.G:
                x = str(i) + "," + str(j)
                self._column_list[x] = count
                count += 1
        #For adding epsilon
        for j in self.G:
            x = "e" + "," + str(j)
            self._column_list[x] = count
            count += 1

    #The function input_delta takes the input of all transition function
    #This takes the input in the form (q, a, X) -> (p, g)
    def input_delta(self):
        #The below print statements prints the rules for inputting the delta
        print ("Please define the delta in the following format")
        print ("(q,a,X) (p,g)")
        print ("(no spaces within the brackets)")
        print ("(where q is the current state)")
        print ("(where a is the current consumable input alphabet)")
        print ("(where X is the stack input alphabet)")
        print ("(where p is the next state)")
        print ("(g is stack string after consuming X)")
        print ("(use z0 (small z) for Z0 )")
        print ("(q, e, z0) must be defined")
        print ("\n(please enter 000 to exit)")

        #The following is the infinite loop
        #Ends when the infinite loop is encountered
        while True:
            x = input()
            if "000" in x:
                break
            tr = x.split()

            if len(tr) != 2:
                print ("please enter in the proper format as mentioned above")
                continue

            d_input = tr[0]
            d_output = tr[1]
            d_input_array = d_input.split(",")
            d_output_array = d_output.split(",")

            if len(d_input_array) != 3:
                print ("Please enter properly")
                continue

            q = d_input_array[0].replace("(", "")
            a = d_input_array[1]
            X = d_input_array[2].replace(")", "")

            if len(d_output_array) != 2:
                print ("Please enter properly")

            p = d_output_array[0].replace("(", "")
            g = d_output_array[1].replace(")", "")

            assert q in self.Q, str(q) + " is not in the Q"
            assert (a in self.S) or (a == "e"), str(a) + " is not in the Sigma"
            assert X in self.G, str(X) + " is not in the Gamma"
            assert p in self.Q, str(p) + " is not in the Q"

            temp_string = str(a) + "," + str(X)
            temp_list = [p, g]

            self.M[int(q.replace("q", ""))][self._column_list[temp_string]] = temp_list

    #This is the transition function
    #This takes (q, a, X) as input
    #This outputs p the next state, and,
    #g the stack string after consuming X
    def _transition(self, q, a, X):
        assert q in Q, str(q) + " is not in the Q"
        assert a in S, str(a) + " is not in the Sigma"
        assert X in G, str(X) + " is not in the Gamma"

        temp_q = q.replace("q", "")
        temp_a = a
        temp_X = X

        p, g = M[int(temp_q)][self._column_list]
        return p, g

    #The funciton below checks whether a given PDA is accepted by empty stack or not
    def _check_acceptance_by_empty_stack(self):
        #First assuming the PDA is not accepted by empty stack
        flag = False
        temp_str = "e,z0"
        index = self._column_list[temp_str]

        #Later if epsilon found in the epsilon, z0
        for i in range(n):
            if M[i][index] == None:
                continue
            else:
                p, g = M[i][index]
                if g == "e":
                    flag = True
        return flag

        #The function print_PDA prints the given PDA's transitions
        #This function prints it in a Matrix format
        #This also prints Q, Sigma, Gamma, and set of final states
    def print_PDA(self):

        headers = list()
        headers.append("States")
        for key in self._column_list:
            headers.append(str(key))

        pda_m = PrettyTable(headers)

        for i in self.M:
            temp_list = list()
            z = "q"+str(self.M.index(i))
            temp_list.append(z)
            for j in i:
                temp_list.append(str(j))
            pda_m.add_row(temp_list)

        print (pda_m)

        print ("Where q0 is the starting state")
        if self.F != None:
            print (str(self.F) +" is the final state")

        print ("\nQ     is " + str(self.Q))
        print ("Sigma is " + str(self.S))
        print ("Gamma is " + str(self.G))

# The function below takes a PDA as input
# The input PDA is accepted by empty stack
def convert(pda1):
    #Preparing the matrix for the new PDA
    #copying every element from the given PDA
    M2 = pda1.M[:]

    #Making the given PDA matrix's dimensions local
    n1 = pda1.n
    m1 = pda1.m
    #Setting up new PDA matrix's dimensions
    #Rows n will be incremented by 2 which includes a start state and a final state
    #Columns will be incrimented by 1 which includes new transition of X0
    n2 = pda1.n + 2
    m2 = pda1.m + 1

    #Below two operations are performed by keeping the columns constant
    #Inserting a row to the top of the matrix
    M2.insert(0, [None]*pda1.m)
    #Inserting a row at the bottom of the matrix
    M2.append([None]*pda1.m)

    t_str = "q"+str(n2-1)
    for i in M2:
        i.append([t_str, "e"])
    M2[0][m1] = ["q1", "z0x0"]
    M2[n2-1][m1] = None
    #Matrix M2 is ready

    #Preparing the states Q2 for new PDA
    Q1 = pda1.Q
    Q2_length = len(Q1) + 2
    Q2 = set()

    for i in range(Q2_length):
        z = "q" + str(i)
        Q2.add(z)
    #Set of states Q2 for new PDA is ready

    #Preparing the set of input alphabets Sigma S2 for the new PDA
    S2 = pda1.S
    #Set of input alphabets S2 is ready

    #Preparing the set of stack alphabets Gamma G2 for the new PDA
    G2 = pda1.G.union({"X0"})
    #Set of stack alphabets G2 is ready

    q0 = "q0"
    z0 = "x0"
    F2 = {"q"+str(n2-1)}

    #Generating column list
    _column_list = defaultdict()
    for key in pda1._column_list:
        _column_list[key] = pda1._column_list[key]

    _column_list["e,x0"] = n1
    pda2 = PDA(Q = Q2, S = S2, G = G2, q0 = q0, z0 = z0, M = M2, F = F2, _column_list = _column_list)
    #Initialising the column list's last element to the last column number - 1

    return pda2
    #This function ends by returning a new PDA which accepts by final state

#The input_PDA function is the function which takes no argument
#This  returns a PDA which accepts by empty stack
def input_PDA_empty_stack():
    print ("Enter a PDA which accepts by final states")

    #First preparing the number of states n for the PDA which accepts by final state
    print ("Enter the total number of states:", end = " ")
    n = int(input())
    print ("States will be auto generated")
    #The states will be autogenerated as q0, q1, q2, ........
    Q = set()
    #Below for loop generates the states S
    for i in range(n):
        z_str = "q"+str(i)
        Q.add(z_str)
    print ("The states of the PDA are:")
    print (Q)

    #The below code snippet prepares for set of input alphabets Sigma
    S = set()
    print ("Enter the total number of input alphabets:", end = " ")
    #The n_Sigma variable is used to store total number of states
    n_Sigma = int(input())
    print ("Enter the input alphabets:")
    #The below for loop takes Sigma as input
    for i in range (n_Sigma):
        S.add(str(input()))

    #The below code snippet prepares for set of stack alphabets Gamma
    G = set()
    print ("Enter the total number of stack alphabets:", end = " ")
    #The n_Gamma is used to store the total number of stack alphabets
    n_Gamma = int(input())
    print ("Enter the stack alphabets")
    print ("z0, will be automatically added to set whether you enter or not")
    #The below for loop takes Gamma as input
    for i in range(n_Gamma):
        G.add(str(input()))
    #Adding the initial stack symbol of stack
    G.add("z0")

    #Initialising starting state and the start stack symbol
    q0 = "q0"
    z0 = "z0"

    #Creating an instance of the PDA with the above reffered value
    pda1 = PDA(Q = Q, S = S, G = G, q0 = q0, z0 = z0)
    #calling input_delta to fill the Matrix M
    pda1.input_delta()
    return pda1
    #This function returns the PDA which accepts by empty stack

#The function main() is the driver function
#This controls the flow of the program
def main():
    #creates an instance of a PDA class by calling input_PDA_empty_stack function
    #pda1 is the variable which is a PDA accepting by empty stack
    pda1 = input_PDA_empty_stack()
    print ("\nThe given PDA is")
    pda1.print_PDA()
    pda2 = convert(pda1)
    print ("\nThe converted PDA is")
    pda2.print_PDA()

if __name__ == "__main__":
    main()


"""
Train the word2vec embeddings
"""
import gensim, logging
from gensim.test.utils import common_texts, get_tmpfile
from gensim.models import Word2Vec
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

class MySentences(object):
    def __init__(self, fname, N):
        self.fname = fname
        self.N = N

    def __iter__(self):
        with open(self.fname, 'rb') as f:
            for i,line in enumerate(f):
                if i%N==0:
                    yield gensim.utils.simple_preprocess(line)

if __name__=='__main__':

    _inpt = ""
    _size = 80
    _window = 10
    N = 10

    input_file = "dataset/wiki_en"+_inpt+".txt"
    sentences = MySentences(input_file, N)

    model = Word2Vec(
        sentences,
        size=_size,
        window=_window,
        min_count=5,
        workers=10)
    """ Discussion on parameter 'iter'
        有些人會調iter=? (也就是EPOCH) default是5、大一點會比較準，可是:
        1. Word embeddings準，和用起來好不好常常 沒 有 關係
        2. 調幾次就要跑幾遍(dramatically increase the running time)
        => 有多餘時間再試試別的iter
    """
    model.save("word2vec{0}_s{1}_w{2}.model".format(_inpt,_size,_window))


from django.urls import path

from .views import (ListBySearchView, ListProductsView, ListRelatedView,
                    ListSearchView, ReadProductView)

urlpatterns = [
    path('product/<productId>', ReadProductView.as_view()),
    path('get-products', ListProductsView.as_view()),
    path('search', ListSearchView.as_view()),
    path('related/<productId>', ListRelatedView.as_view()),
    path('by/search', ListBySearchView.as_view()),
]


# Download stopwords from NLTK
import nltk

nltk.download('stopwords')

from nltk.corpus import stopwords
import numpy as np
import string
import re

stop = stopwords.words('english')

# Remove stopwords
def rmStopwords(text):
    try:
        result = text.lower().split()
        result = ' '.join(word for word in result if word not in stop)
        return result
    except: return ''

# Remove normal and non-ASCII punctuation
def cleanText(text):
    try:
        result = text.lower()
        result = re.sub(r'[^\x00-\x7F]+', ' ', result)
        result = result.translate(str.maketrans('', '', string.punctuation))
        return result
    except: return ''

# Remove empty lines and nans
def rmBadLines(data, column):
    data = data[data[column] != '']
    data = data[data[column] != ' ']
    data = data[data[column] != np.nan]
    data.drop_duplicates(inplace=True)
    data.reset_index(drop=True, inplace=True)
    return data

import pandas as pd
import pyodbc

#from procFuncs import *

#----- Define path to save data
data_dir = '/Users/kdegrave/Documents/Projects/Search/data/'

#----- Get specialty, condition, procedure, facility, and provider data
con1 = pyodbc.connect(driver='{SQL Server}', server='HGPRODMDMDB01', database='HealthMaster')

q_spec = "SELECT refSpecialtyDescription, refSpecialistDescription, refSpecialistsDescription from HealthMaster.ref.Specialty"
q_cond = "SELECT refConditionDescription from HealthMaster.ref.Condition"
q_faci = "SELECT refFacilityName from HealthMaster.ref.Facility"
q_proc = "SELECT refProcedureMedicalDescription, refProcedureMedicalDescriptions from HealthMaster.ref.ProcedureMedical"

df_spec = pd.read_sql(q_spec, con1)
df_faci = pd.read_sql(q_faci, con1)
df_proc = pd.read_sql(q_proc, con1)
df_cond = pd.read_sql(q_cond, con1)

con2 = pyodbc.connect(driver='{SQL Server}', server='HGPRODBIPROC01', database='ODS1')

q_prov = "SELECT FirstName, MiddleName, LastName from ODS1.Base.Provider"

df_prov = pd.read_sql(q_prov, con2)

#----- Apply functions to clean text
df_spec = df_spec.melt().drop('variable', axis=1)
df_spec.columns = ['refSpecialty']

df_proc = df_proc.melt().drop('variable', axis=1)
df_proc.columns = ['refProcedure']

df_spec['refSpecialty'] = df_spec['refSpecialty'].apply(cleanText).apply(rmStopwords)
df_proc['refProcedure'] = df_proc['refProcedure'].apply(cleanText).apply(rmStopwords)

df_faci['refFacilityName'] = df_faci['refFacilityName'].apply(cleanText).apply(rmStopwords)
df_faci.columns = ['refFacility']

df_cond['refConditionDescription'] = df_cond['refConditionDescription'].apply(cleanText).apply(rmStopwords)
df_cond.columns = ['refCondition']

df_cond = rmBadLines(df_cond, 'refCondition')
df_spec = rmBadLines(df_spec, 'refSpecialty')
df_proc = rmBadLines(df_proc, 'refProcedure')
df_faci = rmBadLines(df_faci, 'refFacility')

#----- Combine provider firstname, middlename, and lastname
df_prov.fillna(' ', inplace=True)

df_prov['refProvider'] = df_prov['FirstName'] + ' ' + df_prov['MiddleName'] + ' ' + df_prov['LastName']
df_prov = df_prov[['refProvider']]
df_prov = rmBadLines(df_prov, 'refProvider')

#----- Remove duplicate entries from facilities, then conditions, then procedures
lcond = df_cond['refCondition'].tolist()
lspec = df_spec['refSpecialty'].tolist()
lproc = df_proc['refProcedure'].tolist()
lfaci = df_faci['refFacility'].tolist()
lprov = df_prov['refProvider'].tolist()

m = lprov + lproc + lcond + lspec
df_faci = pd.DataFrame()
df_faci['refFacility'] = np.setdiff1d(lfaci, m)

m = lprov + lfaci + lproc + lspec
df_cond = pd.DataFrame()
df_cond['refCondition'] = np.setdiff1d(lcond, m)

m = lprov + lfaci + lcond + lspec
df_proc = pd.DataFrame()
df_proc['refProcedure'] = np.setdiff1d(lproc, m)

df_proc = df_proc.drop_duplicates().reset_index(drop=True)
df_spec = df_spec.drop_duplicates().reset_index(drop=True)
df_cond = df_cond.drop_duplicates().reset_index(drop=True)
df_faci = df_faci.drop_duplicates().reset_index(drop=True)
df_prov = df_prov.drop_duplicates().reset_index(drop=True)

#----- Save data
df_prov.to_csv(data_dir + 'provider_gazette_cleaned.csv', index=False)
df_spec.to_csv(data_dir + 'specialty_gazette_cleaned.csv', index=False)
df_cond.to_csv(data_dir + 'condition_gazette_cleaned.csv', index=False)
df_proc.to_csv(data_dir + 'procedure_gazette_cleaned.csv', index=False)
df_faci.to_csv(data_dir + 'facility_gazette_cleaned.csv', index=False)


from flaskr.fetch import index
import os
from flask_fontawesome import FontAwesome
from flask import Flask
from flask import (
    Blueprint, flash, g, redirect, render_template, request, session, url_for
)
from flask_pymongo import PyMongo
from flask_bootstrap import Bootstrap
from flaskr.auth import login_required
# search import
import json
from flaskr.search.index import Index
from flaskr.search.timing import timing
import requests
import os.path
from flaskr.search.documents import Abstract
import time
from lxml import etree
import gzip
from flaskr.db import get_db
from math import ceil
# -------------------------------------

def load_documents():
    start = time.time()
    db = get_db()
    jobs = db.jobs_info.find()
    jobs = list(jobs)
    print(len(jobs))
    doc_id = 0
    for job in jobs:
        # print(doc_id)
        if(job['cleanTitle']):
            title = job['cleanTitle']
        else:
            title = " "
        # url = job['url']
        url = job['url']
        address = job['address']
        company = job['company']
        maxSalary = job['maxSalary']
        yield Abstract(ID=doc_id, title=title, url=url, address=address, company=company, maxSalary=maxSalary)
        doc_id += 1

    end = time.time()
    print(f'Parsing XML took {end - start} seconds')

def index_documents(documents, index):
    for i, document in enumerate(documents):
        index.index_document(document)
        # if i % 50 == 0:
        #     print(f'Indexed {i} documents', end='\r')
    return index

def create_app(test_config=None):
    # create and configure the app
    app = Flask(__name__, instance_relative_config=True)
    app.config.from_mapping(
        SECRET_KEY='dev'
    )
    app.config["MONGO_URI"] = "mongodb://localhost:27017/data_intergration"
    Bootstrap(app)
    mongo = PyMongo(app)
    app.db = mongo.db

    if test_config is None:
        # load the instance config, if it exists, when not testing
        app.config.from_pyfile('config.py', silent=True)
    else:
        # load the test config if passed in
        app.config.from_mapping(test_config)

    # ensure the instance folder exists
    try:
        os.makedirs(app.instance_path)
    except OSError:
        pass

    # Register blueprint
    from . import salaryPredict
    app.register_blueprint(salaryPredict.bp)
    from . import charts
    app.register_blueprint(charts.bp)
    from . import auth
    app.register_blueprint(auth.bp)
    from . import fetch
    app.register_blueprint(fetch.bp)
    app.add_url_rule('/', endpoint='index')
    app.add_url_rule('/home', endpoint='home')
    from . import search1
    app.register_blueprint(search1.bp)
    # a simple page that says hello

    with app.app_context():
        index = index_documents(load_documents(), Index())
        print(f'Index contains {len(index.documents)} documents')

    # @app.route('/home', methods=('GET', 'POST'))
    # def home():
    #     if request.method == 'GET':
    #         text = request.args.get('search_text')
    #         jobs = index.search(text, search_type='AND')
    #         return render_template("jobs/home.html", jobs=jobs, page=1)

    @app.route("/home", methods=('GET', 'POST'))
    @login_required
    def home():
        global db
        page_num = 1
        db = get_db()
        text_s = ""
        jobs = db.jobs_info.find()
        if request.args.get('page_num'):
            page_num = int(request.args.get('page_num'))
        if request.args.get('search_text'):
            text = request.args.get('search_text')
            text_s = request.args.get('search_text')
            if(text != None):
                jobs = index.search(text, search_type='AND')

        jobs = list(jobs)
        jobs = jobs[(page_num-1)*20:page_num*20]
        jobs_count = db.jobs_info.count()
        page_count = ceil(jobs_count/20)
        page = f"{page_num}/{page_count}"
        return render_template("jobs/home.html", jobs=jobs, page=page, jobs_count=jobs_count, text_s=text_s, page_count=page_count)
        # @app.route('/')
        # def home_page():
        #     jobs_info = mongo.db.jobs_info
        #     cur = jobs_info.find({
        #         "address": {"$regex": "Hoàng Quốc Việt+"}
        #     })
        #     job_list = []
        #     for job in cur:
        #         job_list.append(job)
        #     return "\n".join(str(job_list))

    return app


import os
import nltk
from Stemmer import *
from Lemmatizer import *
from Normalizer import *
import math
import operator
import numpy as np
import collections

class Appearance:

    def __init__(self, docId, frequency):
        self.docId = docId
        self.frequency = frequency

    def __repr__(self):
        return str(self.__dict__)

from heapq import heappop, heappush

def heap_sort(array):
    heap = []
    for element in array:
        heappush(heap, element)

    ordered = []

    while heap:
        ordered.append(heappop(heap))

    return ordered

def read_data(path):
    contents = []
    for filename in os.listdir(path):
        data = open(path + '/' + filename, 'r', encoding='utf8').read()
        filename = re.sub('\D', "", filename)
        contents.append({
            'docId': (int(filename)),
            'text': data
        })
    return contents

def tokenize_and_remove_punctuations(s):
    single_chars = ['»', '.', '«', '،', '؟', '"', '#', ')', '(', '*', ',', '-', '/', ':', '[', ']', '،', '?', '…', '۰',
                    '۱', '۲', '۳', '۴', '۵', '۶', '۷', '۸', '۹', '-', '+', '=', '@', '$', '$', '%', '^', '&', '-', '_',
                    '{', '}', '\'', '/', ';', '  ', '>', '<', '•', '٪', '؛']
    text = s.strip(u'\xa0\u200c \t\n\r\f\v')
    text = text.replace('\n', ' ')
    text = Normalizer().normalize(text)
    # print(text)
    for sc in single_chars:
        if text.__contains__(sc):
            text = text.replace(sc, ' ')
    terms = text.split(' ')
    for term in terms:
        if term.__eq__(''):
            terms.remove(term)
        if term.isnumeric():
            terms.remove(term)
    return terms

def get_stopwords():
    stop_words = [word for word in open('stopwords.txt', 'r', encoding='utf8').read().split('\n')]
    return stop_words

def stem_words(tokens):
    stemmer = Stemmer()
    stemmed_words = [stemmer.stem(token) for token in tokens]
    return stemmed_words

def lem_words(tokens):
    lemmetizer = Lemmatizer()
    stemmed_words = [lemmetizer.lemmatize(token) for token in tokens]
    return stemmed_words

def remove_stop_words(tokens):
    stop_words = get_stopwords()
    filtered_words = [token for token in tokens if token not in stop_words and len(token) > 2]
    return filtered_words

def preprocess_data(contents):
    dataDict = {}
    for content in contents:
        tokens = tokenize_and_remove_punctuations(content['text'])
        filtered_tokens = remove_stop_words(tokens)
        stemmed_tokens = stem_words(filtered_tokens)
        lem_tokens = lem_words(filtered_tokens)
        filtered_tokens1 = remove_stop_words(lem_tokens)
        # dataDict[content['docId']] = lem_tokens
        dataDict[content['docId']] = filtered_tokens1
        # dataDict[content['docId']] = filtered_tokens
        # dataDict[content['docId']] = tokens
        # dataDict[content['docId']] = stemmed_tokens
    return dataDict

def preprocess_queries(input_query):
    queriesDict = {}
    queries = input_query.split(' ')
    i = 1
    for query in queries:
        tokens = tokenize_and_remove_punctuations(query)
        filtered_tokens = remove_stop_words(tokens)
        stemmed_tokens = stem_words(filtered_tokens)
        lem_tokens = lem_words(filtered_tokens)
        filtered_tokens1 = remove_stop_words(lem_tokens)
        queriesDict[i] = filtered_tokens1
        i += 1
    return queriesDict

def get_vocabulary(data):
    tokens = []
    for token_list in data.values():
        tokens = tokens + token_list
    fdist = nltk.FreqDist(tokens)
    return list(fdist.keys())

def generate_inverted_index(data):
    all_words = get_vocabulary(data)
    index = {}
    for word in all_words:
        for doc, tokens in data.items():
            if word in tokens:
                if word in index.keys():
                    index[word].append(doc)
                else:
                    index[word] = [doc]
    return index

# def create_inverted_index(data):
#     words = {word: [] for word in np.unique(np.concatenate(list(data.values())))}
#     dicts_keys = data.keys()
#     for key in dicts_keys:
#         u_vocab, counts = np.unique(data[key], return_counts=True)
#         for i in range(len(u_vocab)):
#             words[u_vocab[i]].append({'docID': key, 'frequency': counts[i]})
#     od = collections.OrderedDict(sorted(words.items()))
#     return od

def calculate_tf(tokens):
    tf_score = {}
    for token in tokens:
        tf_score[token] = tokens.count(token)
    return tf_score

def calculate_idf(data):
    idf_score = {}
    N = len(data)
    all_words = get_vocabulary(data)
    for word in all_words:
        word_count = 0
        for token_list in data.values():
            if word in token_list:
                word_count += 1
        idf_score[word] = math.log10(N / word_count)
    return idf_score

def calculate_tfidf(data, idf_score):
    scores = {}
    for key, value in data.items():
        scores[key] = calculate_tf(value)
    for doc, tf_scores in scores.items():
        for token, score in tf_scores.items():
            tf = score
            idf = idf_score[token]
            tf_scores[token] = (1 + math.log10(tf)) * idf
    return scores

def calculate_tfidf_queries(queries, idf_score):
    scores = {}
    for key, value in queries.items():
        scores[key] = calculate_tf(value)
    for key, tf_scores in scores.items():
        for token, score in tf_scores.items():
            idf = 0
            tf = score
            if token in idf_score.keys():
                idf = idf_score[token]
            tf_scores[token] = (1 + math.log10(tf)) * idf
    return scores
lemmatizer = Lemmatizer()
normalizer = Normalizer()
print(normalizer.normalize('می روم'))
print(lemmatizer.lemmatize(normalizer.normalize("می روم")))
# stemmer = Stemmer()
# print(stemmer.stem(normalizer.normalize("رفته بودم")))
# data = read_data("docs")
# preprocessed_data = preprocess_data(data)
#
# inverted_index = generate_inverted_index(preprocessed_data)
# print(inverted_index)
# # inverted_index = create_inverted_index(preprocessed_data)
# query = "لیگ قهرمانان آسیا"
# queries = preprocess_queries(query)
# print(inverted_index)
# idf_scores = calculate_idf(preprocessed_data)
# data_scores = calculate_tfidf(preprocessed_data, idf_scores)
# query_scores = calculate_tfidf_queries(queries, idf_scores)
# # print(query_scores)
# # print(data_scores)
#
#
# query_docs = {}
# for key, value in queries.items():
#     doc_sim = {}
#     for term in value:
#         if term in inverted_index.keys():
#             docs = inverted_index[term]
#             # print(docs)
#             for doc in docs:
#                 doc_score = data_scores[doc][term]
#                 doc_length = math.sqrt(sum(x ** 2 for x in data_scores[doc].values()))
#                 query_score = query_scores[key][term]
#                 query_length = math.sqrt(sum(x ** 2 for x in query_scores[key].values()))
#                 cosine_sim = (doc_score * query_score) / (doc_length * query_length)
#                 if doc in doc_sim.keys():
#                     doc_sim[doc] += cosine_sim
#                 else:
#                     doc_sim[doc] = cosine_sim
#     ranked = sorted(doc_sim.items(), key=operator.itemgetter(1), reverse=True)
    # print(ranked)
    # ranked = heap_sort(doc_sim.items())
    # print(ranked)
    # query_docs[key] = ranked

# print(query_docs)


import discord
import asyncio
import random
import time
import importlib

import commands

client = discord.Client()

@client.event
async def on_ready():
    print("Logged in as:")
    print(client.user.name)
    print(client.user.id)
    print("======")

@client.event
async def on_message(message):
    if message.content.startswith(commands.PREFIX):
        command = message.content.split(" ")
        command[0] = command[0][len(commands.PREFIX):]
        if command[0] == "reload":
            importlib.reload(commands)
            await client.send_message(message.channel, "Reloaded!")
        else:
            for i in commands.COMMANDS:
                if i[0] == command[0]:
                    try:
                        await i[1](client, message, *command[1:])
                    except Exception as e:
                        await client.send_message(message.channel, "```" + e.__repr__() + "```")

client.run(open("bot-token.txt").read())


import os, sys, io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')

file = open("cooccurrences.bin","rb")
lines = file.read()

s = lines.decode(errors="ignore")
print(s)
# print(lines)

file.close()

NLP/word embedding comparison/glove program/read_bin.py


"""
ContractShipping Canada Post API
https://www.canadapost.ca/cpo/mc/business/productsservices/developers/services/shippingmanifest/default.jsf
"""
import logging
from lxml import etree
import requests
from canada_post.service import ServiceBase, CallLinkService
from canada_post.util import InfoObject

class Shipment(InfoObject):
    """
    Shipment class, is the return value of the CreateShipment service.
    It contains
      * tracking pin
      * return tracking pin
      * [shipment ]id
      * [shipment ]status
      * links is a dict of string --> dict where the keys are the rel
         attribute of each link (see the CreateShipment docs in the canadapost
         site. See the CreateShipment docstring for a link) and the keys have an
         'href' which is the link, the same 'rel' value and some other
         attributes, depending on each link
    """
    def __init__(self, xml=None, **kwargs):
        if xml is not None:
            self._from_xml(xml)
        super(Shipment, self).__init__(**kwargs)

    def _from_xml(self, xml):
        # I do this this way because I can't expect all return codes to have all
        #  values, I just fill in every Simple element in self
        for child in xml.getchildren():
            if child.tag == "shipment-id":
                self.id = child.text
            elif child.tag == "shipment-status":
                self.status = child.text
            elif child.tag == "links":
                # I can't make these into InfoObjects because there's a `self`
                #  rel object
                self.links = dict((link['rel'], link)
                    for link in map(lambda l: dict(l.attrib),
                                    child.findall("link")))
            else:
                # expected "tracking-pin" "return-tracking-pin"
                attrname = child.tag.replace("-", "_")
                setattr(self, attrname, child.text)

class CreateShipment(ServiceBase):
    """
    CreateShipment Canada Post API (for ContractShipping)
    https://www.canadapost.ca/cpo/mc/business/productsservices/developers/services/shippingmanifest/createshipment.jsf
    """
    URL ="https://{server}/rs/{customer}/{mobo}/shipment"
    log = logging.getLogger('canada_post.service.contract_shipping'
                            '.CreateShipment')
    def __init__(self, auth, url=None):
        if url:
            self.URL = url
        super(CreateShipment, self).__init__(auth)

    def set_link(self, url):
        """
        Sets the link in case you want to set it after initialization
        """
        self.URL = url

    def get_url(self):
        return self.URL.format(server=self.get_server(),
                               customer=self.auth.customer_number,
                               mobo=self.auth.customer_number)

    def _create_address_detail(self, parent, address, add_child):
        """
        Creates the address detail part of the request. You need to do any
        checks outside of this call
        """
        addr_detail = add_child("address-details", parent)
        add_child("address-line-1", addr_detail).text = address.address1
        add_child("address-line-2", addr_detail).text = address.address2
        add_child("city", addr_detail).text = address.city
        if address.province:
            add_child("prov-state", addr_detail).text = address.province
        add_child("country-code", addr_detail).text = address.country_code
        if address.postal_code:
            add_child("postal-zip-code",
                      addr_detail).text = unicode(address.postal_code)
        else:
            assert address.country_code not in ("US", "CA"), (
                "Addresses within {} require "
                "a postal code").format(address.country_code)

        return addr_detail

    def __call__(self, parcel, origin, destination, service, group):
        """
        Create a shipping order for the given parcels

        parcel: must be a canada_post.util.parcel.Parcel
        origin: must be a canada_post.util.address.Origin instance
        destination: must be a canada_post.util.address.Destination instance
        service: must be a canada_post.service.Service instance with at least
            the code parameter set up
        group: must be a string or unicode defining the parcel group that this
            parcel should be added to
        """
        debug = "( DEBUG )" if self.auth.debug else ""
        self.log.info(("Create shipping for parcel %s, from %s to %s{debug}"
                       .format(debug=debug)), parcel, origin, destination)

        # shipment
        shipment = etree.Element(
            "shipment", xmlns="http://www.canadapost.ca/ws/shipment")
        def add_child(child_name, parent=shipment):
            return etree.SubElement(parent, child_name)
        add_child("group-id").text = group
        add_child("requested-shipping-point").text = unicode(origin.postal_code)
        delivery_spec = add_child("delivery-spec")
        add_child("service-code", delivery_spec).text = service.code

        # sender
        sender = add_child("sender", delivery_spec)
        if origin.name:
            add_child("name", sender).text = origin.name
        assert origin.company, ("The sender needs a company name for "
                                "Contract Shipping service")
        add_child("company", sender).text = origin.company
        assert origin.phone, ("The sender needs a phone for "
                              "Contact Shipping Service")
        add_child("contact-phone", sender).text = origin.phone

        #destination details, first assertions
        assert any((origin.address1, origin.address2)), (
            "The sender needs an address to Create Shipping")
        assert origin.city, "Need the sender's city to Create Shipping"
        assert origin.province, "Need the sender's province to Create Shipping"
        sender_details = self._create_address_detail(sender, origin, add_child)
        # done sender

        # destination
        dest = add_child("destination", delivery_spec)
        if destination.name:
            add_child("name", dest).text = destination.name
        else:
            # TODO: if the Deliver to Post Office option is used, the name
            #  element must be present for the destination.
            pass
        if destination.company:
            add_child("company", dest).text = destination.company
        if destination.extra:
            add_child("additional-address-info").text = destination.extra
        if destination.phone:
            add_child("client-voice-number", dest).text = destination.phone
        else:
            # Required for destination when service is one of
            #  Expedited Parcel-USA, Xpresspost-USA, Xpresspost-International,
            #  Priority Worldwide Parcel or Pak
            #  (USA.EP, USA.XP, INT.XP, USA.PW.PARCEL, USA.PW.PAK,
            #   INT.PW.PARCEL, INT.PW.PAK)
            if service.code in ("USA.EP", "USA.XP", "INT.XP", "USA.PW.PARCEL",
                                "USA.PW.PAK", "INT.PW.PARCEL", "INT.PK.PAK"):
                assert False, ("Service {code} requires destination to have a "
                               "phone number").format(code=service.code)

        # destination details, first assertions
        assert any((destination.address1, destination.address2)), (
            "Must have address to Create Shipping")
        if not destination.province:
            assert destination.country_code not in ("CA", "US"), (
                "Country code is required for international shippings")
        # and then creation
        dest_details = self._create_address_detail(dest, destination, add_child)
        # done destination

        # TODO: options
        #options = add_child("options", delivery_spec)

        # parcel
        parcel_chars = add_child("parcel-characteristics", delivery_spec)
        add_child("weight", parcel_chars).text = unicode(parcel.weight)
        if all((parcel.length > 0, parcel.width > 0, parcel.height > 0)):
            dims = add_child("dimensions", parcel_chars)
            add_child("length", dims).text = unicode(parcel.length)
            add_child("width", dims).text = unicode(parcel.width)
            add_child("height", dims).text = unicode(parcel.height)
        add_child("unpackaged", parcel_chars).text = ("true"
                                                      if parcel.unpackaged
                                                      else "false")

        # TODO: notification
        #notification = add_child("notification", delivery_spec)

        # TODO: print-preferences
        #print_preferences = add_child("print-preferences", delivery_spec)

        # preferences
        preferences = add_child("preferences", delivery_spec)
        add_child("show-packing-instructions", preferences).text = "false"
        # TODO: these two are actually optional (may be "false") for CA
        # shippings
        add_child("show-postage-rate", preferences).text = "false"
        add_child("show-insured-value", preferences).text = "false"

        # settlement-info
        settlement = add_child("settlement-info", delivery_spec)
        # TODO: set paid-by-customer if a different customer is paying for this
        #add_child("paid-by-customer", settlement).text = FOOBAR
        assert self.auth.contract_number, ("Must have a contract number for "
                                           "contract shipping")
        add_child("contract-id", settlement).text = self.auth.contract_number
        # TODO: can be CreditCard as well
        add_child("intended-method-of-payment", settlement).text = "Account"

        headers = {
            'Accept': "application/vnd.cpc.shipment-v2+xml",
            'Content-type': "application/vnd.cpc.shipment-v2+xml",
            'Accept-language': "en-CA",
        }
        url = self.get_url()
        self.log.info("Using url %s", url)
        request = etree.tostring(shipment, pretty_print=self.auth.debug)
        self.log.debug("Request xml: %s", request)
        response = requests.post(url=url, data=request, headers=headers,
                                 auth=self.userpass())
        self.log.info("Request returned with status %s", response.status_code)
        self.log.debug("Request returned content: %s", response.content)

        if not response.ok:
            response.raise_for_status()

        # this is a hack to remove the namespace from the response, since this
        #breaks xpath lookup in lxml
        restree = etree.XML(response.content.replace(' xmlns="',
                                                     ' xmlnamespace="'))
        return Shipment(xml=restree)

class VoidShipment(CallLinkService):
    """
    Cancel a Contract Shipping created Shipment created by CreateShipment
    """
    log = logging.getLogger("canada_post.service.contract_shipping"
                            ".VoidShipment")
    link_rel = 'self'


# Copyright 2017 Bracket Computing, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# https://github.com/brkt/brkt-cli/blob/master/LICENSE
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and
# limitations under the License.

"""Utilities for discovering metavisor version"""

import logging
import re
from distutils.version import LooseVersion

import boto3
from botocore.handlers import disable_signing

log = logging.getLogger(__name__)
logging.getLogger('boto3').setLevel(logging.FATAL)
logging.getLogger('botocore').setLevel(logging.FATAL)
logging.getLogger('s3transfer').setLevel(logging.FATAL)

AWS_PROD_BUCKET_DEFAULT='solo-brkt-prod-net'
RE_METAVISOR_VERSION=re.compile(r'(metavisor)-(\d+)-(\d+)-(\d+)-(g.*?)(-.*)?$')
RE_MAJOR_VERSION_ARG=re.compile(r'^(\d+)$')
RE_MINOR_VERSION_ARG=re.compile(r'^(\d+)-(\d+)$')
RE_PATCH_VERSION_ARG=re.compile(r'^(\d+)-(\d+)-(\d+)$')

class MetavisorVersionNotFoundError(Exception):
    pass

def get_s3_versions(bucket):
    """Return available metavisor versions in S3 bucket

    Return a list of possible metavisor versions from provided AWS bucket.

    Args:
       bucket: AWS bucket where metavisor versions are pubished (str)

    Returns:
       versions: List of metavisor version prefix's for AWS bucket (list)

    """
    log.debug('Fetching Metavisor version from S3')
    versions = []

    # Check for dev bucket
    if bucket == 'packages.int.brkt.net':
        version_prefix = 'metavisor/metavisor-'
    else:
        version_prefix = 'metavisor-'

    s3 = boto3.resource('s3')
    s3.meta.client.meta.events.register('choose-signer.s3.*', disable_signing)
    paginator = s3.meta.client.get_paginator('list_objects')
    page_iterator = paginator.paginate(Bucket=bucket,
                                       Delimiter='/',
                                       Prefix=version_prefix)
    for page in page_iterator:
        if page.get('CommonPrefixes'):
            versions += \
                [ v.get('Prefix').rstrip('/') for v in page.get('CommonPrefixes') ]

    return versions

def get_version(version, bucket):
    """Return a single published metavisor version

    Returns the latest version of metavisor if no specific version is
    provided.

    Args:
       version: Semantic or exact match metavisor version (str)
       bucket: AWS bucket where metavisor versions are pubished (str)

    Returns:
       mversion: Metavisor version prefix for AWS bucket (str)

    """
    mversion = None
    mv_regex = version_regex(version)

    versions = get_s3_versions(bucket)
    mversions = sorted([LooseVersion(v) for v in versions], reverse=True)
    for mv in mversions:
        vcandidate = re.search(mv_regex, mv.vstring)
        if vcandidate:
            # Return exact version matches (e.g. -debug builds)
            if mv_regex.pattern.rstrip('$') == version:
                mversion = mv.vstring
                break
            # Return only official metavisor versions (e.g. no suffix)
            # ('metavisor', '1', '3', '106', 'g0cbdb5597', None)
            mvsearch = re.search(RE_METAVISOR_VERSION, mv.vstring)
            if mvsearch.groups()[-1] == None:
                mversion = mv.vstring
                break

    if not mversion:
        log.exception("Exception determining metavisor version in %s (%s)",
                      bucket, mv_regex.pattern)
        raise MetavisorVersionNotFoundError()

    log.info("Found metavisor version %s", mversion)

    return mversion

def get_amis_url(version, bucket):
    """Return url to amis.json for specified metavisor version

    Args:
       version: Semantic or exact match metavisor version (str)
       bucket: AWS bucket where metavisor versions are pubished (str)

    Returns:
       url: Location to amis.json for given metavisor version (str)

    """
    mversion = get_version(version, bucket)
    url = 'http://%(bucket)s.s3.amazonaws.com/%(mversion)s/amis.json' % locals()

    return url

def version_regex(version):
    """Return a metavisor long version regex for a given version string

    Transforms short semantic version strings (e.g. 1, 1.2, 1.2.3) into
    metavisor long form version strings. If no supported semantic version
    string is provided, default to an exact match.

    Args:
        version: Semantic or exact match version (str)

    Returns:
        version_regex: Compiled regex object (_sre.SRE_Pattern)
    """

    if not version:
        return  RE_METAVISOR_VERSION

    # Be lenient with . vs - version input
    version = version.replace('.','-')

    if re.search(RE_PATCH_VERSION_ARG, version):
        vr = r'metavisor-%s-(g.*?)(-.*)?$' % version
    elif re.search(RE_MINOR_VERSION_ARG, version):
        vr = r'metavisor-%s-(\d+)-(g.*?)(-.*)?$' % version
    elif re.search(RE_MAJOR_VERSION_ARG, version):
        vr = r'metavisor-%s-(\d+)-(\d+)-(g.*?)(-.*)?$' % version
    else:
        vr = r'%s$' % version

    return re.compile(vr)


import mock
import sys

from enstaller.machine_cli.__main__ import main
from enstaller.tests.common import exception_code

if sys.version_info[0] == 2:
    import unittest2 as unittest
else:
    import unittest

class TestMain(unittest.TestCase):
    def test_help(self):
        # given
        args = ["--help"]

        # when
        with self.assertRaises(SystemExit) as exc:
            main(args)

        # then
        self.assertEqual(exc.exception.code, 0)

    @mock.patch("enstaller.machine_cli.__main__.install")
    def test_install(self, install_command):
        # given
        args = ["install"]

        # when
        with self.assertRaises(SystemExit) as exc:
            main(args)

        # then
        self.assertEqual(exception_code(exc), 0)
        install_command.assert_called_with()

    @mock.patch("enstaller.machine_cli.__main__.remove")
    def test_remove(self, remove_command):
        # given
        args = ["remove"]

        # when
        with self.assertRaises(SystemExit) as exc:
            main(args)

        # then
        self.assertEqual(exception_code(exc), 0)
        remove_command.assert_called_with()

    @mock.patch("enstaller.machine_cli.__main__.update_all")
    def test_update_all(self, update_all_command):
        # given
        args = ["update_all"]

        # when
        with self.assertRaises(SystemExit) as exc:
            main(args)

        # then
        self.assertEqual(exception_code(exc), 0)
        update_all_command.assert_called_with()