import sys
import os
from datetime import datetime, timedelta, date
from sqlalchemy import create_engine, MetaData, Table, Integer, DateTime, String, Column, select, BigInteger, Numeric
from sqlalchemy.dialects import postgresql
import json
#sys.path.append("<dir/for/slack_notifier/and/or/bq_uploader/if/different>") #Directory for additional custom Classes, CHANGE TO LOCAL DIR where BigQuery/Slack Notifier is stored if seperate directory
from Appsflyer_api import Appsflyer_API_Pull
from bq_uploader import BigQuery_uploader
from slack_notifier import Error_Notifier

class MSTR:

	# This class uses custom classes from the added directory to download reports from Appsflyer and upload them to BigQuery

	def __init__(self):

		self.AF = Appsflyer_API_Pull() # Sets Appsflyer API object
		self.BQ = BigQuery_uploader() # Sets BigQuery connection

		self.apps = ["<list Appsflyer app ids here>", "<list Appsflyer app ids here>"]
		self.reports = ["installs_report", "in_app_events_report","organic_in_app_events_report", "invalid_installs_report", "invalid_in_app_events_report"] # all available reports, pick and choose accordingly
		self.start_time = "00:00"
		self.end_time = "23:59"
		self.yesterday = datetime.strftime(datetime.now() - timedelta(1), '%Y-%m-%d') # Creates datetime object for the day before date being run
		self.today = str(date.today())

		# Set parameters if custom date range is needed
		#self.yesterday = "e.g. 2019-09-22"
		#self.today = "e.g. 2019-09-23"

		self.yesterday_start = str(self.yesterday + " " + self.start_time)
		self.yesterday_end = str(self.today + " " + self.start_time)

		self.path = str("<dir/for/report/storage/if/desired>" + self.yesterday) # Creates path for created reports

		self.CONFIG = {}
		self.directory_counter = 1

		try:
			with open(sys.argv[1], "r") as f:
				self.CONFIG = json.load(f)
		except Exception as e:
			print('failed to read or parse config file')
			print(e)
			exit(1)
		if not self.CONFIG['user'] or not self.CONFIG['password'] or not self.CONFIG['postgres_server'] \
			or not self.CONFIG['database']:

			print("missing parameter in config.json. see docs.")
			exit(1)

		self.engine = create_engine('postgresql+psycopg2://' + self.CONFIG['user'] + ':' + self.CONFIG['password'] +
									'@' + self.CONFIG['postgres_server'] + '/' + self.CONFIG['database'])

		self.conn = self.engine.connect()
		self.i = 0

	def main(self):

			try:
				if os.path.exists(self.path):
					for a in self.apps: # loops through all app ids listed in self.apps	(only currently for installs and in_app_events reports)
						if os.path.isfile(self.path + "/" + a + "_installs_report_"+ self.yesterday_start + "_to_" + self.yesterday_end) and not os.path.isfile(self.path + "/" + a + "_in_app_events_report_"+ self.yesterday_start + "_to_" + self.yesterday_end):
							self.reports = ["in_app_events_report"]
						elif not os.path.isfile(self.path + "/" + a + "_installs_report_"+ self.yesterday_start + "_to_" + self.yesterday_end) and os.path.isfile(self.path + "/" + a + "_in_app_events_report_"+ self.yesterday_start + "_to_" + self.yesterday_end):
							self.reports = ["installs_report"]
						elif os.path.isfile(self.path + "/" + a + "_installs_report_"+ self.yesterday_start + "_to_" + self.yesterday_end) and os.path.isfile(self.path + "/" + a + "_in_app_events_report_"+ self.yesterday_start + "_to_" + self.yesterday_end):
							print("All reports for {} for {} have already been uploaded.".format(a, self.yesterday))
							if self.directory_counter == len(self.apps):
								print("All reports for {} have been uploaded. Please choose a different date to transfer.".format(self.yesterday))
								sys.exit(1)
							self.directory_counter += 1
				else:
					os.mkdir(self.path) # Creates directory for each days reports to be stored, named with today's date
					os.chdir(self.path) # Moves current working directory to created directory

				for a in self.apps: # loops through all app ids listed in self.apps
					for r in self.reports: # loops through all reports listed in self.reports
						f = self.AF.main(a, self.yesterday_start, self.yesterday_end, r) # Sends Pull API Request and saves report in a file in created directory
						file = str(self.path + "/" + f) # Gathers file information to direct BigQuery uploader to created report
						to_bq = self.BQ.main('Appsflyer_inbound', r, file, 'CSV') # Uploads report to appropriate table in BigQuery
						self.conn.execute("INSERT INTO <sql table name> (job_type, job_id, date, start_time, end_time, app) VALUES ('" + r + "', '"+ to_bq +"', '"+ self.yesterday +"', '"+ self.start_time + "', '"+ self.end_time +"','"+ a +"');")

			except Exception as e:
				print(e)

			finally:
				self.conn.close()

if __name__=="__main__":
	try:
		Mstr = MSTR().main() # Initializes and runs class MSTR with main method
	except Exception as e:
		message = "Appsflyer Autoupload ERROR: " + str(e)
		error = Error_Notifier.main("error_log", message)


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from hashlib import sha1
import random
import string
import tempfile
import time
from unittest import mock

import requests_mock
from testscenarios import load_tests_apply_scenarios as load_tests  # noqa

from openstack.object_store.v1 import account
from openstack.object_store.v1 import container
from openstack.object_store.v1 import obj
from openstack.tests.unit.cloud import test_object as base_test_object
from openstack.tests.unit import test_proxy_base

class FakeResponse:
    def __init__(self, response, status_code=200, headers=None):
        self.body = response
        self.status_code = status_code
        self.headers = headers if headers else {}

    def json(self):
        return self.body

class TestObjectStoreProxy(test_proxy_base.TestProxyBase):
    kwargs_to_path_args = False

    def setUp(self):
        super(TestObjectStoreProxy, self).setUp()
        self.proxy = self.cloud.object_store
        self.container = self.getUniqueString()
        self.endpoint = self.cloud.object_store.get_endpoint() + '/'
        self.container_endpoint = '{endpoint}{container}'.format(
            endpoint=self.endpoint, container=self.container
        )

    def test_account_metadata_get(self):
        self.verify_head(
            self.proxy.get_account_metadata, account.Account, method_args=[]
        )

    def test_container_metadata_get(self):
        self.verify_head(
            self.proxy.get_container_metadata,
            container.Container,
            method_args=["container"],
        )

    def test_container_delete(self):
        self.verify_delete(
            self.proxy.delete_container, container.Container, False
        )

    def test_container_delete_ignore(self):
        self.verify_delete(
            self.proxy.delete_container, container.Container, True
        )

    def test_container_create_attrs(self):
        self.verify_create(
            self.proxy.create_container,
            container.Container,
            method_args=['container_name'],
            expected_args=[],
            expected_kwargs={'name': 'container_name', "x": 1, "y": 2, "z": 3},
        )

    def test_object_metadata_get(self):
        self._verify(
            "openstack.proxy.Proxy._head",
            self.proxy.get_object_metadata,
            method_args=['object'],
            method_kwargs={'container': 'container'},
            expected_args=[obj.Object, 'object'],
            expected_kwargs={'container': 'container'},
        )

    def _test_object_delete(self, ignore):
        expected_kwargs = {
            "ignore_missing": ignore,
            "container": "name",
        }

        self._verify(
            "openstack.proxy.Proxy._delete",
            self.proxy.delete_object,
            method_args=["resource"],
            method_kwargs=expected_kwargs,
            expected_args=[obj.Object, "resource"],
            expected_kwargs=expected_kwargs,
        )

    def test_object_delete(self):
        self._test_object_delete(False)

    def test_object_delete_ignore(self):
        self._test_object_delete(True)

    def test_object_create_attrs(self):
        kwargs = {
            "name": "test",
            "data": "data",
            "container": "name",
            "metadata": {},
        }

        self._verify(
            "openstack.proxy.Proxy._create",
            self.proxy.upload_object,
            method_kwargs=kwargs,
            expected_args=[obj.Object],
            expected_kwargs=kwargs,
        )

    def test_object_create_no_container(self):
        self.assertRaises(TypeError, self.proxy.upload_object)

    def test_object_get(self):
        with requests_mock.Mocker() as m:
            m.get("%scontainer/object" % self.endpoint, text="data")
            res = self.proxy.get_object("object", container="container")
            self.assertIsNone(res.data)

    def test_object_get_write_file(self):
        with requests_mock.Mocker() as m:
            m.get("%scontainer/object" % self.endpoint, text="data")
            with tempfile.NamedTemporaryFile() as f:
                self.proxy.get_object(
                    "object", container="container", outfile=f.name
                )
                dt = open(f.name).read()
                self.assertEqual(dt, "data")

    def test_object_get_remember_content(self):
        with requests_mock.Mocker() as m:
            m.get("%scontainer/object" % self.endpoint, text="data")
            res = self.proxy.get_object(
                "object", container="container", remember_content=True
            )
            self.assertEqual(res.data, "data")

    def test_set_temp_url_key(self):
        key = 'super-secure-key'

        self.register_uris(
            [
                dict(
                    method='POST',
                    uri=self.endpoint,
                    status_code=204,
                    validate=dict(
                        headers={'x-account-meta-temp-url-key': key}
                    ),
                ),
                dict(
                    method='HEAD',
                    uri=self.endpoint,
                    headers={'x-account-meta-temp-url-key': key},
                ),
            ]
        )
        self.proxy.set_account_temp_url_key(key)
        self.assert_calls()

    def test_set_account_temp_url_key_second(self):
        key = 'super-secure-key'

        self.register_uris(
            [
                dict(
                    method='POST',
                    uri=self.endpoint,
                    status_code=204,
                    validate=dict(
                        headers={'x-account-meta-temp-url-key-2': key}
                    ),
                ),
                dict(
                    method='HEAD',
                    uri=self.endpoint,
                    headers={'x-account-meta-temp-url-key-2': key},
                ),
            ]
        )
        self.proxy.set_account_temp_url_key(key, secondary=True)
        self.assert_calls()

    def test_set_container_temp_url_key(self):
        key = 'super-secure-key'

        self.register_uris(
            [
                dict(
                    method='POST',
                    uri=self.container_endpoint,
                    status_code=204,
                    validate=dict(
                        headers={'x-container-meta-temp-url-key': key}
                    ),
                ),
                dict(
                    method='HEAD',
                    uri=self.container_endpoint,
                    headers={'x-container-meta-temp-url-key': key},
                ),
            ]
        )
        self.proxy.set_container_temp_url_key(self.container, key)
        self.assert_calls()

    def test_set_container_temp_url_key_second(self):
        key = 'super-secure-key'

        self.register_uris(
            [
                dict(
                    method='POST',
                    uri=self.container_endpoint,
                    status_code=204,
                    validate=dict(
                        headers={'x-container-meta-temp-url-key-2': key}
                    ),
                ),
                dict(
                    method='HEAD',
                    uri=self.container_endpoint,
                    headers={'x-container-meta-temp-url-key-2': key},
                ),
            ]
        )
        self.proxy.set_container_temp_url_key(
            self.container, key, secondary=True
        )
        self.assert_calls()

    def test_copy_object(self):
        self.assertRaises(NotImplementedError, self.proxy.copy_object)

    def test_file_segment(self):
        file_size = 4200
        content = ''.join(
            random.choice(string.ascii_uppercase + string.digits)
            for _ in range(file_size)
        ).encode('latin-1')
        self.imagefile = tempfile.NamedTemporaryFile(delete=False)
        self.imagefile.write(content)
        self.imagefile.close()

        segments = self.proxy._get_file_segments(
            endpoint='test_container/test_image',
            filename=self.imagefile.name,
            file_size=file_size,
            segment_size=1000,
        )
        self.assertEqual(len(segments), 5)
        segment_content = b''
        for index, (name, segment) in enumerate(segments.items()):
            self.assertEqual(
                'test_container/test_image/{index:0>6}'.format(index=index),
                name,
            )
            segment_content += segment.read()
        self.assertEqual(content, segment_content)

class TestDownloadObject(base_test_object.BaseTestObject):
    def setUp(self):
        super(TestDownloadObject, self).setUp()
        self.the_data = b'test body'
        self.register_uris(
            [
                dict(
                    method='GET',
                    uri=self.object_endpoint,
                    headers={
                        'Content-Length': str(len(self.the_data)),
                        'Content-Type': 'application/octet-stream',
                        'Accept-Ranges': 'bytes',
                        'Last-Modified': 'Thu, 15 Dec 2016 13:34:14 GMT',
                        'Etag': '"b5c454b44fbd5344793e3fb7e3850768"',
                        'X-Timestamp': '1481808853.65009',
                        'X-Trans-Id': 'tx68c2a2278f0c469bb6de1-005857ed80dfw1',
                        'Date': 'Mon, 19 Dec 2016 14:24:00 GMT',
                        'X-Static-Large-Object': 'True',
                        'X-Object-Meta-Mtime': '1481513709.168512',
                    },
                    content=self.the_data,
                )
            ]
        )

    def test_download(self):
        data = self.cloud.object_store.download_object(
            self.object, container=self.container
        )

        self.assertEqual(data, self.the_data)
        self.assert_calls()

    def test_stream(self):
        chunk_size = 2
        for index, chunk in enumerate(
            self.cloud.object_store.stream_object(
                self.object, container=self.container, chunk_size=chunk_size
            )
        ):
            chunk_len = len(chunk)
            start = index * chunk_size
            end = start + chunk_len
            self.assertLessEqual(chunk_len, chunk_size)
            self.assertEqual(chunk, self.the_data[start:end])
        self.assert_calls()

class TestExtractName(TestObjectStoreProxy):
    scenarios = [
        ('discovery', dict(url='/', parts=['account'])),
        ('endpoints', dict(url='/endpoints', parts=['endpoints'])),
        (
            'container',
            dict(url='/AUTH_123/container_name', parts=['container']),
        ),
        ('object', dict(url='/container_name/object_name', parts=['object'])),
        (
            'object_long',
            dict(
                url='/v1/AUTH_123/cnt/path/deep/object_name', parts=['object']
            ),
        ),
    ]

    def test_extract_name(self):
        results = self.proxy._extract_name(self.url, project_id='123')
        self.assertEqual(self.parts, results)

class TestTempURL(TestObjectStoreProxy):
    expires_iso8601_format = '%Y-%m-%dT%H:%M:%SZ'
    short_expires_iso8601_format = '%Y-%m-%d'
    time_errmsg = (
        'time must either be a whole number or in specific ISO 8601 format.'
    )
    path_errmsg = 'path must be full path to an object e.g. /v1/a/c/o'
    url = '/v1/AUTH_account/c/o'
    seconds = 3600
    key = 'correcthorsebatterystaple'
    method = 'GET'
    expected_url = url + (
        '?temp_url_sig=temp_url_signature&temp_url_expires=1400003600'
    )
    expected_body = '\n'.join(
        [
            method,
            '1400003600',
            url,
        ]
    ).encode('utf-8')

    @mock.patch('hmac.HMAC')
    @mock.patch('time.time', return_value=1400000000)
    def test_generate_temp_url(self, time_mock, hmac_mock):
        hmac_mock().hexdigest.return_value = 'temp_url_signature'
        url = self.proxy.generate_temp_url(
            self.url, self.seconds, self.method, temp_url_key=self.key
        )
        key = self.key
        if not isinstance(key, bytes):
            key = key.encode('utf-8')
        self.assertEqual(url, self.expected_url)
        self.assertEqual(
            hmac_mock.mock_calls,
            [
                mock.call(),
                mock.call(key, self.expected_body, sha1),
                mock.call().hexdigest(),
            ],
        )
        self.assertIsInstance(url, type(self.url))

    @mock.patch('hmac.HMAC')
    @mock.patch('time.time', return_value=1400000000)
    def test_generate_temp_url_ip_range(self, time_mock, hmac_mock):
        hmac_mock().hexdigest.return_value = 'temp_url_signature'
        ip_ranges = [
            '1.2.3.4',
            '1.2.3.4/24',
            '2001:db8::',
            b'1.2.3.4',
            b'1.2.3.4/24',
            b'2001:db8::',
        ]
        path = '/v1/AUTH_account/c/o/'
        expected_url = path + (
            '?temp_url_sig=temp_url_signature'
            '&temp_url_expires=1400003600'
            '&temp_url_ip_range='
        )
        for ip_range in ip_ranges:
            hmac_mock.reset_mock()
            url = self.proxy.generate_temp_url(
                path,
                self.seconds,
                self.method,
                temp_url_key=self.key,
                ip_range=ip_range,
            )
            key = self.key
            if not isinstance(key, bytes):
                key = key.encode('utf-8')

            if isinstance(ip_range, bytes):
                ip_range_expected_url = expected_url + ip_range.decode('utf-8')
                expected_body = '\n'.join(
                    [
                        'ip=' + ip_range.decode('utf-8'),
                        self.method,
                        '1400003600',
                        path,
                    ]
                ).encode('utf-8')
            else:
                ip_range_expected_url = expected_url + ip_range
                expected_body = '\n'.join(
                    [
                        'ip=' + ip_range,
                        self.method,
                        '1400003600',
                        path,
                    ]
                ).encode('utf-8')

            self.assertEqual(url, ip_range_expected_url)

            self.assertEqual(
                hmac_mock.mock_calls,
                [
                    mock.call(key, expected_body, sha1),
                    mock.call().hexdigest(),
                ],
            )
            self.assertIsInstance(url, type(path))

    @mock.patch('hmac.HMAC')
    def test_generate_temp_url_iso8601_argument(self, hmac_mock):
        hmac_mock().hexdigest.return_value = 'temp_url_signature'
        url = self.proxy.generate_temp_url(
            self.url,
            '2014-05-13T17:53:20Z',
            self.method,
            temp_url_key=self.key,
        )
        self.assertEqual(url, self.expected_url)

        # Don't care about absolute arg.
        url = self.proxy.generate_temp_url(
            self.url,
            '2014-05-13T17:53:20Z',
            self.method,
            temp_url_key=self.key,
            absolute=True,
        )
        self.assertEqual(url, self.expected_url)

        lt = time.localtime()
        expires = time.strftime(self.expires_iso8601_format[:-1], lt)

        if not isinstance(self.expected_url, str):
            expected_url = self.expected_url.replace(
                b'1400003600',
                bytes(str(int(time.mktime(lt))), encoding='ascii'),
            )
        else:
            expected_url = self.expected_url.replace(
                '1400003600', str(int(time.mktime(lt)))
            )
        url = self.proxy.generate_temp_url(
            self.url, expires, self.method, temp_url_key=self.key
        )
        self.assertEqual(url, expected_url)

        expires = time.strftime(self.short_expires_iso8601_format, lt)
        lt = time.strptime(expires, self.short_expires_iso8601_format)

        if not isinstance(self.expected_url, str):
            expected_url = self.expected_url.replace(
                b'1400003600',
                bytes(str(int(time.mktime(lt))), encoding='ascii'),
            )
        else:
            expected_url = self.expected_url.replace(
                '1400003600', str(int(time.mktime(lt)))
            )
        url = self.proxy.generate_temp_url(
            self.url, expires, self.method, temp_url_key=self.key
        )
        self.assertEqual(url, expected_url)

    @mock.patch('hmac.HMAC')
    @mock.patch('time.time', return_value=1400000000)
    def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock):
        hmac_mock().hexdigest.return_value = 'temp_url_signature'
        url = self.proxy.generate_temp_url(
            self.url,
            self.seconds,
            self.method,
            temp_url_key=self.key,
            iso8601=True,
        )
        key = self.key
        if not isinstance(key, bytes):
            key = key.encode('utf-8')

        expires = time.strftime(
            self.expires_iso8601_format, time.gmtime(1400003600)
        )
        if not isinstance(self.url, str):
            self.assertTrue(url.endswith(bytes(expires, 'utf-8')))
        else:
            self.assertTrue(url.endswith(expires))
        self.assertEqual(
            hmac_mock.mock_calls,
            [
                mock.call(),
                mock.call(key, self.expected_body, sha1),
                mock.call().hexdigest(),
            ],
        )
        self.assertIsInstance(url, type(self.url))

    @mock.patch('hmac.HMAC')
    @mock.patch('time.time', return_value=1400000000)
    def test_generate_temp_url_prefix(self, time_mock, hmac_mock):
        hmac_mock().hexdigest.return_value = 'temp_url_signature'
        prefixes = ['', 'o', 'p0/p1/']
        for p in prefixes:
            hmac_mock.reset_mock()
            path = '/v1/AUTH_account/c/' + p
            expected_url = path + (
                '?temp_url_sig=temp_url_signature'
                '&temp_url_expires=1400003600'
                '&temp_url_prefix=' + p
            )
            expected_body = '\n'.join(
                [
                    self.method,
                    '1400003600',
                    'prefix:' + path,
                ]
            ).encode('utf-8')
            url = self.proxy.generate_temp_url(
                path,
                self.seconds,
                self.method,
                prefix=True,
                temp_url_key=self.key,
            )
            key = self.key
            if not isinstance(key, bytes):
                key = key.encode('utf-8')
            self.assertEqual(url, expected_url)
            self.assertEqual(
                hmac_mock.mock_calls,
                [
                    mock.call(key, expected_body, sha1),
                    mock.call().hexdigest(),
                ],
            )

            self.assertIsInstance(url, type(path))

    def test_generate_temp_url_invalid_path(self):
        self.assertRaisesRegex(
            ValueError,
            'path must be representable as UTF-8',
            self.proxy.generate_temp_url,
            b'/v1/a/c/\xff',
            self.seconds,
            self.method,
            temp_url_key=self.key,
        )

    @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature")
    def test_generate_absolute_expiry_temp_url(self, hmac_mock):
        if isinstance(self.expected_url, bytes):
            expected_url = self.expected_url.replace(
                b'1400003600', b'2146636800'
            )
        else:
            expected_url = self.expected_url.replace(
                u'1400003600', u'2146636800'
            )
        url = self.proxy.generate_temp_url(
            self.url,
            2146636800,
            self.method,
            absolute=True,
            temp_url_key=self.key,
        )
        self.assertEqual(url, expected_url)

    def test_generate_temp_url_bad_time(self):
        for bad_time in [
            'not_an_int',
            -1,
            1.1,
            '-1',
            '1.1',
            '2015-05',
            '2015-05-01T01:00',
        ]:
            self.assertRaisesRegex(
                ValueError,
                self.time_errmsg,
                self.proxy.generate_temp_url,
                self.url,
                bad_time,
                self.method,
                temp_url_key=self.key,
            )

    def test_generate_temp_url_bad_path(self):
        for bad_path in [
            '/v1/a/c',
            'v1/a/c/o',
            'blah/v1/a/c/o',
            '/v1//c/o',
            '/v1/a/c/',
            '/v1/a/c',
        ]:
            self.assertRaisesRegex(
                ValueError,
                self.path_errmsg,
                self.proxy.generate_temp_url,
                bad_path,
                60,
                self.method,
                temp_url_key=self.key,
            )

class TestTempURLUnicodePathAndKey(TestTempURL):
    url = u'/v1/\u00e4/c/\u00f3'
    key = u'k\u00e9y'
    expected_url = (
        u'%s?temp_url_sig=temp_url_signature' u'&temp_url_expires=1400003600'
    ) % url
    expected_body = u'\n'.join(
        [
            u'GET',
            u'1400003600',
            url,
        ]
    ).encode('utf-8')

class TestTempURLUnicodePathBytesKey(TestTempURL):
    url = u'/v1/\u00e4/c/\u00f3'
    key = u'k\u00e9y'.encode('utf-8')
    expected_url = (
        u'%s?temp_url_sig=temp_url_signature' u'&temp_url_expires=1400003600'
    ) % url
    expected_body = '\n'.join(
        [
            u'GET',
            u'1400003600',
            url,
        ]
    ).encode('utf-8')

class TestTempURLBytesPathUnicodeKey(TestTempURL):
    url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8')
    key = u'k\u00e9y'
    expected_url = url + (
        b'?temp_url_sig=temp_url_signature' b'&temp_url_expires=1400003600'
    )
    expected_body = b'\n'.join(
        [
            b'GET',
            b'1400003600',
            url,
        ]
    )

class TestTempURLBytesPathAndKey(TestTempURL):
    url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8')
    key = u'k\u00e9y'.encode('utf-8')
    expected_url = url + (
        b'?temp_url_sig=temp_url_signature' b'&temp_url_expires=1400003600'
    )
    expected_body = b'\n'.join(
        [
            b'GET',
            b'1400003600',
            url,
        ]
    )

class TestTempURLBytesPathAndNonUtf8Key(TestTempURL):
    url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8')
    key = b'k\xffy'
    expected_url = url + (
        b'?temp_url_sig=temp_url_signature' b'&temp_url_expires=1400003600'
    )
    expected_body = b'\n'.join(
        [
            b'GET',
            b'1400003600',
            url,
        ]
    )


import csv
import itertools
import operator
import re

i = 0
contacts_dict = []
with open('phonebook_raw.csv', encoding="utf8") as f:
    reader = csv.reader(f, delimiter=",")
    contacts_list = list(reader)
    for items in contacts_list:
        pattern_phone = r'(\+7|8)?\s*\(?(\d{3})\)?[\s*-]?(\d{3})[\s*-]?(\d{2})[\s*-]?(\d{2})(\s*)\(?(доб\.?)?\s*(\d*)?\)?'
        fixed_phone = re.sub(pattern_phone, r'+7(\2)\3-\4-\5\6\7\8', items[5])
        contacts_list[i][5] = fixed_phone
        try:
            i += 1
        except IndexError:
            break

    keys = contacts_list[0]
    values = contacts_list[1:]
    for num, vals in enumerate(values):
        contacts_dict.append({})
        for key, val in zip(keys, vals):
            contacts_dict[num].update({key: val})
    for v in contacts_dict:
        splt = v['lastname'].split(' ')
        if len(splt) > 1:
            v['lastname'] = splt[0]
            v['firstname'] = splt[1]
            if len(splt) > 2:
                v['surname'] = splt[2]

        splt = v['firstname'].split(' ')
        if len(splt) > 1:
            v['firstname'] = splt[0]
            v['surname'] = splt[1]
    all_keys = set(contacts_dict[0].keys())
    group_list = ['firstname', 'lastname']
    group = operator.itemgetter(*group_list)
    cols = operator.itemgetter(*(all_keys ^ set(group_list)))
    contacts_dict.sort(key=group)
    grouped = itertools.groupby(contacts_dict, group)

    merge_data = []
    for (firstname, lastname), g in grouped:
        merge_data.append({'lastname': lastname, 'firstname': firstname})
        for gr in g:
            d1 = merge_data[-1]
            for k, v in gr.items():
                if k not in d1 or d1[k] == '':
                    d1[k] = v
keys = list(merge_data[0].keys())
with open('phonebook.csv', "w", encoding="utf8") as f:
    datawriter = csv.writer(f, delimiter=',')
    datawriter.writerow(keys)
    for d in merge_data:
        datawriter.writerow(d.values())

print('Data updated.')


import torch
from helper import load_data
from solution import PCA, AE, frobeniu_norm_error
import numpy as np
import os

def test_pca(A, p):
    pca = PCA(A, p)
    Ap, G = pca.get_reduced()
    A_re = pca.reconstruction(Ap)
    error = frobeniu_norm_error(A, A_re)
    print('PCA-Reconstruction error for {k} components is'.format(k=p), error)
    return G

def test_ae(A, p):
    model = AE(d_hidden_rep=p)
    model.train(A, A, 128, 300)
    A_re = model.reconstruction(A)
    final_w = model.get_params()
    error = frobeniu_norm_error(A, A_re)
    print('AE-Reconstruction error for {k}-dimensional hidden representation is'.format(k=p), error)
    return final_w

if __name__ == '__main__':
    dataloc = "../data/USPS.mat"
    A = load_data(dataloc)
    A = A.T
    ## Normalize A
    A = A/A.max()

    ### YOUR CODE HERE
    # Note: You are free to modify your code here for debugging and justifying your ideas for 5(f)

    #we need to normalize A by subtracting mean for PCA for minimum recontruction error as per notes, we can use same A_norm
    #for both AE and PCA for proper comparision
    size = A.shape[1]
    A_norm = (A - ((1/size)*A@np.ones((size,1)))@np.ones((1,size)))

    ps = [32, 64, 128]
    #ps = [64]
    for p in ps:
        G = test_pca(A_norm, p)
        final_w = test_ae(A_norm, p)

    #for relation between W and G
    ps = [32, 64, 128]
    for p in ps:
        G = test_pca(A_norm, p)
        W = test_ae(A_norm, p)
        print("G-W", frobeniu_norm_error(G, W))
        M = G.T @ W
        U, S, V = np.linalg.svd(M, full_matrices=True)
        Mp = U @ np.eye(U.shape[0], V.shape[0]) @ V
        Gp = G @ Mp
        print("Gp-W", frobeniu_norm_error(Gp, W))
        print('M-Mp', frobeniu_norm_error(M, Mp))
        A_re = Gp @ Gp.T @ A_norm
        print("A -A_Re:", frobeniu_norm_error(A_norm, A_re))

    ### END YOUR CODE


import sys, os
from sklearn.datasets import load_files
from sklearn.datasets import fetch_20newsgroups
import settings

data_path = settings.data_path
def load(path):
	if path=="newsgroups":
		categories=['alt.atheism', 'soc.religion.christian']
		data=fetch_20newsgroups(subset='all',categories=categories)
	else:
		data_container_path = data_path + 'corpora/raw/' + path
		print("Loading files dataset")
		data = load_files(data_container_path, load_content=True, shuffle=True, encoding='utf8', charset='utf8', charset_error='', decode_error='strict', random_state=42)

	rawdata, classes = data.data, data.target
	print("classes: %s" % data.target_names)
	print('data loaded')
	return rawdata, classes, data.target_names
def load_dataset(dataset_name):
    """Load and return dataset (classification).

    The dataset is a classic and very easy multi-class classification
    dataset.

    Returns
    -------
    data : Bunch
        Dictionary-like object, the interesting attributes are:
        'data', the data to learn, 'target', the classification labels,
        'target_names', the meaning of the labels, 'feature_names', the
        meaning of the features

    Examples
    --------
    Let's say you are interested in the samples 10, 25, and 50, and want to
    know their class name.

    >>> data = load_dataset("en_room_facebook")
    >>> data.target[[10, 25, 50]]
    array([0, 0, 1])
    >>> list(data.target_names)
    ['room', 'rent', 'amsterdam']
    """
    data_file = csv.reader(open(join(data_path, 'datasets', dataset_name + '.csv')))
    first_line = next(data_file)
    n_samples = int(first_line[0])
    n_features = int(first_line[1])
    target_names = np.array(first_line[2:])
    second_line = next(data_file)
    feature_names = second_line
    data = np.empty((n_samples, n_features))
    target = np.empty((n_samples,), dtype=np.int)

    for i, line in enumerate(data_file):
        data[i] = np.asarray(line[:-1], dtype=np.float)
        target[i] = np.asarray(line[-1], dtype=np.int)

    return Bunch(data=data, target=target, target_names=target_names, feature_names=feature_names)
def save_dataset():
	return None


import ssl
import smtplib

from email.message import EmailMessage

# email the error if there has happened something faulty
def email_error(website, error, huis):
    message = EmailMessage()
    message.set_content(str(error)+'\n'+str(huis))
    message['FROM'] = "huizzoeker@outlook.com"
    message['TO'] = ["rensevdzee@hotmail.com"]
    message['SUBJECT'] = "Error bij "+website
    context = ssl.create_default_context()
    # set up SMTP server
    with smtplib.SMTP('smtp-mail.outlook.com', 587) as smtp:
        smtp.starttls(context=context)
        smtp.login(message['FROM'], "Ludosanders")
        smtp.send_message(message)
        smtp.quit()

# email the results if there has been new houses spotted
def email_new(results, alert):
    #print(products, new_products, updated_products)
    message = EmailMessage()
    message.set_content(results)
    message['FROM'] = "huizzoeker@outlook.com"
    message['TO'] = ["rensevdzee@hotmail.com"]
    if alert == 1:
        message['SUBJECT'] = "1 nieuwe osso gevonden"
    if alert > 1:
        message['SUBJECT'] = str(alert)+" nieuwe ossos gevonden"
    context = ssl.create_default_context()
    # set up SMTP server
    with smtplib.SMTP('smtp-mail.outlook.com', 587) as smtp:
        smtp.starttls(context=context)
        smtp.login(message['FROM'], "Ludosanders")
        smtp.send_message(message)
        smtp.quit()

# combine the data of the house into an appropiate sentence for the email
def write_msg(new,old):
    str_list = []
    for item in new:
        str_list.append("(**Nieuw**) Potentieel huis gevonden op de {} met {} m2 voor {} {}!  {}".format(item[1],item[3], item[5], item[6], item[8]))
    for item in old:
        str_list.append("Potentieel huis gevonden op de {} met {} m2 voor {} {}!  {}".format(item[1],item[3], item[5], item[6], item[8]))
        # if item[0] == "(Nieuw)":
        #     str_list.append("{}Potentieel huis gevonden met {} m2 voor {} {}!  {}".format(
        #         item[0], item[1], item[2], item[3], item[4]))
        # else:
        #     str_list.append("Potentieel huis gevonden met {} m2 voor {} {}!  {}".format(
        #         item[1], item[2], item[3], item[4]))
    return "\n".join(str_list)


import socket

import threading

import json

import copy

import socket_protocol as sp

class PeerBroadcast(threading.Thread):

    def __init__(self, peer, message, broadcast_list, node_info):
        threading.Thread.__init__(self)
        self.peer = peer
        self.message = message
        self.broadcast_list = broadcast_list
        self.node_info = node_info

    def run(self):
        print ("broadcasting message")

        peer = self.peer
        message = self.message
        broadcast_list = self.broadcast_list
        node_info = self.node_info

        PEER_IP = peer["address"]
        PORT = peer["port"]

        peer_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        peer_conn.settimeout(2.0)

        try:
            peer_conn.connect((peer["address"], peer["port"]))
        except ConnectionRefusedError:
            print ("peer not online")

            print (peer["address"])
            print (peer["port"])

            return

        new_broadcast_list = copy.copy(broadcast_list)
        new_broadcast_list.append(node_info)

        broadcast_json = {
            "message_type": "broadcast",
            "message": message,
            "broadcast_sent_to": new_broadcast_list
        }

        broadcast_string = json.dumps(broadcast_json)

        try:

            try:

                sp.send_msg(peer_conn, broadcast_string.encode('utf-8'))

                response_data = sp.recv_msg(peer_conn).decode('utf-8')

                response_data_json = json.loads(response_data)

                if (response_data_json["message_type"] == "success"):

                    print ("BROADCAST SUCCESS")

                else:
                    print ("BROADCAST FAILURE")

                peer_conn.close()
                return

            except Exception as err:
                print ("unable to send broadcast")
                return
        except ConnectionRefusedError:
            print ("connection refused")


from modules.playogo_db import PlayogoDb, Venue
from modules.venue_search import VenueSearch
from bs4 import BeautifulSoup
import os, urllib, requests, json, logging

class Controller():
    pass

class VenuesController(Controller):

    def search_results(search_query):

        db = PlayogoDb()

        lat, lng, err = VenueSearch.get_lat_lng(search_query)

        if err:
            logging.error("VenueSearch::get_lat_lng failed.")
            return {
                'closest_venues': [],
                'search_query': search_query,
                'error': "Sorry but there was a problem processing your request. If this problem persists please report it to playogosports@gmail.com."
            }

        closest_venues = VenueSearch.get_closest_venues(lat, lng)

        return {
            'closest_venues': closest_venues,
            'search_query': search_query,
            'encoded_search_query': urllib.parse.quote_plus(search_query)
        }

    def venue(slug):

        db = PlayogoDb()

        venue = db.session.query(Venue).filter_by(id=slug).first();

        return {
            'venue_name': venue.name,
            'venue_formatted_address': venue.formatted_address,
            'venue_street_number': str(venue.street_number),
            'venue_route_short': venue.route_short,
            'venue_city': venue.city_long,
            'venue_province_short': venue.admin_area_level_1_short,
            'venue_country': venue.country_short,
            'venue_postal_code': venue.postal_code,
            'venue_lat': venue.lat,
            'venue_lng': venue.lng,
            'venue_phone_number': venue.phone_number,
            'venue_website': venue.website,
            'google_encoded_location': urllib.parse.quote_plus(venue.formatted_address),
            'google_encoded_timhortons': urllib.parse.quote_plus('Tim Horton\'s near ' + venue.formatted_address),
            'google_encoded_hotels': urllib.parse.quote_plus('Hotels near ' + venue.formatted_address),
            'google_api_key': os.environ['PLAYOGO_GMAPS_API_KEY']
        }

    def home_page():
        return {}


import cv2
import numpy as np
from PIL import Image
import os

def load_cv(path):
    loaded_images = []

    x = cv2.imread(path)
    if x is not None:

        x = cv2.resize(x,(640,480))
        x = cv2.flip(x,-1)

        x = x/255.
        loaded_images.append(x)
        return np.stack(loaded_images, axis=0)
    else:
        return None

# if __name__ == "__main__":
#     # try:
#     test = []
#     image_files = os.listdir("examples")

#     for image in image_files:
#         image = "examples/{}".format(image)
#         # print(image)
#         test.append(image)

#     print(test[1])
#     x = load_images([test[1]])
#     print("First is")
#     print(x)

#     y = load_cv([test[1]])
#     print("First111 is")
#     print(y)
#     # pass

#     if x.all()==y.all():
#         print("OK")


import torch
import numpy as np
import matplotlib.pyplot as plt
from torchvision import transforms
import pandas as pd
from torch.utils.data import Dataset , DataLoader
from roboflow import Roboflow

rf = Roboflow(api_key="TJJkQCgCgKoIpfHHTeRT")
project = rf.workspace("erushiki").project("side-ktyou")
dataset = project.version(5).download("multiclass")

csv_or = pd.read_csv('/content/Side-5/train/_classes.csv')

csv_or.keys()

csv_bottom = csv_or[' Bottom']
csv_left = csv_or[' Left']

csv_right = csv_or[' Right']
csv_top = csv_or[' Top']

csv_file = csv_or['filename']

DEFAULT_PATH_TRAIN = '/content/Side-5/train'
DEFAULT_PATH_TEST = '/content/Side-5/test'

def test(fname, b, l, r, t):
    img = plt.imread(f"{DEFAULT_PATH_TRAIN}/{fname}")

    plt.imshow(img)
    # plt.legend()

    print(f"{b, l, r, t}")

    if b == 1:
        print("Show Bottom")
    if l == 1:
        print("Show Left")
    if r == 1:
        print("Show Right")
    if t == 1:
        print("Show Top")

class dset(Dataset):

    def __init__(self, csv, root, transforms=None):

        super().__init__()

        self.csv = csv
        self.root = root
        self.transforms = transforms

    def __len__(self):

        return len(self.csv)

    def __getitem__(self, idx):

        if torch.is_tensor(idx):
            idx = idx.tolist()

        readed = pd.read_csv(self.csv)

        csv_bottom = readed[' Bottom']
        csv_left = readed[' Left']
        csv_right = readed[' Right']
        csv_top = readed[' Top']
        csv_file = readed['filename']

        img_name = f'{self.root}/{csv_file[idx]}'
        label = np.array((csv_bottom[idx], csv_left[idx], csv_right[idx], csv_top[idx]))
        img = plt.imread(img_name)
        # img = torch.from_numpy(np.float32(img))
        # label = torch.from_numpy(np.float32(label))
        sample = {'image': img, 'label': label}

        if self.transforms != None:
            img = self.transforms(img)

        return img, label

transform = transforms.Compose([

    transforms.ToTensor(),
    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])

Datas = dset('/content/Side-5/train/_classes.csv', '/content/Side-5/train', transform)

DataL = DataLoader(Datas, 1, True)


"""
This script is watching a specific file for the presence of 2D points,
    and plots them in a live fashion.
"""
import csv
import os
import random
import tkinter as tk

import matplotlib
import matplotlib.animation as animation
from matplotlib import style
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure

matplotlib.use("TkAgg")

matplotlib.style.use("ggplot")

file_location = "data.txt"

f = Figure(figsize=(5, 5), dpi=100)
a = f.add_subplot(111)

def create_a_file_and_input_random_data_to_it():
    with open(file_location, "w") as data_file:
        x_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        y_values = [random.randint(0, 10) for _ in range(10)]
        for x, y in zip(x_values, y_values):
            line = "{},{}\n".format(x, y)
            data_file.write(line)

def animate(i):
    with open(file_location, "r") as data_file:
        csvReader = csv.reader(data_file)

        x_list = []
        y_list = []
        for row in csvReader:
            if row:
                x_list.append(float(row[0]))
                y_list.append(float(row[1]))

        a.clear()
        a.plot(x_list, y_list)

main_window = tk.Tk()
main_window.state("zoomed")
main_window.title("File point watcher")

f = Figure(figsize=(5, 5), dpi=100)
a = f.add_subplot(111)

a.clear()
x_list, y_list = [1, 2, 3, 4, 5, 6], [1, 2, 3, 4, 5, 6]
a.plot(x_list, y_list)

canvas = FigureCanvasTkAgg(f, main_window)
canvas.draw()
canvas.get_tk_widget().pack(side=tk.BOTTOM, fill=tk.BOTH, expand=True)

# Creates the file if it does not exist
if not os.path.isfile(file_location):
    create_a_file_and_input_random_data_to_it()

ani = animation.FuncAnimation(f, animate, interval=500)
main_window.mainloop()


import sys
from robot import Robot
import numpy as np
import skfuzzy as fuzz
from skfuzzy import control as ctrl
import matplotlib.pyplot as plt
import time
import helper
import math

# Antecedents objects
distancesAntecedents = []
for i in range(8):
    distancesAntecedents.append(ctrl.Antecedent(np.arange(0, 5, 0.01), 'distance_' + str(i)))

direction = ctrl.Antecedent(np.arange(-math.pi, math.pi, 0.01), 'direction')
direction['error_left'] = fuzz.trimf(direction.universe, [-math.pi, -math.pi, 0])
direction['error_right'] = fuzz.trimf(direction.universe, [0, math.pi, math.pi])
direction['no_error'] = fuzz.trimf(direction.universe, [-0.2, 0, 0.2])
goal_distance = ctrl.Antecedent(np.arange(0, 500, 0.01), 'goal_distance')
goal_distance['perto'] = fuzz.trimf(goal_distance.universe, [0, 0, 0.05])
goal_distance['longe'] = fuzz.trapmf(goal_distance.universe, [0.05, 1, 500, 500])

for i in range(8):
    distancesAntecedents[i]["perto"] = fuzz.trapmf(distancesAntecedents[i].universe, [0, 0, 0.5, 1.5])
    distancesAntecedents[i]["longe"] = fuzz.trapmf(distancesAntecedents[i].universe, [0.5, 1.5, 6, 6])

vel_left = ctrl.Consequent(np.arange(-4, 4, 0.1), 'vel_left')
vel_right = ctrl.Consequent(np.arange(-4, 4, 0.1), 'vel_right')

vel_left['pos_fast'] = fuzz.trimf(vel_left.universe, [1, 4, 4])
vel_left['pos_slow'] = fuzz.trimf(vel_left.universe, [0, 1.5, 1.5])
vel_left['neg_fast'] = fuzz.trimf(vel_left.universe, [-4, -4, -1])
vel_left['neg_slow'] = fuzz.trimf(vel_left.universe, [-1.5, -1.5, 0])
vel_right['pos_fast'] = fuzz.trimf(vel_left.universe, [1, 4, 4])
vel_right['pos_slow'] = fuzz.trimf(vel_left.universe, [0, 1.5, 1.5])
vel_right['neg_fast'] = fuzz.trimf(vel_left.universe, [-4, -4, -1])
vel_right['neg_slow'] = fuzz.trimf(vel_left.universe, [-1.5, -1.5, 0])

# go to goal rules
rule1 = ctrl.Rule(direction['error_left'] & goal_distance['longe'], vel_left['neg_fast'])
rule2 = ctrl.Rule(direction['error_left'] & goal_distance['longe'], vel_right['pos_fast'])
rule3 = ctrl.Rule(direction['error_right'] & goal_distance['longe'], vel_right['neg_fast'])
rule4 = ctrl.Rule(direction['error_right'] & goal_distance['longe'], vel_left['pos_fast'])
rule5 = ctrl.Rule(direction['no_error'] & goal_distance['longe'], vel_left['pos_fast'])
rule6 = ctrl.Rule(direction['no_error'] & goal_distance['longe'], vel_right['pos_fast'])
rule7 = ctrl.Rule(goal_distance['perto'], vel_right['pos_slow'])
rule8 = ctrl.Rule(goal_distance['perto'], vel_right['neg_slow'])
rule9 = ctrl.Rule(goal_distance['perto'], vel_left['neg_slow'])
rule10 = ctrl.Rule(goal_distance['perto'], vel_left['pos_slow'])

# Avoid obstacle rules
rule11 = ctrl.Rule(distancesAntecedents[1]['perto'] | distancesAntecedents[2]['perto'] | distancesAntecedents[3]['perto'], vel_left['pos_fast'])
rule12 = ctrl.Rule(distancesAntecedents[1]['perto'] | distancesAntecedents[2]['perto'] | distancesAntecedents[3]['perto'], vel_right['neg_fast'])
rule13 = ctrl.Rule(distancesAntecedents[4]['perto'] | distancesAntecedents[5]['perto'] | distancesAntecedents[6]['perto'], vel_right['pos_fast'])
rule14 = ctrl.Rule(distancesAntecedents[4]['perto'] | distancesAntecedents[5]['perto'] | distancesAntecedents[6]['perto'], vel_left['neg_fast'])
rule15 = ctrl.Rule(distancesAntecedents[1]['longe'] & distancesAntecedents[2]['longe'] & distancesAntecedents[3]['longe'] & distancesAntecedents[4]['longe'] & distancesAntecedents[5]['longe'] & distancesAntecedents[6]['longe'] & goal_distance['longe'], vel_right['pos_fast'])
rule16 = ctrl.Rule(distancesAntecedents[1]['longe'] & distancesAntecedents[2]['longe'] & distancesAntecedents[3]['longe'] & distancesAntecedents[4]['longe'] & distancesAntecedents[5]['longe'] & distancesAntecedents[6]['longe'] & goal_distance['longe'], vel_left['pos_fast'])

go_to_goal_ctrl = ctrl.ControlSystem([rule1, rule2, rule3, rule4, rule5, rule6, rule7, rule8, rule9, rule10, rule11, rule12, rule13, rule14, rule15, rule16])
go_to_goal = ctrl.ControlSystemSimulation(go_to_goal_ctrl)

positions = [[], []]
velocities_left = []
velocities_right = []
distances = []
direction_error = []

def get_info_to_plot(vel, distance, position, error, it):
    if it % 250 == 0:
        positions[0].append(position[0])
        positions[1].append(position[1])
        velocities_left.append(vel[0])
        velocities_right.append(vel[1])
        distances.append(distance)
        direction_error.append(error)

def fuzzy(error, goal_distance, distances):
    go_to_goal.input['direction'] = error
    go_to_goal.input['goal_distance'] = goal_distance
    for i in range(1, 7):
        go_to_goal.input['distance_' + str(i)] = distances[i]
    go_to_goal.compute()
    return [go_to_goal.output['vel_left'], go_to_goal.output['vel_right']]

# goal = [0, 0]
x = float(input("Insira as coordenadas de objetivo do robô.\nX: "))
y = float(input("Y: "))
goal = (x, y)
print(goal)

robot = Robot()
i=0
while(robot.get_connection_status() != -1):
    us_distances = robot.read_ultrassonic_sensors()
    position = robot.get_current_position()
    orientation = robot.get_current_orientation()
    distance = helper.euclidian_distance(position[:2], goal)
    angle = helper.diff_angle(goal, position)
    error = orientation[2] - angle + math.pi
    if error > math.pi:
        error = 2*math.pi - error
        error *= -1
    if error < -math.pi:
        error += 2*math.pi

    vel = fuzzy(error, distance, us_distances[:8])
    get_info_to_plot(vel, distance, position, error, i)

    robot.set_left_velocity(vel[0])
    robot.set_right_velocity(vel[1])
    i += 1

plt.plot(positions[0], positions[1])
plt.plot(positions[0][0], positions[1][0], 'go', label="Posição Inicial")
plt.plot(goal[0], goal[1], 'gx', label="Objetivo")
plt.plot(positions[0][-1], positions[1][-1], 'rx', label="Posição Final")
plt.title('Posição do robô')
plt.legend()
plt.show()

plt.plot(velocities_left, 'b', label='Motor Esquerdo')
plt.plot(velocities_right, 'r', label='Motor Direito')
plt.title('Velocidade dos motores (rad/s) por iteração')
plt.legend()
plt.show()

plt.plot(distances)
plt.title('Distância para o objetivo (m)')
plt.show()

fig, ax1 = plt.subplots()

color = 'tab:red'
ax1.set_xlabel('Iterações')
ax1.set_ylabel('Distância para o objetivo (m)', color=color)
ax1.plot(distances, color=color)
ax1.tick_params(axis='y', labelcolor=color)

ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis
color = 'tab:blue'
ax2.set_ylabel('Erro na direção para o objetivo (radianos)', color=color)  # we already handled the x-label with ax1
ax2.plot(direction_error, color=color)
ax2.tick_params(axis='y', labelcolor=color)

fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.show()


import math
import matplotlib.pyplot as plt
import numpy as np
import random

def f(x):
    f = 94 * (np.cos(x) ** 3) - 24 * np.cos(x) + 177 * (np.sin(x) ** 2) - 108 * (np.sin(x) ** 4) - 72 * (
            np.cos(x) ** 3) * (np.sin(x) ** 2) - 65
    return f

def fder1(x):
    f = -6 * ((2*np.cos(x) - 1) ** 2) * (15*np.cos(x)**2 - 3*np.cos(x) - 4) * np.sin(x)
    return f

def fder2(x):
    f = (1728 - 1800*np.cos(x))*np.sin(x)**4 + (2358*np.cos(x) - 2004)*np.sin(x)**2 - 402*np.cos(x) + 354
    return f

def bisection_method(a, b, tolerance):
    print(f"\nChecking in [{a}, {b}]")
    if f(a) == 0:
        print(f"Root is {a}, initial lower boundary!\n")
        return a
    elif f(b) == 0:
        print(f"Root is {b}, initial upper boundary!\n")
        return b
    elif f(a) * f(b) >= 0:
        print(f"No or multiple roots in [{a},{b}], bisection does not work!\n")
        return "0"
    error = abs(b - a)
    N = math.ceil((math.log(abs(b - a)) - math.log((1 / 2) * (1 / pow(10, tolerance)))) / math.log(2))
    for i in range(N):

        xi = random.uniform(a, b)

        try:
            if f(a) * f(xi) < 0:
                b = xi
                error = abs(b - a)
            elif f(b) * f(xi) < 0:
                a = xi
                error = abs(b - a)
            else:
                print("Good guess.")
                print(f"Lower boundary a is: {a} and upper b is: {b}, approximate root: {xi}, rounded to: "
                      f"{round(xi, tolerance)}"
                      f" after {i + 1} iterations.\n")
                return [round(xi, tolerance), i + 1]
        except Exception as e:
            print(type(e))
            print("Something went wrong\n")
            return [round(xi, tolerance), i + 1]

    print(f"Error is: {error}")
    print(f"Lower boundary a is: {a} and upper b is: {b}, approximate root: {xi}, rounded to: {round(xi, tolerance)}"
          f" after {i + 1} iterations.\n")
    return [round(xi, tolerance), i + 1]

def newton_raphson_method(a, b, tolerance):
    print(f"\nChecking in [{a}, {b}]")
    if f(a) == 0:
        print(f"Root is {a}, initial lower boundary!\n")
        return a
    elif f(b) == 0:
        print(f"Root is {b}, initial upper boundary!\n")
        return b

    x = random.uniform(a, b)
    counter = 0
    while f(x) * fder2(x) <= 0 or fder1(x) == 0 or f(x) == 0:
        if f(x) == 0:
            print(f"Nice guess! Root is {x}\n")
            return [round(x, tolerance), 0.0, 0]
        x = random.uniform(a, b)
        counter += 1
        if counter > 50:
            return "Couldn't find appropriate guess"

    print(f"Guess is: {x}")

    try:
        for i in range(100):

            xi = x - (1 / ((fder1(x) / f(x)) - (1 / 2) * (fder2(x) / fder1(x))))

            error = abs(xi - x)
            if error < (1 / 2) * (1 / pow(10, tolerance)):
                print(
                    f"Error is: {error}\nApproximate root: {xi}, rounded to: {round(xi, tolerance)} after {i+1} iterations\n")
                return [round(xi, tolerance), error, i + 1]
            x = xi
    except Exception as e:
        print(type(e))
        print(f"Something went wrong after {i} iterations.\n")

    print(f"Maximum iterations reached, failed to find root in [{a},{b}]")

    return [round(xi, tolerance), error, i + 1]

def secant_method(x0, x1, tolerance):

    print(f"\nChecking in [{x0}, {x1}]")

    if f(x0) == f(x1):
        return "Line is parallel to x axis! Secant doesn't work!"

    if f(x0) == 0:
        print(f"Root at lower boundary. Root is {x0}\n")
        return x0
    if f(x1) == 0:
        print(f"Root at upper boundary. Root is {x1}\n")
        return x1

    def q():
        return f(x0) / f(x1)

    def r():
        return f(x2) / f(x1)

    def s():
        return f(x2) / f(x0)

    x2 = (x0 + x1) / 2
    error = abs(x2 - x1)
    counter = 0
    while s() == 1 or r() == 1 or q() == 1:
        x2 = random.uniform(x0, x1)
        if f(x2) == 0:
            print(f"Nice guess! Root is {round(x2, tolerance)}\n")
            return [round(x2, tolerance), 0.0, 0]
        counter += 1
        if counter > 50:
            return "Couldn't find appropriate x2"

    for i in range(1, 100):

        try:
            xi = x2 - ((r() * (r() - q()) * (x2 - x1) + (1 - r()) * s() * (x2 - x0)) /
                       ((q() - 1) * (r() - 1) * (s() - 1)))
        except Exception as e:
            print(type(e))
            print(f"Oh no, division by zero after {i + 1} iterations!")
            print(f"x={x2}, rounded to: {round(x2, tolerance)}\nf(x)={f(x2)}\nf'(x)={fder1(x2)}\n")
            return [round(xi, tolerance), error, i + 1]
        x0 = x1
        x1 = x2
        x2 = xi
        error = abs(x2 - x1)
        if error < (1/2)*(1/pow(10, tolerance)):
            print(f"Error is: {error}\nApproximate root: {xi}, rounded to: {round(xi, tolerance)} "
                  f"after {i + 1} iterations\n")
            return [round(xi, tolerance), error, i + 1]
        if f(x0) == f(x1):
            print("Line is parallel to x axis!")
            print(f"Error is: {error}\nApproximate root: {xi}, rounded to: {round(xi, tolerance)} "
                  f"after {i + 1} iterations\n")
            return [round(xi, tolerance), error, i + 1]

    print(f"Maximum iterations reached. Approximation: {xi}, rounded to: {round(xi, tolerance)} "
          f"after {i + 1} iterations\n")
    return [round(xi, tolerance), error, i + 1]

def main():
    # 1
    tolerance = 5
    print("\n\nA \n\n")

    print("Bisection")
    for j in range(3):
        print(bisection_method(j, j + 1, tolerance))

    print("\nNewton-Raphson")
    for j in range(3):
        print(newton_raphson_method(j, j + 1, tolerance))

    print("\nSecant")
    for j in range(3):
        print(secant_method(j, j + 1, tolerance))

    # 2

    print("\n\nB\n\n")

    for i in range(10):
        for j in range(3):
            print(bisection_method(j, j + 1, tolerance))

    x = np.linspace(0, 3, 100)
    fig = plt.figure(figsize=(10, 5))
    plt.plot(x, f(x))
    plt.grid()
    plt.axis()
    plt.title("Ex 2")
    plt.show()

    print(f(0), fder1(0), fder2(0))
    return 0

main()


# -*- coding: utf8 -*-
# sybolic processor 기능을 가진 sympy를 불러 들여 sp 라는 이름과 연결시킴
import sympy as sp

# sympy 의 symbols 기능을 이용해 기호 x y를 정함
x, y = sp.symbols('x y')

# 기호 x y를 이용해 기호 z 를 정함
z = x+2*y

# 기호 z 를 화면에 표시함
print('z = %s' %z)


from pathlib import Path
import leveldb  # pylint: disable=no-name-in-module
from typing import Dict
import json
import os
from pydantic import BaseModel
from utils.subject_factory import dict_to_model

class Persistence:
    """
    Persistence class for storing data in leveldb
    """

    def __init__(self, db_path):
        """
        :param db_path: path to the leveldb database
        """
        self.db_path = db_path
        self.__make_to_path(self.db_path)
        self.db = leveldb.LevelDB(self.db_path, create_if_missing=True)

    @staticmethod
    def __make_to_path(path:str):
        """
        Make a path to a file
        """
        full_path = Path(path)
        full_path.parent.mkdir(parents=True, exist_ok=True)

    def get(self, key: str) -> Dict|BaseModel|None:
        """
        Get value from leveldb
        """
        key = key.encode("utf-8")
        try:
            result =  self.db.Get(key)
            result = result.decode("utf-8")
            result = json.loads(result)
            if "pydantic" in result:
                return dict_to_model(result["pydantic"], result["data"])
            return result
        except KeyError:
            return None

    def put(self, key: str, value: Dict|BaseModel) -> None:
        """
        Put value in leveldb
        """
        key = key.encode("utf-8")
        if isinstance(value, BaseModel):
            value = value.to_level_db()
        value = json.dumps(value).encode("utf-8")
        self.db.Put(key, value)

    def delete(self, key: str) -> None:
        """
        Delete value from leveldb
        """
        key = key.encode("utf-8")
        self.db.Delete(key)


import argparse
import hashlib
import logging
import os
import subprocess
import sys
import tempfile
from datetime import datetime
from os.path import join
from typing import Optional, List

fastr_default_testdir = 'test.fastr'
gnur_default_testdir = 'test.gnur'

_opts = argparse.Namespace()
_common_arg_parser = argparse.ArgumentParser()
_common_arg_parser.add_argument('--fastr-home', metavar='FASTR_HOME', dest="fastr_home", type=str, default=None,
                                required=True,
                                help='The FastR standalone repo home directory (required).')
_common_arg_parser.add_argument('--gnur-home', metavar="GNUR_HOME", dest="gnur_home", default=None, required=True,
                                help='The GnuR home directory (required).')
_common_arg_parser.add_argument('--graalvm-home', metavar="GRAALVM_HOME", dest="graalvm_home", default=None,
                    help='The GraalVM root directory (optional). If provided, FastR binaries are taken from this directory')
_common_arg_parser.add_argument('--repos', metavar='REPO_NAME=URL', dest="repos", type=str, default=None,
                    help='Repos to install packages from. Can be set by "FASTR_REPOS" env var. '
                         'Example: "--repos FASTR=file://$HOME/fastr_repo,CRAN=file://$HOME/minicran"')
_common_arg_parser.add_argument('-v', '--verbose', dest="verbose", action="store_const", const=1, default=0,
                    help='Do verbose logging.')
_common_arg_parser.add_argument('-V', '--very-verbose', dest="verbose", action="store_const", const=2,
                    help='Do very verbose logging.')
_common_arg_parser.add_argument('--dump-preprocessed', dest="dump_preprocessed", action="store_true",
                    help='Dump processed output files where replacement filters have been applied.')
_common_arg_parser.add_argument('-q', '--quiet', dest="quiet", action="store_true",
                    help="Don't log anything")
_common_arg_parser.add_argument('--fastr-testdir', metavar="FASTR_TESTDIR", dest="fastr_testdir", default=fastr_default_testdir,
                    help='FastR test result directory (default: "test.fastr").')
_common_arg_parser.add_argument('--gnur-testdir', metavar="GNUR_TESTDIR", dest="gnur_testdir", default=gnur_default_testdir,
                    help='GnuR test result directory (default: "test.gnur").')
_common_arg_parser.add_argument('-l', '--log-file', dest="log_file", default="pkgtest.log",
                    help='Log file name (default: "FASTR_TESTDIR/pkgtest.log").')
pkg_spec_group = _common_arg_parser.add_mutually_exclusive_group()
pkg_spec_group.add_argument('--pkg-pattern', metavar='PATTERN', dest='pattern', type=str, default=None,
                            help='Pattern of packages to install and potentially test')
pkg_spec_group.add_argument('--pkg-filelist', metavar='FILE', dest='filelist', type=str, default=None,
                            help='File containing a list of packages to install and potentially test')

def get_opts() -> argparse.Namespace:
    return _opts

def get_common_arg_parser() -> argparse.ArgumentParser:
    return _common_arg_parser

def get_fastr_repo_dir() -> str:
    return _opts.fastr_home

def get_gnur_home() -> str:
    '''
    Returns path to GnuR home dir, e.g., gnur/gnur/R-3.4.0/.
    '''
    return _opts.gnur_home

def get_gnur_rscript() -> str:
    '''
    returns path to Rscript in sibling gnur directory
    '''
    # return _mx_gnur().extensions._gnur_rscript_path()
    return join(get_gnur_home(), "bin", "Rscript")

def get_gnur_include_path() -> str:
    # if graalvm():
    #     return join(_mx_gnur().dir, 'gnur', _mx_gnur().extensions.r_version(), 'include')
    # return join(mx_fastr._gnur_path(), "include")
    return join(get_gnur_home(), 'include')

def get_fastr_home() -> str:
    if get_graalvm_home():
        fastr_home = join(get_graalvm_home(), "languages", "R")
        if os.path.exists(fastr_home):
            return fastr_home
        else:
            logging.error(f"Could not find FastR home at {fastr_home}")
    return get_fastr_repo_dir()

def get_fastr_include_path() -> str:
    return join(get_fastr_home(), 'include')

def graalvm_rscript() -> str:
    assert get_graalvm_home() is not None
    return join(get_graalvm_home(), 'bin', 'Rscript')

def get_fastr_rscript() -> str:
    graalvm_dir = get_graalvm_home()
    if graalvm_dir is not None:
        return join(graalvm_dir, "bin", "Rscript")
    return join(get_fastr_repo_dir(), 'bin', 'Rscript')

def get_default_cran_mirror() -> str:
    default_cran_mirror_file = join(get_fastr_home(), 'etc', 'DEFAULT_CRAN_MIRROR')
    url = None
    try:
        f = open(default_cran_mirror_file, "r")
        url = f.readline()
        if url:
            url = url.strip()
        if 'FASTR_MRAN_MIRROR' in os.environ:
            overlay = os.environ['FASTR_MRAN_MIRROR'].strip()
            overlay += '' if overlay.endswith('/') else '/'
            date = url[-len('2000-00-00'):]
            url = overlay + 'snapshot/' + date
            logging.info("Using '{0}' instead of the default MRAN mirror.".format(url))
    except IOError as e:
        logging.error("Could not read %s: %s" % (default_cran_mirror_file, e))
    finally:
        f.close()
    return url

def get_r_version(rscript_binary: str, tmp_dir: str) -> str:
    r_version_file = join(tmp_dir, "R.version")
    args = [rscript_binary, "--verbose", "-e", "cat(R.Version()[['major']], '.', R.Version()[['minor']], file='%s', sep='')" % r_version_file]
    if not os.path.exists(rscript_binary):
        abort(1, "Rscript binary '%s' does not exist.", rscript_binary)
    try:
        logging.debug("Running command: %s", args)
        output = subprocess.check_output(args, stderr=subprocess.STDOUT, universal_newlines=True)

        with open(r_version_file, "r") as f:
            lines = f.readlines()
        return lines[-1].strip()
    finally:
        os.unlink(r_version_file)

def get_graalvm_home() -> Optional[str]:
    return _opts.graalvm_home

def abort(status: int, *args) -> None:
    if args:
        logging.fatal(*args)
    quit(status)

def warn(*args) -> None:
    logging.warning(*args)

def log_step(state: str, step: str, rvariant: str) -> None:
    if not _opts.quiet:
        logging.info("{0} {1} with {2}".format(state, step, rvariant))
        log_timestamp()

def log_timestamp() -> None:
    if not _opts.quiet:
        logging.info("timestamp: {0}".format(str(datetime.now())))

def check_r_versions() -> None:
    '''
    Checks that FastR and GnuR have the same version.
    '''
    tmp_dir = tempfile.mkdtemp()
    gnur_version = get_r_version(get_gnur_rscript(), tmp_dir)
    fastr_version = get_r_version(get_fastr_rscript(), tmp_dir)
    logging.info("Using FastR version = %r ; GnuR version = %r: " % (fastr_version, gnur_version))
    if gnur_version != fastr_version:
        abort(1, 'GraalVM R version does not match GnuR version: %r (FastR) vs. %r (GnuR)' % (fastr_version, gnur_version))

VERY_VERBOSE = logging.DEBUG - 5

def parse_arguments(argv: List[str], r_version_check=True) -> List[str]:
    """
    Parses the given argument vector and stores the values of the arguments known by this script to appropriate globals.
    This represents the common arguments for both pkgtest and pkgcache.
    The unknown arguments are returned for further processing.
    """
    global _opts
    _opts, r_args = _common_arg_parser.parse_known_args(args=argv)

    if not _opts.repos and os.environ.get('FASTR_REPOS'):
        _opts.repos = os.environ['FASTR_REPOS']
    elif _opts.repos and os.environ.get('FASTR_REPOS'):
        warn('Both --repos option and FASTR_REPOS env var set, using value from command-line argument.')

    # ensure that first arg is neither the name of this package nor a py file
    if r_args and (r_args[0] == __package__ or r_args[0].endswith(".py")):
        r_args = r_args[1:]

    log_format = '%(message)s'
    if _opts.verbose == 1:
        log_level = logging.DEBUG
    elif _opts.verbose == 2:
        log_level = VERY_VERBOSE
    else:
        log_level = logging.INFO
    logging.basicConfig(filename=_opts.log_file, filemode="w", level=log_level, format=log_format)

    logger = logging.getLogger("")
    if len(logger.handlers) < 2:
        # We want to have 2 handlers in total - one that logs in _opts.log_file and one that logs
        # to console.
        console_handler = logging.StreamHandler(stream=sys.stdout)
        console_handler.setLevel(log_level)
        console_handler.setFormatter(logging.Formatter(log_format))
        logger.addHandler(console_handler)

    logging.log(VERY_VERBOSE, "known_args: %s" % _opts)

    # print info if _opts.graalvm is used
    if get_graalvm_home():
        logging.info("Using GraalVM at %r" % get_graalvm_home())

    # ensure that FastR and GnuR have the same version
    if r_version_check:
        check_r_versions()

    return r_args

def computeApiChecksum(includeDir: str) -> str:
    """
    Computes a checksum of the header files found in the provided directory (recursively).
    The result is a SHA256 checksum (as string with hex digits) of all header files.
    """
    m = hashlib.sha256()
    rootDir = includeDir
    fileList = list()
    for root, _, files in os.walk(rootDir):
        for f in files:
            fileName = join(root, f)
            if fileName.endswith('.h'):
                fileList.append(fileName)

    # sorting makes the checksum independent of the FS traversal order
    fileList.sort()
    for fileName in fileList:
        try:
            with open(fileName) as f:
                fileContent = f.read().encode()
                m.update(fileContent)
        except IOError as e:
            # Ignore errors on broken symlinks
            if not os.path.islink(fileName) or os.path.exists(fileName):
                raise e

    hxdigest = str(m.hexdigest())
    logging.debug("Computed API version checksum {0}".format(hxdigest))
    return hxdigest

def ensure_dir(path):
    if not os.path.exists(path):
        os.makedirs(path)
    return path


#!/usr/bin/env python3

from fairseq import dictionary
from fbtranslate import vocab_constants

class Dictionary(dictionary.Dictionary):
    """A mapping from symbols to consecutive integers"""

    def __init__(
        self,
        pad='<pad>',
        eos='</s>',
        unk='<unk>',
        max_special_tokens=vocab_constants.MAX_SPECIAL_TOKENS,
    ):
        self.unk_word, self.pad_word, self.eos_word = unk, pad, eos
        self.symbols = []
        self.count = []
        self.indices = {}

        self.pad_index = self.add_symbol(pad)
        assert self.pad_index == vocab_constants.PAD_ID

        # Adds a junk symbol for vocab_constants' GO_ID
        self.add_symbol('<reserved>')

        self.eos_index = self.add_symbol(eos)
        assert self.eos_index == vocab_constants.EOS_ID

        self.unk_index = self.add_symbol(unk)
        assert self.unk_index == vocab_constants.UNK_ID

        # Adds junk symbols to pad up to the number of special tokens.
        num_reserved = max_special_tokens - len(self.symbols)
        for i in range(num_reserved):
            self.add_symbol(f'<reserved_{i}>')

        self.nspecial = len(self.symbols)
        assert self.nspecial == max_special_tokens

class CharDictionary(Dictionary):
    """Character vocab with its additonal special tokens."""

    def __init__(
        self,
        word_delim='<space>',
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.word_delim = word_delim
        self.bow_index = self.add_symbol('<bow>')
        self.eow_index = self.add_symbol('<eow>')
        self.word_delim_index = self.add_symbol(word_delim)
        self.nspecial += 3


from django.test import TestCase

from ..forms import SearchConditionForm
from ..models import Type, Region, Continent

class FormTest(TestCase):
    fixtures = ['disaster.json']

    def setUp(self):
        self.types = ['All'] + [t.id for t in Type.objects.all()]
        self.continents = ['All'] + [c.id for c in Continent.objects.all()]
        self.regions = ['All'] + [r.id for r in Region.objects.all()]

    def test_valid_form(self):
        data = {
            'country': 'Bangladesh',
            'continent': 'Asia',
            'region': 'SouthernAsia',
            'type': 'Flood',
            'order': 'totalDeaths'
        }
        form = SearchConditionForm(data=data)
        self.assertTrue(form.is_valid())


from . import table as tb
from . import table_fetcher
from .expression import RqExpression
from .grouping import RqGrouping
from .ordering import RqOrdering
from .ordering import RqTableOrdering

class RqSession:
    """A class for RecruitQuery sessions."""

    def __init__(self, github, field_exprs, source=None, condition=None, groups=None, having=None, orders=None, limit=None):
        self._field_exprs = field_exprs
        self._source = source
        self._condition = condition
        self._groups = groups
        self._having = having
        self._orders = orders
        self._limit = limit

        rel_keys = RqExpression.ExtractTokensFromExpressions(self._field_exprs)
        if self._condition:
            rel_keys += RqExpression.ExtractTokensFromExpressions([self._condition])
        if self._groups:
            rel_keys += RqExpression.ExtractTokensFromExpressions(self._groups)
        if self._having:
            rel_keys += RqExpression.ExtractTokensFromExpressions([self._having])
        if self._orders:
            rel_keys += RqExpression.ExtractTokensFromExpressions(self._orders[0])
        rel_keys = list(set(rel_keys))

        # if there's '*' in select, where, group by, having, orders argument then the rel_keys is reduced to just '*'
        if u"*" in rel_keys:
            rel_keys = [u"*"]
        self._fetcher = table_fetcher.RqTableFetcher(github, rel_keys)

    def _GetEmptyTable(self):
        table = tb.RqTable()
        table.SetFields(self._field_exprs)
        return table

    def Execute(self):
        # source is either a label (eg. "google.issues") or a RqSession
        if self._source:
            source_table = self._source.Execute() if isinstance(self._source, RqSession) else self._fetcher.Fetch(self._source)
            if not source_table[:]:
                return self._GetEmptyTable()
            else:
                if u"*" in self._field_exprs:
                    self._field_exprs = source_table.GetFields()
        else:
            source_table = tb.RqTable()
            source_table.SetFields([u"Dummy Field"])
            source_table.Append([u"Dummy Value"])

        # evaluate where clause
        if self._condition:
            filtered_table = tb.RqTable()
            filtered_table.SetFields(source_table.GetFields())
            meets = RqExpression.EvaluateExpression(source_table, self._condition)
            for i, row in enumerate(source_table):
                if meets[i]:
                    filtered_table.Append(row)
        else:
            filtered_table = source_table
        if not filtered_table[:]:
            return self._GetEmptyTable()

        # evaluate all necessary expressions
        # in reversed order because we process from the rightmost item first
        select_tokens = RqExpression.ExtractTokensFromExpressions(self._field_exprs[:])
        eval_exprs = select_tokens
        if self._orders:
            order_tokens = RqExpression.ExtractTokensFromExpressions(self._orders[0])
            eval_exprs += order_tokens
        if self._having:
            having_tokens = RqExpression.ExtractTokensFromExpressions([self._having])
            eval_exprs += having_tokens
        if self._groups:
            eval_exprs += self._groups

        res_table = RqExpression.EvaluateExpressions(filtered_table, eval_exprs)

        # group by
        if self._groups:
            res_tables = RqGrouping.GenerateGroups(res_table, self._groups)
        else:
            res_tables = [res_table]

        # having
        if self._having:
            filtered_tables = []
            for table in res_tables:
                if all(RqExpression.EvaluateExpression(table, self._having)):
                    filtered_tables.append(table.SliceCol(0, len(table.GetFields()) - len(having_tokens)))
            res_tables = filtered_tables

        # order by
        if self._orders:
            for table in res_tables:
                table.Copy(table.SliceCol(0, len(table.GetFields()) - len(order_tokens)).Chain(RqExpression.EvaluateExpressions(table, self._orders[0])))
                ordering = RqOrdering(table, self._orders[1])
                table.Copy(ordering.Sort(keep_order_fields=True))
            ordering = RqTableOrdering(res_tables, self._orders[1])
            res_tables = ordering.Sort()

        # process select
        for table in res_tables:
            table.Copy(RqExpression.EvaluateExpressions(table, self._field_exprs))

        # check if all tokens in expressions are contained in aggregate functions
        check_exprs = [expr for expr in self._field_exprs if expr not in self._groups] if self._groups else self._field_exprs
        if RqExpression.IsAllTokensInAggregate(check_exprs):
            for table in res_tables:
                table = table.SetTable([table[0]])

        merged_table = tb.RqTable()
        merged_table.SetFields(res_tables[0].GetFields())
        for table in res_tables:
            for row in table:
                merged_table.Append(row)

        # process limit
        if self._limit:
            merged_table.SetTable(merged_table[:self._limit])

        return merged_table


import re
import csv
from typing import Dict
from .stocks import Cost, Transaction, Stock

def process_transaction(transaction):
    costs = []

    headline = transaction.pop(0)

    ticker = headline[2]
    optype = headline[3].upper()
    date = headline[0]
    shares = headline[4]
    price_per_share = headline[5]

    for line in transaction:
        name = line[1]
        value = line[6].replace(",", ".")
        costs.append(Cost(name, value))

    return Transaction(ticker, optype, date, shares, price_per_share, costs)

def is_transaction_header(line):
    date_pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
    try:
        return bool(re.match(date_pattern, line[0]))
    except IndexError:
        return False

def process_stock_operations(stock_operations, stocks):
    start = -1
    for index, line in enumerate(stock_operations):
        has_start = start > -1
        has_more = index < len(stock_operations) - 1
        is_next_transaction_header = has_more and is_transaction_header(
            stock_operations[index + 1]
        )

        if not has_start and is_transaction_header(line):
            start = index
            continue

        if has_start and (is_next_transaction_header or not has_more):
            end = index + 1
            op = process_transaction(stock_operations[start:end])
            if not op.ticker in stocks:
                stocks[op.ticker] = Stock(op.ticker)
            stocks[op.ticker].add_transaction(op)
            start = -1

def sanitize(content):
    new_content = []
    for line in content:
        if (
            line[0].startswith("Total")
            or (len(line) > 1 and line[1].startswith("Transferir") and line[2] == "")
            or any([i == "Dividend" for i in line])
            or any([i == "Rendimento" for i in line])
        ):
            continue
        new_content.append(line)

    return new_content

def process_csv(file) -> Dict[str, Stock]:
    stocks = {}

    with open(file, newline="") as csvfile:
        month = csv.reader(csvfile, delimiter=",", quotechar='"')

        content = list(month)
        content = sanitize(content)

        start = -1
        for index, line in enumerate(content):
            has_start = start > -1
            has_more = index < len(content) - 1
            is_next_title = has_more and len(content[index + 1]) == 1

            if not has_start and len(line) == 1:
                start = index
                continue

            if has_start and is_next_title:
                end = index + 1
                process_stock_operations(content[start:end], stocks)
                start = -1

    return stocks


import types
import urllib

import models
import utils.batch.batch_op as batchop

def get_batch_query_args(query):
    """From a query string, retrieve the key-value pairs.

    The query string has to be built as a normal HTTP query: it can either
    start with a question mark or not, key and values must be separated
    by an equal sign (=), and multiple key-value pairs must be separated
    by an ampersand (&):

        [?]key=value[&key=value&key=value...]

    The values are then retrieved and stored in a set to avoid duplicates.

    :param query: The query string to analyze.
    :type query: string
    :return A dictionary with keys the keys from the query, and values the
    values stored in a list.
    """
    args = {}

    if all([query, isinstance(query, types.StringTypes)]):
        if query.startswith("?"):
            query = query[1:]

        query = query.split("&")
        if isinstance(query, types.ListType):
            for arg in query:
                arg = arg.split("=")
                # Can't have query with just one element, they have to be
                # key=value.
                if len(arg) == 2:
                    name, value = arg
                    value = urllib.unquote(value)
                    values = args.setdefault(name, set())
                    values.add(value)

    return {k: list(v) for k, v in args.iteritems()}

def create_batch_operation(json_obj, db_options):
    """Create a `BatchOperation` object from a JSON object.

    No validity checks are performed on the JSON object, it must be a valid
    batch operation JSON structure.

    :param json_obj: The JSON object with all the necessary paramters.
    :type json_obj: dict
    :param db_options: The mongodb configuration parameters.
    :type db_options: dict
    :return A `BatchOperation` object, or None if the `BatchOperation` cannot
    be constructed.
    """
    batch_op = None
    get_func = None

    def _complete_batch_op():
        batch_op.db_options = db_options
        batch_op.query_args = get_batch_query_args(
            get_func(models.QUERY_KEY, None))

        for key, val in json_obj.iteritems():
            setattr(batch_op, key, val)

    if json_obj:
        get_func = json_obj.get
        resource = get_func(models.RESOURCE_KEY, None)
        distinct = get_func(models.DISTINCT_KEY, None)
        document = get_func(models.DOCUMENT_KEY, None)

        if resource in models.COLLECTIONS:
            # Check first if we have a count-distinct or distinct operation
            # to perform.
            if all([distinct, document]):
                batch_op = batchop.BatchCountDistinctOperation()
            elif all([distinct, not document]):
                batch_op = batchop.BatchDistinctOperation()
            # Then in case proceed with the normal operations.
            elif resource:
                if resource == models.COUNT_COLLECTION:
                    batch_op = batchop.BatchCountOperation()
                elif resource == models.JOB_COLLECTION:
                    batch_op = batchop.BatchJobOperation()
                elif resource == models.BUILD_COLLECTION:
                    batch_op = batchop.BatchBuildOperation()
                elif resource == models.TEST_CASE_COLLECTION:
                    batch_op = batchop.BatchTestCaseOperation()
                elif resource == models.TEST_GROUP_COLLECTION:
                    batch_op = batchop.BatchTestGroupOperation()
                elif resource == models.TEST_REGRESSION_COLLECTION:
                    batch_op = batchop.BatchTestRegressionOperation()

            _complete_batch_op()

    return batch_op

def execute_batch_operation(json_obj, db_options):
    """Create and execute the batch op as defined in the JSON object.

    :param json_obj: The JSON object that will be used to create the batch
    operation.
    :type json_obj: dict
    :param db_options: The mongodb database connection parameters.
    :type db_options: dict
    :return The result of the operation execution, or None.
    """
    batch_op = create_batch_operation(json_obj, db_options)

    result = None
    if batch_op:
        result = batch_op.run()

    return result


import json
import csv
import os
from flask import (
    Blueprint,  send_file, flash, g, redirect, render_template, request, session, url_for, current_app
)
from app.db import get_mongo
from app.db import get_db

admin_blueprint = Blueprint('admin', __name__, url_prefix='/admin')

data_dir = os.path.join(admin_blueprint.root_path, 'data')

print("root path", admin_blueprint.root_path)

@admin_blueprint.route('/', methods=['GET', 'POST'])
def admin():
    db = get_db()

    if ('admin' not in session):
        # session['admin'] = False;
        return redirect(url_for('index'))

    data = {}

    exams_in_db = db.execute(
        "SELECT count(*) as count from exam").fetchone()['count']

    users_in_db = db.execute(
        "SELECT count(*) as count from user").fetchone()['count']

    tests_in_db = db.execute(
        "SELECT count(*) as count from test").fetchone()['count']

    data['exams_in_db'] = exams_in_db
    data['users_in_db'] = users_in_db - 1
    data['tests_in_db'] = tests_in_db

    users = []
    user_details_in_db = db.execute(
        "SELECT * FROM user WHERE configured=1"
    ).fetchall()

    for user in user_details_in_db:
        print(user['name'])
        users.append({
            "id": user['id'],
            "email": user['email'],
            "name": user['name']
        })

    return render_template("admin/dashboard.html", data=data, users=users)

@admin_blueprint.route('/plan/<email>', methods=['GET', 'POST'])
def admin_user_plan_progress(email):
    db = get_db()

    if ("admin" not in session):
        return redirect(url_for('index'))

    plan_in_db = db.execute(
        "SELECT * FROM plan\
            WHERE email=?", (email,)
    ).fetchone()

    plan_id = plan_in_db["id"]
    start_date = plan_in_db["start_date"]
    end_date = plan_in_db["end_date"]

    dates = {
        "start_date": start_date.strftime("%m/%d/%Y"),
        "end_date": end_date.strftime("%m/%d/%Y")
    }

    uncompleted_topics_in_db = db.execute(
        "SELECT * FROM plan_details WHERE completed = 0 AND plan_id=?", (
            plan_id,)
    ).fetchall()

    completed_topics_in_db = db.execute(
        "SELECT * FROM plan_details WHERE completed = 1 AND plan_id=?", (
            plan_id,)
    ).fetchall()

    topic_number_data = {
        "completed": len(completed_topics_in_db),
        "uncompleted": len(uncompleted_topics_in_db),
    }
    print(topic_number_data)

    subject_data = {}

    for topic_row in completed_topics_in_db:
        if topic_row["subject_name"] in subject_data:
            subject_data[topic_row["subject_name"]] += 1
        else:
            subject_data[topic_row["subject_name"]] = 1

    subject_data["Uncompleted"] = len(uncompleted_topics_in_db)

    print(subject_data)

    return render_template('admin/plan_progress.html', email=email, topic_number_data=json.dumps(topic_number_data), start_end=dates, dates=json.dumps(dates), subject_data=json.dumps(subject_data))

@admin_blueprint.route('/syllabus', methods=['GET', 'POST'])
def admin_syllabus():
    db = get_db()

    if ('admin' not in session):
        # session['admin'] = False;
        return redirect(url_for('index'))

    exams_in_db = db.execute(
        "SELECT * FROM exam"
    ).fetchall()

    exams = []

    for exam in exams_in_db:
        exams.append({"id": exam["id"], "exam_name": exam["exam_name"]})
    print(exams)

    return render_template("admin/syllabus.html", exams=exams)

@admin_blueprint.route('/questions', methods=['GET', 'POST'])
def admin_question():
    db = get_db()

    if ('admin' not in session):
        # session['admin'] = False;
        return redirect(url_for('index'))

    topics_in_db = db.execute(
        "SELECT * FROM exam_details"
    ).fetchall()

    topics = []

    for topic in topics_in_db:
        topics.append({"id": topic["id"], "topic_name": topic["topic_name"]})

    return render_template("admin/questions.html", topics=topics)

@admin_blueprint.route('/download/questions', methods=['GET', 'POST'])
def download_questions():
    db = get_db()
    if request.method == "POST":
        topic_id = request.form['topic_id']
        questions_in_db = db.execute(
            "SELECT * FROM questions \
            WHERE topic_id = ?", (topic_id,)
        ).fetchall()
        column_names = [
            "id",
            "topic_id",
            "question_statement",
            "a",
            "b",
            "c",
            "d",
            "answer"
        ]
        topic_name = db.execute(
            "SELECT * FROM exam_details WHERE id = ?", (topic_id,)
        ).fetchone()["topic_name"]

        file_name = f"{topic_id}_{topic_name}.csv"
        file_path = os.path.join(data_dir, file_name)
        fp = open(file_path, 'w+')
        # use lineterminator for windows
        myFile = csv.writer(fp, lineterminator='\n')
        myFile.writerow(column_names)

        for row in questions_in_db:
            question = [
                row["id"],
                row["topic_id"],
                row["question_statement"],
                row["a"],
                row["b"],
                row["c"],
                row["d"],
                row["answer"]
            ]
            myFile.writerow(question)

        fp.close()
        return send_file(file_path, as_attachment=True)

@admin_blueprint.route('/upload/questions', methods=['GET', 'POST'])
def upload_questions():
    db = get_db()
    if request.method == "POST":

        if 'file' not in request.files:
            flash('No files')
            return redirect(url_for('admin.admin'))
        file = request.files['file']
        if file.filename == '':
            flash('No selected file')
            return redirect(url_for('admin.admin'))
        file_name = file.filename

        topic_id, topic_name = file_name.split('_')
        file_path = os.path.join(data_dir, file_name)

        file.save(file_path)

        update_rows = []
        new_rows = []
        delete_rows = []

        with open(file_path, mode='r')as file:

            # reading the CSV file
            csvFile = csv.reader(file)

            # displaying the contents of the CSV file
            for lines in csvFile:
                print(lines)
                if (lines[0] == ''):
                    new_rows.append(lines)
                elif (lines[1] == '' and lines[2] == '' and lines[3] == ''):
                    delete_rows.append(lines[0])
                else:
                    update_rows.append(lines)

        update_rows.pop(0)

        print("update", update_rows)
        print("new", new_rows)
        print("delete", delete_rows)

        for id in delete_rows:
            db.execute(
                "DELETE FROM questions where id=?", (id,)
            )
        for lines in update_rows:
            db.execute(
                "UPDATE questions SET question_statement=?, a=?, b=?, c=?, d=?, answer=? WHERE id=?", (
                    lines[2], lines[3], lines[4], lines[5], lines[6], lines[7], lines[0])
            )

        for lines in new_rows:
            db.execute(
                "INSERT INTO questions (topic_id, question_statement, a, b, c, d, answer) VALUES(?,?,?,?,?,?,?)",
                (topic_id, lines[2], lines[3],
                 lines[4], lines[5], lines[6], lines[7])
            )
        db.commit()
        error = "updated questions for " + topic_name
        flash(error)
        return redirect(url_for('admin.admin_question'))

@admin_blueprint.route('/download/syllabus', methods=['GET', 'POST'])
def download_syllabus():
    db = get_db()
    if request.method == "POST":
        exam_id = request.form['exam']
        topics_in_db = db.execute(
            "SELECT * FROM exam_details \
            WHERE exam_id = ?", (exam_id,)
        ).fetchall()
        exam_name = db.execute(
            "SELECT * FROM exam WHERE id = ?", (exam_id,)
        ).fetchone()["exam_name"]

        column_names = [
            "id",
            "topic_name",
            "subject_name",
            "required_hours"
        ]
        file_name = f"{exam_id}_{exam_name}.csv"
        file_path = os.path.join(data_dir, file_name)
        fp = open(file_path, 'w+')
        # use lineterminator for windows
        myFile = csv.writer(fp, lineterminator='\n')
        myFile.writerow(column_names)
        for row in topics_in_db:
            topic = [
                row['id'],
                row['topic_name'],
                row['subject_name'],
                row['required_hours']
            ]
            myFile.writerow(topic)
        fp.close()
        return send_file(file_path, as_attachment=True)

@admin_blueprint.route('/upload/syllabus', methods=['GET', 'POST'])
def upload_syllabus():
    db = get_db()
    if request.method == "POST":

        if 'file' not in request.files:
            flash('No files')
            return redirect(url_for('admin.admin'))
        file = request.files['file']
        if file.filename == '':
            flash('No selected file')
            return redirect(url_for('admin.admin'))
        file_name = file.filename

        exam_id, exam_name = file_name.split('_')
        file_path = os.path.join(data_dir, file_name)

        file.save(file_path)

        update_rows = []
        new_rows = []
        delete_rows = []

        with open(file_path, mode='r')as file:

            # reading the CSV file
            csvFile = csv.reader(file)

            # displaying the contents of the CSV file
            for lines in csvFile:
                print(lines)
                if (lines[0] == ''):
                    new_rows.append(lines)
                elif (lines[1] == '' and lines[2] == '' and lines[3] == ''):
                    delete_rows.append(lines[0])
                else:
                    update_rows.append(lines)

        update_rows.pop(0)

        print("update", update_rows)
        print("new", new_rows)
        print("delete", delete_rows)

        for id in delete_rows:
            db.execute(
                "DELETE FROM exam_details where id=?", (id,)
            )

        for lines in update_rows:
            db.execute(
                "UPDATE exam_details SET topic_name=?, subject_name=?, required_hours=? WHERE id=?", (
                    lines[1], lines[2], lines[3], lines[0],)
            )

        for lines in new_rows:
            db.execute(
                "INSERT INTO exam_details (exam_id, topic_name, subject_name, required_hours) VALUES(?,?,?,?)",
                (exam_id, lines[1], lines[2], lines[3])
            )
        db.commit()

        os.remove(file_path)

        error = "updated syllabus"
        flash(error)
        return redirect(url_for('admin.admin_syllabus'))