"""
Contain logic related to index
"""

import hashlib
from pathlib import Path

INDEX_FILE = '.ftpsync'

def compute_file_checksum(file_path: str) -> str:
    """
    Compute the SHA-256 checksum of given file
    :param file_path: path to the file to compute checksum for
    :return: computed file checksum
    """
    hasher = hashlib.sha256()

    with open(file_path, 'rb') as file:
        while True:
            chunk = file.read(hasher.block_size)
            if not chunk:
                break
            hasher.update(chunk)

    return hasher.hexdigest()

def load_index(file_path: str) -> dict:
    """
    Load the checksums from given index file
    :param file_path: path to the index file
    :return: loaded checksums
    """
    checksums = {}
    with open(file_path, 'r') as file:
        for line in file:
            clean_line = line.rstrip()
            part = clean_line.split(":")
            checksums[part[0]] = part[1]
    return checksums

def save_index(file_path: str, checksums: dict):
    """
    Save given index file
    :param file_path: path to the index file
    :param checksums: checksums to be saved
    """
    with open(file_path, 'w+') as file:
        for entry, checksum in checksums.items():
            file.write("{}:{}\n".format(entry, checksum))

def compute_index(directory: str) -> dict:
    """
    Compute index for given directory
    :param directory: directory to compute index for
    :return: computed index
    """
    checksums = {}
    for entry in Path(directory).rglob("*"):
        if entry.is_file() and INDEX_FILE not in entry.as_posix():
            checksums[entry.as_posix().lstrip(directory)] = compute_file_checksum(entry.as_posix())
    return checksums

def diff_index(previous: dict, current: dict) -> (dict, dict):
    """
    Compute the difference between previous & current index
    :param previous: the previous index (last state)
    :param current: the current index (new state)
    :return: tuple, with (new/changed) entries first, deleted entries second
    """
    changed = []
    deleted = []
    for entry, value in current.items():
        if entry not in previous or previous.get(entry) != value:
            changed.append(entry)

    for entry in previous:
        if entry not in current:
            deleted.append(entry)

    return changed, deleted


from __future__ import unicode_literals

from collections import defaultdict
from copy import copy
from pupa.scrape import Bill
from utils import CanadianScraper

import datetime
import lxml.etree as etree
import traceback
import pytz
import re

# TODO: Create ticket to move lxmlize into pupa.scrape.Base

ACTION_CLASSIFICATION = {
    'Adopted': 'passage',
    'Adopted on Consent': 'passage',
    'Amended': 'amendment-amended',
    'Confirmed': 'passage',
    'Deferred': 'deferred',
    'Deferred Indefinitely': 'deferred',
    'Intro Failed': None,
    'No Action': None,
    'No Quorum': 'failure',
    'Not Adopted': None,
    'Noted/Filed': 'filing',
    'Received': None,
    'Referred': 'committee-referral',
    'Recinded': 'failure',
    'Withdrawn': 'withdrawal',
    'Without Recs': None,
    'Waive Referral': None,
    # Made this one up
    'Introduced': 'introduction',
}

class TorontoBillScraper(CanadianScraper):
    AGENDA_ITEM_SEARCH_URL = 'http://app.toronto.ca/tmmis/findAgendaItem.do?function=doSearch&itemsPerPage=1000&sortBy=meetingDate&sortOrder=A'
    AGENDA_ITEM_URL_TEMPLATE = 'http://app.toronto.ca/tmmis/viewAgendaItemHistory.do?item={}'

    TIMEZONE = 'America/Toronto'
    date_format = '%B %d, %Y'

    start_date = datetime.datetime(2014, 12, 2)
    end_date = datetime.datetime.today() + datetime.timedelta(days=14)

    def scrape(self):
        for agenda_item in self.agendaItems(date_from=self.start_date, date_to=self.end_date):
            # TODO: Add agenda_item type to OCD
            leg_type = 'bill'

            title = agenda_item['Title'].replace('\n', ' ')
            title_re = re.compile('^(.+?)(?: - (?:by )?((?:Deputy )?Mayor|Councillor) (.+), seconded by ((?:Deputy )?Mayor|Councillor) (.+))?$')
            title, primary_role, primary_sponsor, secondary_role, secondary_sponsor = re.match(title_re, title).groups()

            b = Bill(
                identifier=agenda_item['Item No.'],
                title=title,
                legislative_session=None,
                classification=leg_type,
                from_organization={'name': self.jurisdiction.name},
            )
            b.add_source(agenda_item['url'], note='web')

            if primary_sponsor and secondary_sponsor:
                b.add_sponsorship(primary_sponsor, 'mover', 'person', True)
                b.add_sponsorship(secondary_sponsor, 'seconder', 'person', False)

            # TODO: Fake session for now
            b.legislative_session = '2014-2018'

            agenda_item_versions = self.agendaItemVersions(agenda_item['url'])

            # Use one version's full_text (will be most recent)
            b.extras['full_text'] = agenda_item_versions[0]['full_text']

            for version in agenda_item_versions:
                action_date = self.toDate(version['date'])

                if 'Summary' in version['sections']:
                    # TODO: Investigate whether these vary between versions, as
                    # we perhaps don't need to add one for each
                    b.add_abstract(version['sections']['Summary'], note='', date=action_date)

                if not version['action']:
                    continue
                if re.match(r'\d+:\d+ [A|P]M', version['action']):
                    continue

                action_description = version['action']
                responsible_org = version['responsible_org']
                action_class = ACTION_CLASSIFICATION.get(version['action'])

                def is_recommendation(version):
                    return any('Recommendations' in s for s in version['sections'].keys())

                if responsible_org == 'City Council':
                    responsible_org = self.jurisdiction.name
                else:
                    if action_class == 'passage':
                        action_class = 'committee-passage'

                        if is_recommendation(version):
                            action_class = 'committee-passage-favorable'

                b.add_action(
                    action_description,
                    action_date,
                    organization={'name': responsible_org},
                    classification=action_class
                )

            yield b

    def agendaItems(self, date_from=None, date_to=None):
        for agenda_item_summary in self.searchAgendaItems(date_from, date_to):
            yield agenda_item_summary

    def searchAgendaItems(self, date_from=None, date_to=None):
        """
        Submit a search query on the agenda item search page, and return a list
        of result pages.
        """
        page = self.lxmlize(self.AGENDA_ITEM_SEARCH_URL + '&fromDate={}&toDate={}'.format(date_from.strftime('%Y-%m-%d'), date_to.strftime('%Y-%m-%d')))
        for agenda_item_summary in self.parseSearchResults(page):
            yield agenda_item_summary

    def parseSearchResults(self, page):
        """Take a page of search results and return a sequence of data
        of tuples about the agenda_item, of the form

        TODO: Fix column names
        ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
        'Passed Date', 'Main Sponsor', 'Title')
        """
        for agenda_item, headers, _ in self.parseDataTable(page):
            id_key = headers[1]

            agenda_item_id = agenda_item[id_key]['label']
            agenda_item[id_key] = agenda_item_id

            agenda_item_url = self.AGENDA_ITEM_URL_TEMPLATE.format(agenda_item_id)
            agenda_item['url'] = agenda_item_url

            yield agenda_item

    def agendaItemVersions(self, agenda_item_url):
        page = self.lxmlize(agenda_item_url)
        versions = []
        for version in self.parseAgendaItemVersions(page):
            versions.append(version)

        return versions

    def parseAgendaItemVersions(self, page):
        script_text = page.xpath('//head/script[not(@src)]/text()')[0]
        index_qs = re.findall(r'if\(index == (\d)\){', script_text)
        function_qs = re.findall(r'var f = "(.*)";', script_text)
        agenda_item_id_qs = re.findall(r'agendaItemId:"(.*)"', script_text)
        url_template = 'http://app.toronto.ca/tmmis/viewAgendaItemDetails.do?function={}&agendaItemId={}'
        for i, f, id in sorted(zip(index_qs, function_qs, agenda_item_id_qs), key=lambda tup: tup[2]):
            agenda_item_version_url = url_template.format(f, id)
            version = self.agendaItemVersion(agenda_item_version_url)

            xpr = '//div[@id="header{}"]'.format(i)
            header = page.xpath(xpr)[0].text_content()
            header_re = re.compile('^(.+) consideration on (.+)$')
            org, date = re.match(header_re, header).groups()
            version.update({
                'responsible_org': org,
                'date': date,
            })

            if 'Origin' in version['sections']:
                origin_text = version['sections']['Origin']
                intro_date_re = re.compile('\((.+?)\) .+')
                intro_date = re.match(intro_date_re, origin_text).group(1)
                intro_version = copy(version)
                intro_version.update({
                    'date': intro_date,
                    'action': 'Introduced',
                })
                yield intro_version

            yield version

    def parseDataTable(self, table):
        """
        Legistar uses the same kind of data table in a number of
        places. This will return a list of dictionaries using the
        table headers as keys.
        """
        headers = table.xpath(".//th")
        rows = table.xpath(".//tr[@class='hoverOver']")

        keys = []
        for header in headers:
            text_content = header.text_content().replace(' ', ' ').strip()
            if text_content:
                keys.append(text_content)
            else:
                keys.append(header.xpath('.//input')[0].value)

        for row in rows:
            try:
                data = defaultdict(lambda: None)

                for key, field in zip(keys, row.xpath("./td")):
                    text_content = self._stringify(field)

                    if field.find('.//a') is not None:
                        address = self._get_link_address(field.find('.//a'))
                        if address:
                            value = {'label': text_content,
                                     'url': address}
                        else:
                            value = text_content
                    else:
                        value = text_content

                    data[key] = value

                yield data, keys, row

            except Exception as e:
                print('Problem parsing row:')
                print(etree.tostring(row))
                print(traceback.format_exc())
                raise e

    def _get_link_address(self, link):
        url = None
        if 'onclick' in link.attrib:
            onclick = link.attrib['onclick']
            if (onclick is not None and
                    onclick.startswith(("radopen('",
                                        "window.open",
                                        "OpenTelerikWindow"))):
                url = self.BASE_URL + onclick.split("'")[1]
        elif 'href' in link.attrib:
            url = link.attrib['href']

        return url

    def _stringify(self, field):
        for br in field.xpath("*//br"):
            br.tail = "\n" + br.tail if br.tail else "\n"
        for em in field.xpath("*//em"):
            if em.text:
                em.text = "--em--" + em.text + "--em--"
        return field.text_content().replace(' ', ' ').strip()

    def agendaItemVersion(self, agenda_item_version_url):
        """
        Details:
            * type
            * ward(s)

        Possible sections:
            * [ Board | Community Council | Committee ] Decision Advice and Other Information
            * Origin
            * [ Board | Community Council | Committee ] Recommendations
            * Summary
            * Financial Impact
            * Background Information [ (Board | Community Council | Committee | City Council) ] (parsed)
            * Speakers
            * Communications [ (Board | Community Council | Committee | City Council) ] (parsed)
            * Declared Interests [ (Board | Community Council | Committee | City Council) ]
            * Subsections? (recursive)
            * Motions (parsed)
                * Votes (optional)
            * Rulings (parsed) Ex: http://app.toronto.ca/tmmis/viewAgendaItemHistory.do?item=2016.EY12.29
                * Votes (optional on challenge)

        TODO: Investigate "Bills and By-law" [BL] code for bill context
        """
        page = self.lxmlize(agenda_item_version_url)
        version = {}
        version.update({
            'type': page.xpath("//table[@class='border'][1]//td[2]")[0].text_content().strip().lower(),
            'action': page.xpath("//table[@class='border'][1]//td[3]")[0].text_content().strip(),
            'full_text': etree.tostring(page, pretty_print=True).decode(),
        })

        wards = page.xpath("//table[@class='border'][1]//td[5]")[0].text_content().strip().lower()
        wards_re = re.compile('ward:(.*)')
        matches = re.match(wards_re, wards)
        if matches:
            wards = matches.group(1)
            if wards != 'all':
                wards = wards.split(', ')
        else:
            wards = 'all'

        version.update({'wards': wards})

        section_nodes = page.xpath("//table[@width=620 and .//font[@face='Arial' and @size=3] and .//tr[3]]")
        sections = {}
        for node in section_nodes:
            section_title = node.find('.//tr[1]/td//font/b').text_content().strip()
            section_content = node.find('.//tr[2]/td')
            sections[section_title] = section_content.text_content()

        if 'Motions' in sections:
            sections['Motions'] = self.parseAgendaItemVersionMotions(sections['Motions'])

        version.update({'sections': sections})

        return version

    def parseAgendaItemVersionMotions(self, motions_section):
        return motions_section

    def toTime(self, text):
        time = datetime.datetime.strptime(text, self.date_format)
        time = pytz.timezone(self.TIMEZONE).localize(time)
        return time

    def toDate(self, text):
        return self.toTime(text).date().isoformat()


"""
Export Google Keep notes to markdown files.

After obtaining your Google Keep data from https://takeout.google.com/,
unzip the folder and cd into it.

Copy this file in that folder and run:
    python export.py
"""

import os
import json
import datetime

PRINT_RAW = False
PRINT_PRETTY_JSON = True

class Note:
    def __init__(self, filename, data):
        self._name = filename.replace('.json', '')
        self._title = data['title']
        if 'labels' in data:
            self.labels = data['labels'][0]['name']
        else:
            self.labels = 'ROOT'
        #self.savename = self.labels + ' -- ' + self._name
        self.savename = self._name
        self._raw_date = data['userEditedTimestampUsec']
        self._date = datetime.datetime.fromtimestamp(self._raw_date/1_000_000)
        self._isTrashed = data['isTrashed']
        self._isArchived = data['isArchived']
        self._isList = True if 'listContent' in data else False
        if self._isList:
            checklist = ""
            for item in data['listContent']:
                tick = '+' if item['isChecked'] else ' '
                checklist += "- [{}] {}\n".format(tick, item['text'])
            self._content = checklist
            del checklist
        else:
            self._content = data['textContent']

    def _format_date(self):
        # return a date of this type: Tuesday November 03, 2015, 03:20:51 PM
        return self._date.strftime("%A %B %d, %Y, %I:%M:%S %p") # https://strftime.org/

    def isTrashed(self):
        return self._isTrashed

    def __repr__(self):
        # Add header to self._content containing title and last edited
        note  = "# {} {}\n".format(self._name, "(Archived)" if self._isArchived else "")
        note += "Last edited: {}\n\n".format(self._format_date())
        note += self._content
        return note

if __name__ == '__main__':

    ROOT_FOLDER = 'Keep'
    notes = [filename for filename in os.listdir(ROOT_FOLDER) if '.json' in filename]

    parsed_notes = {}
    for filename in notes:

        # Parse note
        with open("{}/{}".format(ROOT_FOLDER, filename), 'r') as json_file:
            data = json.load(json_file)

            if PRINT_RAW:
                if PRINT_PRETTY_JSON:
                    print(json.dumps(data, indent=4, sort_keys=True))
                    print()
                else:
                    print("{}\n\n".format(data))

        # Save note to dictionary if not trashed
        note = Note(filename, data)
        if not note.isTrashed():
            parsed_notes[note.savename] = note

    del notes

    # Create "markdown" folder
    if not os.path.isdir('markdown'):
        os.mkdir('markdown')

    # Write dictionary to markdown files
    for savename in parsed_notes:
        name = savename.replace('/', '-', 999)
        subdir = parsed_notes[savename].labels.replace('/', '-', 999)
        if not os.path.isdir(os.path.join('markdown', subdir)):
            os.mkdir(os.path.join('markdown', subdir))
        with open(os.path.join('markdown', subdir, f'{name}.md'), 'w') as f:
            f.write(str(parsed_notes[savename]))
    print(f'Saved {len(parsed_notes.keys())} notes to /markdown')


# coding=utf-8
"""
Public utils
"""
import base64
import cStringIO
import numpy as np
import os
import threading

import cv2
import dlib
import imagehash
import openface
from PIL import Image

from classes import FaceDetected

_DEFAULT_IMG_W = 400
_DEFAULT_IMG_H = 300

_IMG_RESIZE_BASE = 725.0

# Model paths
modelDir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'face_models')
dlibModelDir = os.path.join(modelDir, 'dlib')
openfaceModelDir = os.path.join(modelDir, 'openface')

class FaceCapture:
    """
    面部图像抓取类
    """

    def __init__(self, cam_id):
        self._video_capture = cv2.VideoCapture(cam_id)
        self._timer = None
        self._image = None

    def __del__(self):
        self._video_capture.release()

    def _capture(self):
        ret, frame = self._video_capture.read()
        # resize_frame = cv2.resize(frame, (0, 0), fx=0.5, fy=0.5)
        pImg = Image.fromarray(frame, "RGB")
        pImg.show()
        self._image = pImg

    def getFaces(self, count):
        """
        捕获用户面部图像
        :param count:捕获图像张数
        :return: 图像的nparray数组
        """
        for i in range(count):
            self._timer = threading.Timer(1, self._capture)
            self._timer.start()
            self._timer.join()
        self._timer.cancel()
        self._video_capture.release()
        return self._image

class FaceUtil:
    """
    面部图像处理工具类
    """

    @staticmethod
    def getAlign():
        """
        get faces by landmarks model
        :return:
        """
        try:
            return openface.AlignDlib(os.path.join(
                dlibModelDir,
                "shape_predictor_68_face_landmarks.dat"))
        except RuntimeError:
            print ('Please configure face_models.')

    @staticmethod
    def getNeuralNet():
        """
        获取面部特征向量神经网络
        :return:
        """
        return openface.TorchNeuralNet(os.path.join(
            openfaceModelDir, 'nn4.small2.v1.t7'), 96, False)

    @staticmethod
    def eigenValue(image):
        """
        面部向量特征提取
        :param image:
        :return:
        """
        pil_img = FaceUtil.convertImg(image, 'pil')
        np_img = FaceUtil.convertImg(image, 'np')
        representation = FaceUtil.getNeuralNet().forward(np_img)
        perceptual_hash = str(imagehash.phash(pil_img))
        return perceptual_hash, representation.tolist()

    @staticmethod
    def convertImg(image, type):
        # global pil_img
        # global np_img
        # Loding image from base64
        if isinstance(image, basestring):
            head = "data:image/jpeg;base64,"
            assert (image.startswith(head))
            imgdata = base64.b64decode(image[len(head):])
            img = cStringIO.StringIO(imgdata)
            pil_img = Image.open(img)
            # pil_img.show()
            np_img = np.asarray(pil_img)
            print('image convert:basestring')
        elif isinstance(image, Image.Image):
            pil_img = image
            np_img = np.asarray(image)
            print('image convert:Image.Image')
        elif isinstance(image, np.ndarray):
            np_img = image
            pil_img = Image.fromarray(image)
            print('image convert:np.ndarray')
        else:
            print('image convert:unknown')

        if type is 'np':
            return np_img
        elif type is 'pil':
            return pil_img

    @staticmethod
    def _resize(image):
        """
        重置图像尺寸
        :param image:
        :return:
        """
        im_width, im_height = image.size
        ratio = 1.0

        if max(im_width, im_height) < _IMG_RESIZE_BASE:
            return ratio, image

        if im_width >= im_height:
            # resize base on width
            ratio = _IMG_RESIZE_BASE / im_width
            im_height = int(ratio * im_height)
            im_width = int(_IMG_RESIZE_BASE)
        else:
            # resize base on height
            ratio = _IMG_RESIZE_BASE / im_height
            im_width = int(ratio * im_width)
            im_height = int(_IMG_RESIZE_BASE)

        image = image.resize((im_width, im_height), Image.BILINEAR)
        return ratio, image

    @staticmethod
    def faceDetect(image):
        scale, img = FaceUtil._resize(image)
        temp = np.asarray(img)
        temp = np.fliplr(temp)
        img_width, img_height = img.size
        rgbFrame = np.zeros(
            (img_height, img_width, 3),
            dtype=np.uint8
        )
        rgbFrame[:, :, 0] = temp[:, :, 0]
        rgbFrame[:, :, 1] = temp[:, :, 1]
        rgbFrame[:, :, 2] = temp[:, :, 2]

        all_faces = FaceUtil.getAlign().getAllFaceBoundingBoxes(rgbFrame)
        all_faces = sorted(
            all_faces,
            key=lambda rect: rect.width() * rect.height(),
            reverse=True
        )

        face_list = []
        for x in xrange(0, min(3, len(all_faces))):
            face_list.append(all_faces[x])

        detected_list = []
        for face_box in face_list:
            landmarks = FaceUtil.getAlign().findLandmarks(rgbFrame, face_box)
            alignedFace = FaceUtil.getAlign().align(
                96, rgbFrame, face_box,
                landmarks=landmarks,
                landmarkIndices=FaceUtil.getAlign().OUTER_EYES_AND_NOSE
            )
            if alignedFace is None:
                continue

            face = FaceDetected()
            face.img = alignedFace
            face.area = dlib.rectangle(
                left=int(face_box.left() / scale),
                top=int(face_box.top() / scale),
                right=int(face_box.right() / scale),
                bottom=int(face_box.bottom() / scale))

            face.landmarks = []
            for p in landmarks:
                face.landmarks.append((int(p[0] / scale), int(p[1] / scale)))

            detected_list.append(face)
        return detected_list


import logging
import os
import re
import shutil
import threading
from re import findall
import json
import uuid
from time import sleep
from shutil import copyfile
from PIL import Image
import pytesseract

from pdf2image import convert_from_path
import webapp

DOCUMENT_PATH = "public/documents/"
TEMP_PATH = "public/temp/"
SCANS_PATH = "scans/"
DATA_PATH = "public/data/"

logging.basicConfig(level=logging.INFO, format='%(asctime)s: %(name)s - %(levelname)s - %(message)s')
filesData = []  # Nested Dic

# We need a dic before and a dic after
def save_file_temp(file, id, companies, category, dates):
    # Save json
    meta_data = {
        "id": str(id),
        "project": "",
        "company": companies,
        "category": category,
        "date": dates
    }

    jsonFile = open(DATA_PATH + str(id) + ".json", "w")
    json.dump(meta_data, jsonFile)
    jsonFile.close()

    # Move from scans to temp
    shutil.move(SCANS_PATH + file, "public/temp/" + str(id) + ".pdf")

def get_Info(file_list):
    # define keys
    rechnung = "Rechnung"
    angebot = "Angebot"
    gutachten = "Gutachten"
    text = ""
    # file = ""

    for file in file_list:
        id = uuid.uuid4().hex

        # 1. kopieren
        copyfile(SCANS_PATH + file, TEMP_PATH + file + "_copy")
        # 2. copy convert
        pdf_to_png(file + "_copy", TEMP_PATH)
        # 3. analize copy
        text = pytesseract.image_to_string(Image.open (TEMP_PATH + file + "_copy.png"), lang="deu")
        # 4. delete copy
        if os.path.isfile (TEMP_PATH + file + "_copy"):
            os.remove (TEMP_PATH + file + "_copy")

        if os.path.isfile (TEMP_PATH + file + "_copy.png"):
            os.remove (TEMP_PATH + file + "_copy.png")
        # Extract Date
        dates = find_date(text)

        # Find company
        file1 = open('companies.json', 'r')

        companies = json.load(file1)

        found_companies = []

        found_companies = findall(r'((?:\w(?:\, ?| |(?: ?\& ?)|-|\. ?)?)*)(?:Bautenschutz|eG|Architekturbüro|Abruch-Entsorgungskonzepte|Elektroinstallation|mbH|Versorgungstechnik|Fliesenlegermeister|Bedachungen|Bauelemente|Golaschewski|Malerfachbetrieb|Fugentechnik|Trockenbau|GbR|e.?K.?|GmbH|AG|AkNW|AKNW|SE|(?:GmbH.?&.?Co.?KG))',text)
        found_companies = list(dict.fromkeys(found_companies))
        try:
            found_companies += findall(r'?i)(?:\bINGENIEURBÜRO|Projektbau|Architekturbüro|Fugentechnik\b) ((?:\w(?:\, ?| |(?: ?\& ?)|-|\. ?)?)*)(?:\n|,|.)',text)
        except:
            print ("")
        found_companies = list(dict.fromkeys(found_companies))

        # Determine if Rechnung or Angebot or Gutachten
        category_r = findall(rechnung, text)
        catergory_a = findall(angebot, text)
        catergory_g = findall(gutachten, text)

        category_r = list(dict.fromkeys(category_r))
        catergory_a = list(dict.fromkeys(catergory_a))
        catergory_g = list(dict.fromkeys(catergory_g))

        if category_r is not None:
            save_file_temp(file, id, found_companies, category_r, dates)
        elif catergory_a is not None:
            save_file_temp(file, id, found_companies, catergory_a, dates)
        elif catergory_g is not None:
            save_file_temp(file, id, found_companies, catergory_a, dates)

def get_temp_files():
    list = []
    for file in os.listdir (TEMP_PATH):
        if file.endswith('.pdf'):
            id = file.split(".")[0]
            f = open(DATA_PATH + id + '.json', 'r')
            data = json.load(f)

            list.append(
                {
                    "path": file,
                    "data": data
                }
            )
    return list

def set_companies(company_name):
    f = open('companies.json', )
    data = json.load(f)
    data.append(company_name)
    with open('companies.json', "w") as company:
        json.dump(data, company)

def get_companies():
    f = open('companies.json', )
    data = json.load(f)
    f.close()
    return data

def set_projects(project_name):
    f = open('projects.json', )
    data = json.load(f)
    data.append(project_name)
    with open('projects.json', "w") as project:
        json.dump(data, project)

def get_projects():
    f = open('projects.json', )
    data = json.load(f)
    f.close()
    return data

def read_files():
    # Folder scan
    dirs = os.listdir(SCANS_PATH)
    dirs.sort()
    file_list = []
    # Liste an zu kontrollierenden Files erstellen
    for file in dirs:
        if file.endswith('.pdf'):
            file_list.append(file)
    return file_list

def pdf_to_png(filename, path):
    images = convert_from_path(path + "/" +filename)
    images[0].save(path + filename + '.png', 'PNG')

def find_date(text):
    months = [ #Maybe use it later TODO
        [
            "Januar", "januar", "01"
        ], [
            "Februar", "februar", "02"
        ], [
            "März", "märz", "03"
        ] , [
            "April", "april", "04"
        ] , [
            "Mai", "Mai", "05"
        ] , [
            "Juni", "juni", "06"
        ] , [
            "Juli", "juli", "07"
        ] , [
            "August", "08", "august"
        ] , [
            "September", "september","09"
        ] , [
            "Oktober","oktober", "10"
        ] , [
            "November","november", "11"
        ] , [
            "Dezember","dezember", "12"
        ]
    ]

    month_list = ['Januar', 'januar', 'Februar', 'februar', 'März', 'märz', 'April', 'mai', 'Mai',
                  'juni', 'Juni', 'juli', 'Juli', 'august', 'August', 'September',
                  'september', 'oktober', 'Oktober', 'november', 'November', 'dezember', 'Dezember']
    dd_mm_yyyy_pattern = "\d{2}.\d{2}.\d{4}"
    dd_mm_yyyy_pattern2 = "\d{2}-\d{2}-\d{4}"
    yyyy_mm_dd_pattern = "\d{4}.\d{2}.\d{2}"
    yyyy_mm_dd_pattern2 = "\d{4}-\d{2}-\d{2}"
    date_list = []
    date_list_ausgeschrieben = []
    date_list += search_date(dd_mm_yyyy_pattern, text)
    date_list += search_date(dd_mm_yyyy_pattern2, text)
    date_list += search_date(yyyy_mm_dd_pattern, text)
    date_list += search_date(yyyy_mm_dd_pattern2, text)

    # Mit ausgeschrieben Monat und .FYI I know Dennis regex richtig nutzen. Dont tell me ok?
    for month in month_list:
        date_list_ausgeschrieben += search_date("\d{2}." + month + " \d{4}", text)
        date_list_ausgeschrieben += search_date("\d{2}-" + month + "-\d{4}", text)
        date_list_ausgeschrieben += search_date("\d{2}" + month + "\d{4}", text)
        date_list_ausgeschrieben += search_date("\d{2}." + month + "\d{4}", text)
        date_list_ausgeschrieben += search_date("\d{1}." + month + "\d{4}", text) #TODO add zero

    for date in date_list_ausgeschrieben:   # für jedes gefundene datum
        for month in months:                # suche jeweils eine liste pro Monat raus
            if month.index(0) in date:      # Checke ob Mmonat großgeschrieben
                date.replace(month.index(0), month.index(2))
                continue
            if month.index(1) in date:      #Checke ob Monat kleingeschrieben
                date.replace(month.index(1), month.index(2))

    date_list += date_list_ausgeschrieben
    print(date_list+"hello")
    return date_list #Schmeißt sofort alle duplikate raus

def search_date(pattern, text):
    dates = re.findall(pattern, text)
    new_dates = []
    if dates:
        for date in dates:
            new_dates.append(date.replace(".", "-"))
    print(new_dates)
    return list(dict.fromkeys(new_dates))

# Save after editing
def save_Final(data):
    # If folder doesnt exist, create it. Afterwards add file with count if alrady existing
    if (os.path.isdir(DOCUMENT_PATH + data["project"]) is False):
        os.mkdir(DOCUMENT_PATH + data["project"])

    if (os.path.isdir(DOCUMENT_PATH + data["project"] + '/' + data["category"]) is False):
        os.mkdir(DOCUMENT_PATH + data["project"] + '/' + data["category"])

    if (os.path.isdir(DOCUMENT_PATH + data["project"] + '/' + data["category"] + '/' + data["company"]) is False):
        os.mkdir(DOCUMENT_PATH + data["project"] + '/' + data["category"] + '/' + data["company"])

    #Save with counter
    dirs = os.listdir(DOCUMENT_PATH + data["project"] + '/' + data["category"] + '/' + data["company"])
    counter = len(dirs) + 1
    shutil.move (TEMP_PATH + data["id"] + ".pdf",
                DOCUMENT_PATH + data["project"] + '/' + data["category"] + '/' + data["company"] + "/" + data[
                    "date"] + "_"+ str(counter) + ".pdf")

if __name__ == "__main__":

    logging.getLogger("main").info("start")

    webapp = threading.Thread(target=webapp.run)
    webapp.daemon = True
    webapp.start()
    while True:
        logging.getLogger("main").info("check for new files")
        file_list = read_files()
        get_Info(file_list)
        sleep(5)


#   Coyright 2017-2019 Nativepython Authors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from typed_python import (
    Bool,
    Int8, Int16, Int32, Int64,
    UInt8, UInt16, UInt32, UInt64,
    Float32, Float64,
    NoneType, TupleOf, ListOf, OneOf, Tuple, NamedTuple, Dict,
    ConstDict, Alternative, serialize, deserialize, Class, Member,
    TypeFilter, Function, Forward
)

from typed_python.test_util import currentMemUsageMb
import typed_python._types as _types
import psutil
import unittest
import time
import numpy
import os

def typeFor(t):
    assert not isinstance(t, list), t
    return type(t)

def typeForSeveral(t):
    ts = set(typeFor(a) for a in t)
    if len(ts) == 1:
        return list(ts)[0]
    return OneOf(*ts)

def makeTupleOf(*args):
    if not args:
        return TupleOf(int)()
    return TupleOf(typeForSeveral(args))(args)

def makeNamedTuple(**kwargs):
    if not kwargs:
        return NamedTuple()()
    return NamedTuple(**{k: typeFor(v) for k, v in kwargs.items()})(**kwargs)

def makeTuple(*args):
    if not args:
        return Tuple()()
    return Tuple(*[typeFor(v) for v in args])(args)

def makeDict(d):
    if not d:
        return ConstDict(int, int)()

    return ConstDict(typeForSeveral(d.keys()), typeForSeveral(d.values()))(d)

def makeAlternative(severalDicts):
    types = list(
        set(
            tuple(
                (k, typeFor(v)) for k, v in ntDict.items()
            )
            for ntDict in severalDicts
        )
    )

    alt = Alternative("Alt", **{
        "a_%s" % i: dict(types[i]) for i in range(len(types))
    })

    res = []
    for thing in severalDicts:
        did = False
        for i in range(len(types)):
            try:
                res.append(getattr(alt, "a_%s" % i)(**thing))
                did = True
            except Exception:
                pass

            if did:
                break
    assert len(res) == len(severalDicts)

    return res

def choice(x):
    # numpy.random.choice([1,(1,2)]) blows up because it looks 'multidimensional'
    # so we have to pick from a list of indices
    if not isinstance(x, list):
        x = list(x)
    return x[numpy.random.choice(list(range(len(x))))]

class RandomValueProducer:
    def __init__(self):
        self.levels = {0: [b'1', b'', '2', '', 0, 1, 0.0, 1.0, None, False, True, "a ", "a string", "b string", "b str"]}

    def addEvenly(self, levels, count):
        for level in range(1, levels+1):
            self.addValues(level, count)

    def all(self):
        res = []
        for valueList in self.levels.values():
            res.extend(valueList)
        return res

    def addValues(self, level, count, sublevels=None):
        assert level > 0

        if sublevels is None:
            sublevels = list(range(level))
        sublevels = [x for x in sublevels if x in self.levels]

        assert sublevels

        def picker():
            whichLevel = choice(sublevels)
            try:
                return choice(self.levels[whichLevel])
            except Exception:
                print(self.levels[whichLevel])
                raise

        for _ in range(count):
            val = self.randomValue(picker)
            if not isinstance(val, list):
                val = [val]
            self.levels.setdefault(level, []).extend(val)

    def randomValue(self, picker):
        def randomTuple():
            return makeTuple(*[picker() for i in range(choice([0, 1, 2, 3, 4]))])

        def randomNamedTupleDict():
            return {"x_%s" % i: picker() for i in range(choice([0, 1, 2, 3, 4]))}

        def randomNamedTuple():
            return makeNamedTuple(**randomNamedTupleDict())

        def randomDict():
            return makeDict({picker(): picker() for i in range(choice([0, 1, 2, 3, 4]))})

        def randomTupleOf():
            return makeTupleOf(*[picker() for i in range(choice([0, 1, 2, 3, 4]))])

        def randomAlternative():
            return makeAlternative([randomNamedTupleDict() for i in range(choice([1, 2, 3, 4]))])

        return choice([randomTuple, randomNamedTuple, randomDict, randomTupleOf, randomAlternative, picker])()

    def pickRandomly(self):
        return choice(self.levels[choice(list(self.levels))])

class NativeTypesTests(unittest.TestCase):

    def check_expected_performance(self, elapsed, expected=1.0):
        if os.environ.get('TRAVIS_CI', None) is not None:
            expected = 2 * expected

        self.assertTrue(
            elapsed < expected,
            "Slow Performance: expected to take {expected} sec, but took {elapsed}"
            .format(expected=expected, elapsed=elapsed)
        )

    def test_objects_are_singletons(self):
        self.assertTrue(Int8 is Int8)
        self.assertTrue(NoneType is NoneType)

    def test_object_binary_compatibility(self):
        ibc = _types.isBinaryCompatible

        self.assertTrue(ibc(NoneType, NoneType))
        self.assertTrue(ibc(Int8, Int8))

        NT = NamedTuple(a=int, b=int)

        class X(NamedTuple(a=int, b=int)):
            pass

        class Y(NamedTuple(a=int, b=int)):
            pass

        self.assertTrue(ibc(X, X))
        self.assertTrue(ibc(X, Y))
        self.assertTrue(ibc(X, NT))
        self.assertTrue(ibc(Y, NT))
        self.assertTrue(ibc(NT, Y))

        self.assertFalse(ibc(OneOf(int, float), OneOf(float, int)))
        self.assertTrue(ibc(OneOf(int, X), OneOf(int, Y)))

    def test_binary_compatibility_incompatible_alternatives(self):
        ibc = _types.isBinaryCompatible

        A1 = Alternative("A1", X={'a': int}, Y={'b': float})
        A2 = Alternative("A2", X={'a': int}, Y={'b': str})

        self.assertTrue(ibc(A1, A1.X))
        self.assertTrue(ibc(A1, A1.Y))
        self.assertTrue(ibc(A1.Y, A1.Y))
        self.assertTrue(ibc(A1.Y, A1))
        self.assertTrue(ibc(A1.X, A1))
        self.assertFalse(ibc(A1.X, A1.Y))

        self.assertFalse(ibc(A1, A2))
        self.assertFalse(ibc(A1.X, A2.X))
        self.assertFalse(ibc(A1.Y, A2.Y))

    def test_binary_compatibility_compatible_alternatives(self):
        ibc = _types.isBinaryCompatible

        A1 = Alternative("A1", X={'a': int}, Y={'b': float})
        A2 = Alternative("A2", X={'a': int}, Y={'b': float})

        self.assertTrue(ibc(A1.X, A2.X))
        self.assertTrue(ibc(A1.Y, A2.Y))

        self.assertFalse(ibc(A1.X, A2.Y))
        self.assertFalse(ibc(A1.Y, A2.X))

    def test_callable_alternatives(self):
        def myCall(self):
            if self.matches.One:
                return 1
            elif self.matches.Two:
                return 2
            else:
                raise TypeError("Unexpected alternative kind")

        alt = Alternative("alts", One={}, Two={}, __call__=myCall)

        one = alt.One()
        self.assertEqual(one(), 1)

        two = alt.Two()
        self.assertEqual(two(), 2)

        alt = Alternative("alts", One={}, Two={}, myCall=myCall)
        with self.assertRaises(TypeError):
            one = alt.One()
            one()

        with self.assertRaises(TypeError):
            two = alt.Two()
            two()

    def test_callable_class(self):
        class CallableClass(Class):
            x = Member(int)

            def __call__(self, x):
                return self.x + x

            def __call__(self):  # noqa: F811
                return -1

        class RegularClass(Class):
            x = Member(int)

            def call(self, x):
                return self.x + x

        obj = CallableClass(x=42)
        self.assertEqual(obj(0), 42)
        self.assertEqual(obj(1), 43)
        self.assertEqual(obj(), -1 )

        exceptionMsg = "Cannot find a valid overload of '__call__' with arguments of type"
        with self.assertRaisesRegex(TypeError, exceptionMsg):
            obj(1, 2, 3)

        obj = RegularClass(x=42)
        self.assertEqual(obj.call(5), 47)
        with self.assertRaises(TypeError):
            obj()

    def test_object_bytecounts(self):
        self.assertEqual(_types.bytecount(NoneType), 0)
        self.assertEqual(_types.bytecount(Int8), 1)
        self.assertEqual(_types.bytecount(Int64), 8)

    def test_type_stringification(self):
        for t in ['Int8', 'NoneType']:
            self.assertEqual(str(getattr(_types, t)()), "<class '%s'>" % t)

    def test_tuple_of(self):
        tupleOfInt = TupleOf(int)
        i = tupleOfInt(())
        i = tupleOfInt((1, 2, 3))

        self.assertEqual(len(i), 3)
        self.assertEqual(tuple(i), (1, 2, 3))

        for x in range(10):
            self.assertEqual(
                tuple(tupleOfInt(tuple(range(x)))),
                tuple(range(x))
            )

        with self.assertRaisesRegex(AttributeError, "do not accept attributes"):
            tupleOfInt((1, 2, 3)).x = 2

    def test_one_of_alternative(self):
        X = Alternative("X", V={'a': int})
        Ox = OneOf(None, X)

        self.assertEqual(Ox(X.V(a=10)), X.V(a=10))

    def test_one_of_py_subclass(self):
        class X(NamedTuple(x=int)):
            def f(self):
                return self.x

        Ox = OneOf(None, X)

        self.assertEqual(NamedTuple(x=int)(x=10).x, 10)
        self.assertEqual(X(x=10).f(), 10)
        self.assertEqual(Ox(X(x=10)).f(), 10)

    def test_one_of_distinguishes_py_subclasses(self):
        class X(NamedTuple(x=int)):
            def f(self):
                return self.x

        class X2(NamedTuple(x=int)):
            def f(self):
                return self.x + 2

        XorX2 = OneOf(X, X2)

        self.assertTrue(isinstance(XorX2(X()), X))
        self.assertTrue(isinstance(XorX2(X2()), X2))

    def test_tuple_of_tuple_of(self):
        tupleOfInt = TupleOf(int)
        tupleOfTupleOfInt = TupleOf(tupleOfInt)

        pyVersion = (1, 2, 3), (1, 2, 3, 4)
        nativeVersion = tupleOfTupleOfInt(pyVersion)

        self.assertEqual(len(nativeVersion), 2)
        self.assertEqual(len(nativeVersion[0]), 3)
        self.assertEqual(tuple(tuple(x) for x in nativeVersion), pyVersion)

        bigTup = tupleOfInt(list(range(1000)))

        t0 = time.time()
        t = (bigTup, bigTup, bigTup, bigTup, bigTup)
        for i in range(1000000):
            tupleOfTupleOfInt(t)

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_default_initializer_oneof(self):
        x = OneOf(None, int)
        self.assertTrue(x() is None, repr(x()))

    def test_tuple_of_various_things(self):
        for thing, typeOfThing in [("hi", str), (b"somebytes", bytes),
                                   (1.0, float), (2, int),
                                   (None, type(None))
                                   ]:
            tupleType = TupleOf(typeOfThing)
            t = tupleType((thing,))
            self.assertTrue(type(t[0]) is typeOfThing)
            self.assertEqual(t[0], thing)

    def test_tuple_assign_fails(self):
        with self.assertRaisesRegex(TypeError, "does not support item assignment"):
            (1, 2, 3)[10] = 20
        with self.assertRaisesRegex(TypeError, "does not support item assignment"):
            TupleOf(int)((1, 2, 3))[10] = 20

    def test_list_of(self):
        L = ListOf(int)
        self.assertEqual(L.__qualname__, "ListOf(Int64)")

        l1 = L([1, 2, 3, 4])

        self.assertEqual(l1[0], 1)
        self.assertEqual(l1[-1], 4)

        l1[0] = 10
        self.assertEqual(l1[0], 10)

        l1[-1] = 11
        self.assertEqual(l1[3], 11)

        with self.assertRaisesRegex(IndexError, "index out of range"):
            l1[100] = 20

        l2 = L((10, 2, 3, 11))

        self.assertEqual(l1, l2)
        self.assertNotEqual(l1, (10, 2, 3, 11))
        self.assertEqual(l1, [10, 2, 3, 11])

        self.assertEqual(str(l1), str([10, 2, 3, 11]))

        l3 = l1 + l2
        self.assertEqual(l3, [10, 2, 3, 11, 10, 2, 3, 11])

        l3.append(23)
        self.assertEqual(l3, [10, 2, 3, 11, 10, 2, 3, 11, 23])

    def test_list_resize(self):
        l1 = ListOf(TupleOf(int))()

        l1.resize(10)
        self.assertEqual(l1.reserved(), 10)
        self.assertEqual(len(l1), 10)

        emptyTup = TupleOf(int)()
        aTup = TupleOf(int)((1, 2, 3))

        self.assertEqual(list(l1), [emptyTup] * 10)
        l1.resize(20, aTup)
        self.assertEqual(list(l1), [emptyTup] * 10 + [aTup] * 10)

        self.assertEqual(_types.refcount(aTup), 11)

        self.assertEqual(l1.pop(15), aTup)
        self.assertEqual(l1.pop(5), emptyTup)

        self.assertEqual(_types.refcount(aTup), 10)

        l1.resize(15)

        with self.assertRaises(IndexError):
            l1.pop(100)

        self.assertEqual(_types.refcount(aTup), 7)  # 6 in the list because we popped at '5'

        l1.pop()

        self.assertEqual(_types.refcount(aTup), 6)

        # this pops one of the empty tuples
        l1.pop(-10)

        self.assertEqual(_types.refcount(aTup), 6)

        l1.clear()
        self.assertEqual(len(l1), 0)

    def test_one_of(self):
        o = OneOf(None, str)

        self.assertEqual(o("hi"), "hi")
        self.assertTrue(o(None) is None)

        o = OneOf(None, "hi", 1.5, 1, True, b"hi2")

        self.assertTrue(o(None) is None)
        self.assertTrue(o("hi") == "hi")
        self.assertTrue(o(b"hi2") == b"hi2")
        self.assertTrue(o(1.5) == 1.5)
        self.assertTrue(o(1) == 1)
        self.assertIs(o(True), True)

        with self.assertRaises(TypeError):
            o("hi2")
        with self.assertRaises(TypeError):
            o(b"hi")
        with self.assertRaises(TypeError):
            o(3)
        with self.assertRaises(TypeError):
            o(False)

    def test_ordering(self):
        o = OneOf(None, "hi", 1.5, 1, True, b"hi2")

        self.assertIs(o(True), True)

    def test_one_of_flattening(self):
        self.assertEqual(OneOf(OneOf(None, 1.0), OneOf(2.0, 3.0)), OneOf(None, 1.0, 2.0, 3.0))

    def test_one_of_order_matters(self):
        self.assertNotEqual(OneOf(1.0, 2.0), OneOf(2.0, 1.0))

    def test_type_filter(self):
        EvenInt = TypeFilter(int, lambda i: i % 2 == 0)

        self.assertTrue(isinstance(2, EvenInt))
        self.assertFalse(isinstance(1, EvenInt))
        self.assertFalse(isinstance(2.0, EvenInt))

        EvenIntegers = TupleOf(EvenInt)

        e = EvenIntegers(())
        e2 = e + (2, 4, 0)

        with self.assertRaises(TypeError):
            EvenIntegers((1,))

        with self.assertRaises(TypeError):
            e2 + (1,)

    def test_tuple_of_one_of_fixed_size(self):
        t = TupleOf(OneOf(0, 1, 2, 3, 4))

        ints = tuple([x % 5 for x in range(1000000)])

        typedInts = t(ints)

        self.assertEqual(len(serialize(t, typedInts)), len(ints) * 2 + 6)  # 3 bytes for extra flags
        self.assertEqual(tuple(typedInts), ints)

    def test_tuple_of_one_of_multi(self):
        t = TupleOf(OneOf(int, bool))

        someThings = tuple([100 + x % 5 if x % 17 != 0 else bool(x%19) for x in range(1000000)])

        typedThings = t(someThings)

        self.assertEqual(
            len(serialize(t, typedThings)),
            sum(3 if isinstance(t, bool) else 4 for t in someThings) +
            2 +  # two bytes for begin / end flags
            2 +  # two bytes for the id
            2    # two bytes for the size
        )

        self.assertEqual(tuple(typedThings), someThings)

    def test_compound_oneof(self):
        producer = RandomValueProducer()
        producer.addEvenly(1000, 2)

        for _ in range(1000):
            vals = (producer.pickRandomly(), producer.pickRandomly(), producer.pickRandomly())

            a = OneOf(vals[0], vals[1], type(vals[2]))

            for v in vals:
                self.assertEqual(a(v), v, (a(v), v))

            tup = TupleOf(a)
            tupInst = tup(vals)

            for i in range(len(vals)):
                self.assertEqual(tupInst[i], vals[i], vals)

    def test_one_of_conversion_failure(self):
        o = OneOf(None, str)

        with self.assertRaises(TypeError):
            o(b"bytes")

    def test_one_of_in_tuple(self):
        t = Tuple(OneOf(None, str), str)

        self.assertEqual(t(("hi", "hi2"))[0], "hi")
        self.assertEqual(t(("hi", "hi2"))[1], "hi2")
        self.assertEqual(t((None, "hi2"))[1], "hi2")
        self.assertEqual(t((None, "hi2"))[0], None)
        with self.assertRaises(TypeError):
            t((None, None))
        with self.assertRaises(IndexError):
            t((None, "hi2"))[2]

    def test_one_of_composite(self):
        t = OneOf(TupleOf(str), TupleOf(float))

        self.assertIsInstance(t((1.0, 2.0)), TupleOf(float))
        self.assertIsInstance(t(("1.0", "2.0")), TupleOf(str))

        with self.assertRaises(TypeError):
            t((1.0, "2.0"))

    def test_named_tuple(self):
        t = NamedTuple(a=int, b=int)

        with self.assertRaisesRegex(AttributeError, "object has no attribute"):
            t().asdf

        with self.assertRaisesRegex(AttributeError, "immutable"):
            t().a = 1

        self.assertEqual(t()[0], 0)
        self.assertEqual(t().a, 0)
        self.assertEqual(t()[1], 0)

        self.assertEqual(t(a=1, b=2).a, 1)
        self.assertEqual(t(a=1, b=2).b, 2)

    def test_named_tuple_construction(self):
        t = NamedTuple(a=int, b=int)

        self.assertEqual(t(a=10).a, 10)
        self.assertEqual(t(a=10).b, 0)
        self.assertEqual(t(a=10, b=2).a, 10)
        self.assertEqual(t(a=10, b=2).b, 2)
        self.assertEqual(t({'a': 10, 'b': 2}).a, 10)
        self.assertEqual(t({'a': 10, 'b': 2}).b, 2)

        self.assertEqual(t({'b': 2}).a, 0)
        self.assertEqual(t({'b': 2}).b, 2)

        with self.assertRaises(TypeError):
            t({'c': 10})
        with self.assertRaises(TypeError):
            t(c=10)

    def test_named_tuple_str(self):
        t = NamedTuple(a=str, b=str)

        self.assertEqual(t(a='1', b='2').a, '1')
        self.assertEqual(t(a='1', b='2').b, '2')

        self.assertEqual(t(b='2').a, '')
        self.assertEqual(t(b='2').b, '2')
        self.assertEqual(t().a, '')
        self.assertEqual(t().b, '')

    def test_tuple_of_string_perf(self):
        t = NamedTuple(a=str, b=str)

        t0 = time.time()
        for i in range(1000000):
            t(a="a", b="b").a

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_comparisons_in_one_of(self):
        t = OneOf(None, float)

        def map(x):
            if x is None:
                return -1000000.0
            else:
                return x

        lt = lambda a, b: map(a) < map(b)
        le = lambda a, b: map(a) <= map(b)
        eq = lambda a, b: map(a) == map(b)
        ne = lambda a, b: map(a) != map(b)
        gt = lambda a, b: map(a) > map(b)
        ge = lambda a, b: map(a) >= map(b)

        funcs = [lt, le, eq, ne, gt, ge]
        ts = [None, 1.0, 2.0, 3.0]

        for f in funcs:
            for t1 in ts:
                for t2 in ts:
                    self.assertTrue(f(t1, t2) is f(t(t1), t(t2)))

    def test_comparisons_equivalence(self):
        t = TupleOf(OneOf(None, str, bytes, float, int, bool, TupleOf(int)),)

        def lt(a, b):
            return a < b

        def le(a, b):
            return a <= b

        def eq(a, b):
            return a == b

        def ne(a, b):
            return a != b

        def gt(a, b):
            return a > b

        def ge(a, b):
            return a >= b

        funcs = [lt, le, eq, ne, gt, ge]

        tgroups = [
            [1.0, 2.0, 3.0],
            [1, 2, 3],
            [True, False],
            ["a", "b", "ab", "bb", "ba", "aaaaaaa", "", "asdf"],
            ["1", "2", "3", "12", "13", "23", "24", "123123", "0", ""],
            [b"a", b"b", b"ab", b"bb", b"ba", b"aaaaaaa", b"", b"asdf"],
            [(1, 2), (1, 2, 3), (), (1, 1), (1,)]
        ]

        for ts in tgroups:
            for f in funcs:
                for t1 in ts:
                    for t2 in ts:
                        self.assertTrue(
                            f(t1, t2) is f(t((t1,)), t((t2,))),
                            (f, t1, t2, f(t1, t2), f(t((t1,)), t((t2,))))
                        )

    def test_const_dict(self):
        t = ConstDict(str, str)

        self.assertEqual(len(t()), 0)
        self.assertEqual(len(t({})), 0)
        self.assertEqual(len(t({'a': 'b'})), 1)
        self.assertEqual(t({'a': 'b'})['a'], 'b')
        self.assertEqual(t({'a': 'b', 'b': 'c'})['b'], 'c')

        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b'}))))

        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c'}))))
        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd'}))))
        self.assertTrue("a" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd', 'd': 'e'}))))
        self.assertTrue("c" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd', 'def': 'e'}))))
        self.assertTrue("def" in deserialize(t, serialize(t, t({'a': 'b', 'b': 'c', 'c': 'd', 'def': 'e'}))))

    def test_const_dict_get(self):
        a = ConstDict(str, str)({'a': 'b', 'c': 'd'})

        self.assertEqual(a.get('a'), 'b')
        self.assertEqual(a.get('asdf'), None)
        self.assertEqual(a.get('asdf', 20), 20)

    def test_const_dict_items_keys_and_values(self):
        a = ConstDict(str, str)({'a': 'b', 'c': 'd'})

        self.assertEqual(sorted(a.items()), [('a', 'b'), ('c', 'd')])
        self.assertEqual(sorted(a.keys()), ['a', 'c'])
        self.assertEqual(sorted(a.values()), ['b', 'd'])

    def test_empty_string(self):
        a = ConstDict(str, str)({'a': ''})

        print(a['a'])

    def test_dict_to_oneof(self):
        t = ConstDict(str, OneOf("A", "B", "ABCDEF"))
        a = t({'a': 'A', 'b': 'ABCDEF'})

        self.assertEqual(a['a'], "A")
        self.assertEqual(a['b'], "ABCDEF")

        self.assertEqual(a, deserialize(t, serialize(t, a)))

    def test_deserialize_primitive(self):
        x = deserialize(str, serialize(str, "a"))
        self.assertTrue(isinstance(x, str))

    def test_dict_containment(self):
        for _ in range(100):
            producer = RandomValueProducer()
            producer.addEvenly(20, 2)

            values = producer.all()

            for v in values:
                if str(type(v))[:17] == "<class 'ConstDict":
                    v = deserialize(type(v), serialize(type(v), v))
                    for k in v:
                        self.assertTrue(k in v)

    def test_named_tuple_from_dict(self):
        N = NamedTuple(x=int, y=str, z=OneOf(None, "hihi"))
        self.assertEqual(N().x, 0)
        self.assertEqual(N().y, "")
        self.assertEqual(N().z, None)

        self.assertEqual(N({}).x, 0)
        self.assertEqual(N({}).y, "")
        self.assertEqual(N({}).z, None)

        self.assertEqual(N({'x': 20}).x, 20)
        self.assertEqual(N({'x': 20, 'y': "30"}).y, "30")
        self.assertEqual(N({'y': "30", 'x': 20}).y, "30")
        self.assertEqual(N({'z': "hihi"}).z, "hihi")

        with self.assertRaises(Exception):
            N({'r': 'hi'})
            N({'y': 'hi', 'z': "not hihi"})
            N({'a': 0, 'b': 0, 'c': 0, 'd': 0})

    def test_const_dict_mixed(self):
        t = ConstDict(str, int)
        self.assertTrue(t({"a": 10})["a"] == 10)

        t = ConstDict(int, str)
        self.assertTrue(t({10: "a"})[10] == "a")

    def test_const_dict_comparison(self):
        t = ConstDict(str, str)

        self.assertEqual(t({'a': 'b'}), t({'a': 'b'}))
        self.assertLess(t({}), t({'a': 'b'}))

    def test_const_dict_lookup(self):
        for type_to_use, vals in [(int, list(range(20))),
                                  (bytes, [b'1', b'2', b'3', b'4', b'5'])]:
            t = ConstDict(type_to_use, type_to_use)

            for _ in range(10):
                ks = list(vals)
                vs = list(vals)

                numpy.random.shuffle(ks)
                numpy.random.shuffle(vs)

                py_d = {}
                for i in range(len(ks)):
                    py_d[ks[i]] = vs[i]

                typed_d = t(py_d)

                for k in py_d:
                    self.assertEqual(py_d[k], typed_d[k])

                last_k = None
                for k in typed_d:
                    assert last_k is None or k > last_k, (k, last_k)
                    last_k = k

    def test_const_dict_lookup_time(self):
        int_dict = ConstDict(int, int)

        d = int_dict({k: k for k in range(1000000)})

        for k in range(1000000):
            self.assertTrue(k in d)
            self.assertTrue(d[k] == k)

    def test_const_dict_of_dict(self):
        int_dict = ConstDict(int, int)
        int_dict_2 = ConstDict(int_dict, int_dict)

        d = int_dict({1: 2})
        d2 = int_dict({1: 2, 3: 4})

        big = int_dict_2({d: d2})

        self.assertTrue(d in big)
        self.assertTrue(d2 not in big)
        self.assertTrue(big[d] == d2)

    def test_dict_hash_perf(self):
        str_dict = ConstDict(str, str)

        s = str_dict({'a' * 1000000: 'b' * 1000000})

        t0 = time.time()
        for k in range(1000000):
            hash(s)

        elapsed = time.time() - t0
        print(elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_mutable_dict_not_hashable(self):
        with self.assertRaisesRegex(Exception, "not hashable"):
            hash(Dict(int, int)())

    def test_const_dict_str_perf(self):
        t = ConstDict(str, str)

        t0 = time.time()
        for i in range(100000):
            t({str(k): str(k+1) for k in range(10)})

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_const_dict_int_perf(self):
        t = ConstDict(int, int)

        t0 = time.time()
        for i in range(100000):
            t({k: k+1 for k in range(10)})

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed)

    def test_const_dict_iter_int(self):
        t = ConstDict(int, int)

        aDict = t({k: k+1 for k in range(100)})
        for k in aDict:
            self.assertEqual(aDict[k], k+1)

    def test_const_dict_iter_str(self):
        t = ConstDict(str, str)

        aDict = t({str(k): str(k+1) for k in range(100)})
        for k in aDict:
            self.assertEqual(aDict[str(k)], str(int(k)+1))

    def test_alternative_bytecounts(self):
        alt = Alternative(
            "Empty",
            X={},
            Y={}
        )

        self.assertEqual(_types.bytecount(alt), 1)
        self.assertEqual(_types.bytecount(alt.X), 1)
        self.assertEqual(_types.bytecount(alt.Y), 1)

    def test_alternatives_with_Bytes(self):
        alt = Alternative(
            "Alt",
            x_0={'a': bytes}
        )
        self.assertEqual(alt.x_0(a=b''), alt.x_0(a=b''))

    def test_alternatives_with_str_func(self):
        alt = Alternative(
            "Alt",
            x_0={'a': bytes},
            f=lambda self: 1,
            __str__=lambda self: "not_your_usual_str"
        )

        self.assertEqual(alt.x_0().f(), 1)
        self.assertEqual(str(alt.x_0()), "not_your_usual_str")

    def test_named_tuple_subclass_magic_methods(self):
        class X(NamedTuple(x=int, y=int)):
            def __str__(self):
                return "str override"

            def __repr__(self):
                return "repr override"

            def __call__(self):
                return "call implemented"

        self.assertEqual(repr(X()), "repr override")
        self.assertEqual(str(X()), "str override")
        self.assertEqual(X()(), "call implemented")

    def test_empty_alternatives(self):
        a = Alternative(
            "Alt",
            A={},
            B={}
        )

        self.assertEqual(a.A(), a.A())
        self.assertIsInstance(deserialize(a, serialize(a, a.A())), a.A)
        self.assertEqual(a.A(), deserialize(a, serialize(a, a.A())))

        self.assertEqual(a.B(), a.B())
        self.assertNotEqual(a.A(), a.B())
        self.assertNotEqual(a.B(), a.A())

    def test_extracted_alternatives_have_correct_type(self):
        Alt = Alternative(
            "Alt",
            A={},
            B={}
        )
        tOfAlt = TupleOf(Alt)

        a = Alt.A()
        aTup = tOfAlt((a,))

        self.assertEqual(a, aTup[0])
        self.assertTrue(type(a) is type(aTup[0]))  # noqa: F721

    def test_alternatives(self):
        alt = Alternative(
            "Alt",
            child_ints={'x': int, 'y': int},
            child_strings={'x': str, 'y': str}
        )

        self.assertTrue(issubclass(alt.child_ints, alt))
        self.assertTrue(issubclass(alt.child_strings, alt))

        a = alt.child_ints(x=10, y=20)
        a2 = alt.child_ints(x=10, y=20)

        self.assertEqual(a, a2)

        self.assertTrue(isinstance(a, alt))
        self.assertTrue(isinstance(a, alt.child_ints))

        self.assertEqual(a.x, 10)
        self.assertEqual(a.y, 20)
        self.assertTrue(a.matches.child_ints)
        self.assertFalse(a.matches.child_strings)

        with self.assertRaisesRegex(AttributeError, "immutable"):
            a.x = 20

    def test_alternatives_comparison(self):
        empty = Alternative("X", A={}, B={})

        self.assertEqual(empty.A(), empty.A())
        self.assertEqual(empty.B(), empty.B())
        self.assertNotEqual(empty.A(), empty.B())

        a = Alternative(
            "X",
            A={'a': int},
            B={'b': int},
            C={'c': str},
            D={'d': bytes},
        )

        self.assertEqual(a.A(a=10), a.A(a=10))
        self.assertNotEqual(a.A(a=10), a.A(a=11))

        self.assertNotEqual(a.C(c=""), a.C(c="hi"))
        self.assertFalse(a.C(c="") == a.C(c="hi"))
        self.assertNotEqual(a.D(d=b""), a.D(d=b"hi"))

    def test_alternatives_add_operator(self):
        alt = Alternative(
            "Alt",
            child_ints={'x': int, 'y': int},
            __add__=lambda lhs, rhs: (lhs, rhs)
        )

        a = alt.child_ints(x=0, y=2)

        self.assertEqual(a+a, (a, a))

    def test_alternatives_perf(self):
        alt = Alternative(
            "Alt",
            child_ints={'x': int, 'y': int},
            child_strings={'x': str, 'y': str}
        )

        t0 = time.time()

        for i in range(1000000):
            a = alt.child_ints(x=10, y=20)
            a.matches.child_ints
            a.x

        elapsed = time.time() - t0
        print("Took ", elapsed, " to do 1mm")
        self.check_expected_performance(elapsed, expected=2.0)

    def test_object_hashing_and_equality(self):
        for _ in range(100):
            producer = RandomValueProducer()
            producer.addEvenly(20, 2)

            values = producer.all()

            for v1 in values:
                for v2 in values:
                    if hash(v1) != hash(v2) and v1 == v2:
                        print(v1, v2, type(v1), type(v2))

            for v1 in values:
                for v2 in values:
                    if type(v1) == type(v2) and v1 == v2:
                        self.assertEqual(hash(v1), hash(v2), (v1, v2))
                        if type(v1) is type(v2):
                            self.assertEqual(repr(v1), repr(v2), (v1, v2, type(v1), type(v2)))

            values = sorted([makeTuple(v) for v in values])

            for i in range(len(values)-1):
                self.assertTrue(values[i] <= values[i+1])
                self.assertTrue(values[i+1] >= values[i])

    def test_bytes_repr(self):
        for _ in range(100000):
            # always start with a '"' because otherwise python keeps chosing different
            # initial characters.
            someBytes = b'"' + numpy.random.uniform(size=2).tostring()
            self.assertEqual(repr(makeTuple(someBytes)), repr((someBytes,)))

    def test_equality_with_native_python_objects(self):
        tups = [(1, 2, 3), (), ("2",), (b"2",), (1, 2, 3, "b"), (2,), (None,)]

        for tup1 in tups:
            self.assertEqual( makeTuple(*tup1), tup1 )

            for tup2 in tups:
                if tup1 != tup2:
                    self.assertNotEqual( makeTuple(*tup1), tup2 )

        for tup1 in tups:
            self.assertEqual( makeTupleOf(*tup1), tup1 )

            for tup2 in tups:
                if tup1 != tup2:
                    self.assertNotEqual( makeTupleOf(*tup1), tup2 )

    def test_add_tuple_of(self):
        tupleOfInt = TupleOf(int)

        tups = [(), (1, 2), (1,), (1, 2, 3, 4)]

        for tup1 in tups:
            for tup2 in tups:
                self.assertEqual(tupleOfInt(tup1) + tupleOfInt(tup2), tupleOfInt(tup1+tup2))
                self.assertEqual(tupleOfInt(tup1) + tup2, tupleOfInt(tup1+tup2))

    def test_slice_tuple_of(self):
        tupleOfInt = TupleOf(int)

        ints = tuple(range(20))
        aTuple = tupleOfInt(ints)

        for i in range(-21, 21):
            for i2 in range(-21, 21):
                for step in range(-3, 3):
                    if step != 0:
                        self.assertEqual(aTuple[i:i2:step], ints[i:i2:step])

            try:
                ints[i]
                self.assertEqual(aTuple[i], ints[i])
            except IndexError:
                with self.assertRaises(IndexError):
                    aTuple[i]

    def test_dictionary_subtraction_basic(self):
        intDict = ConstDict(int, int)

        self.assertEqual(intDict({1: 2}) - (1,), intDict({}))
        self.assertEqual(intDict({1: 2, 3: 4}) - (1,), intDict({3: 4}))
        self.assertEqual(intDict({1: 2, 3: 4}) - (3,), intDict({1: 2}))

    def test_dictionary_addition_and_subtraction(self):
        someDicts = [{i: choice([1, 2, 3, 4, 5]) for i in range(choice([4, 6, 10, 20]))} for _ in range(20)]
        intDict = ConstDict(int, int)

        for d1 in someDicts:
            for d2 in someDicts:
                addResult = dict(d1)
                addResult.update(d2)

                self.assertEqual(intDict(d1) + intDict(d2), intDict(addResult))

                res = intDict(addResult)

                while len(res):
                    toRemove = []

                    for i in range(choice(list(range(len(res))))+1):
                        key = choice(list(addResult))
                        del addResult[key]
                        toRemove.append(key)

                    res = res - toRemove

                    self.assertEqual(res, intDict(addResult))

    def test_subclassing(self):
        BaseTuple = NamedTuple(x=int, y=float)

        class NTSubclass(BaseTuple):
            def f(self):
                return self.x + self.y

            def __repr__(self):
                return "ASDF"

        inst = NTSubclass(x=10, y=20)

        self.assertTrue(isinstance(inst, BaseTuple))
        self.assertTrue(isinstance(inst, NTSubclass))
        self.assertTrue(type(inst) is NTSubclass)

        self.assertEqual(repr(inst), "ASDF")
        self.assertNotEqual(BaseTuple.__repr__(inst), "ASDF")

        self.assertEqual(inst.x, 10)
        self.assertEqual(inst.f(), 30)

        TupleOfSubclass = TupleOf(NTSubclass)

        instTup = TupleOfSubclass((inst, BaseTuple(x=20, y=20.0)))

        self.assertTrue(isinstance(instTup[0], NTSubclass))
        self.assertTrue(isinstance(instTup[1], NTSubclass))
        self.assertEqual(instTup[0].f(), 30)
        self.assertEqual(instTup[1].f(), 40)

        self.assertEqual(BaseTuple(inst).x, 10)

        self.assertTrue(OneOf(None, NTSubclass)(None) is None)
        self.assertTrue(OneOf(None, NTSubclass)(inst) == inst)

    def test_serialization_primitives(self):
        def checkCanSerialize(x):
            self.assertEqual(x, deserialize(type(x), serialize(type(x), x)), x)

        checkCanSerialize(0)
        checkCanSerialize(1)
        checkCanSerialize(2)
        checkCanSerialize(4)
        checkCanSerialize(8)
        checkCanSerialize(16)
        checkCanSerialize(32)
        checkCanSerialize(64)
        checkCanSerialize(128)
        checkCanSerialize(-1)
        checkCanSerialize(290)
        checkCanSerialize(1000)
        checkCanSerialize(99.5)
        checkCanSerialize("hi")
        checkCanSerialize(b"bye")
        checkCanSerialize(None)
        checkCanSerialize(True)
        checkCanSerialize(False)

    def test_serialization_bytecounts(self):
        ints = TupleOf(int)((1, 2, 3, 4))

        def varintBytecount(value):
            """the length (in bytes) of a varint"""
            res = 1
            while value >= 128:
                res += 1
                value /= 128
            return res

        while len(ints) < 1000000:
            ints = ints + ints
            t0 = time.time()

            expectedBytecount = (
                sum(varintBytecount(0) + varintBytecount(i) for i in ints) +
                (varintBytecount(0) * 3) + varintBytecount(len(ints))
            )

            self.assertEqual(len(serialize(TupleOf(int), ints)), expectedBytecount)

            print(time.time() - t0, " for ", len(ints))

    def test_serialization_roundtrip(self):
        for _ in range(100):
            producer = RandomValueProducer()
            producer.addEvenly(30, 3)

            values = producer.all()
            for v in values:
                ser = serialize(type(v), v)

                v2 = deserialize(type(v), ser)

                ser2 = serialize(type(v), v2)

                self.assertTrue(type(v2) is type(v))
                self.assertEqual(ser, ser2)
                self.assertEqual(str(v), str(v2))
                self.assertEqual(v, v2, (v, v2, type(v), type(v2), type(v) is type(v2)))

    def test_create_invalid_tuple(self):
        with self.assertRaises(TypeError):
            Tuple((int, int))

    def test_roundtrip_tuple(self):
        T = Tuple(str, bool, str)
        v = T(('1', False, ''))

        v2 = deserialize(T, serialize(T, v))

        self.assertEqual(v, v2)

    def test_roundtrip_alternative(self):
        A = Alternative("A", a0=dict(x_0=None))
        T = NamedTuple(x0=A, x1=bool)

        v = T(x0=A.a0(), x1=True)

        v2 = deserialize(T, serialize(T, v))

        self.assertEqual(v, v2)

    def test_serialize_doesnt_leak(self):
        T = TupleOf(int)

        def getMem():
            return psutil.Process().memory_info().rss / 1024 ** 2

        m0 = getMem()

        for passIx in range(100):
            for i in range(1000):
                t = T(list(range(i)))
                deserialize(T, serialize(T, t))

            self.assertTrue(getMem() < m0 + 100)

    def test_const_dict_of_tuple(self):
        K = NamedTuple(a=OneOf(float, int), b=OneOf(float, int))
        someKs = [K(a=0, b=0), K(a=1), K(a=10), K(b=10), K()]

        T = ConstDict(K, K)

        indexDict = {}
        x = T()

        numpy.random.seed(42)

        for _ in range(100):
            i1 = numpy.random.choice(len(someKs))
            i2 = numpy.random.choice(len(someKs))
            add = numpy.random.choice([False, True])

            if add:
                indexDict[i1] = i2
                x = x + {someKs[i1]: someKs[i2]}
            else:
                if i1 in indexDict:
                    del indexDict[i1]
                    x = x - (someKs[i1],)

            self.assertEqual(x, T({someKs[i]: someKs[v] for i, v in indexDict.items()}))
            for k in x:
                self.assertTrue(k in x)
                x[k]

    def test_conversion_of_binary_compatible(self):
        class T1(NamedTuple(a=int)):
            pass

        class T2(NamedTuple(a=int)):
            pass

        class T1Comp(NamedTuple(d=ConstDict(str, T1))):
            pass

        class T2Comp(NamedTuple(d=ConstDict(str, T1))):
            pass

        self.assertTrue(_types.isBinaryCompatible(T1Comp, T2Comp))
        self.assertTrue(_types.isBinaryCompatible(T1, T2))

    def test_binary_compatible_nested(self):
        def make():
            class Interior(NamedTuple(a=int)):
                pass

            class Exterior(NamedTuple(a=Interior)):
                pass

            return Exterior

        E1 = make()
        E2 = make()

        self.assertTrue(_types.isBinaryCompatible(E1, E2))

    def test_python_objects_in_tuples(self):
        class NormalPyClass(object):
            pass

        class NormalPySubclass(NormalPyClass):
            pass

        NT = NamedTuple(x=NormalPyClass, y=NormalPySubclass)

        nt = NT(x=NormalPyClass(), y=NormalPySubclass())
        self.assertIsInstance(nt.x, NormalPyClass)
        self.assertIsInstance(nt.y, NormalPySubclass)

    def test_construct_alternatives_with_positional_arguments(self):
        a = Alternative("A", HasOne={'a': str}, HasTwo={'a': str, 'b': str})

        with self.assertRaises(TypeError):
            a.HasTwo("hi")

        self.assertEqual(a.HasOne("hi"), a.HasOne(a="hi"))

        hasOne = a.HasOne("hi")
        self.assertEqual(a.HasOne(hasOne), hasOne)

        with self.assertRaises(TypeError):
            a.HasOne(a.HasTwo(a='1', b='b'))

    def test_recursive_classes_repr(self):
        A0 = Forward("A0")

        class ASelfRecursiveClass(Class):
            x = Member(OneOf(None, A0))

        A0 = A0.define(ASelfRecursiveClass)

        a = ASelfRecursiveClass()
        a.x = a

        b = ASelfRecursiveClass()
        b.x = b

        print(repr(a))

    def test_unsafe_pointers_to_list_internals(self):
        x = ListOf(int)()
        x.resize(100)
        for i in range(len(x)):
            x[i] = i

        aPointer = x.pointerUnsafe(0)
        self.assertTrue(str(aPointer).startswith("(Int64*)0x"))

        self.assertEqual(aPointer.get(), x[0])
        aPointer.set(100)
        self.assertEqual(aPointer.get(), 100)
        self.assertEqual(x[0], 100)

        aPointer = aPointer + 10

        self.assertEqual(aPointer.get(), x[10])
        self.assertEqual(aPointer[10], x[20])
        aPointer.set(20)
        self.assertEqual(aPointer.get(), 20)
        self.assertEqual(x[10], 20)

        # this is OK because ints are POD.
        aPointer.initialize(30)
        self.assertEqual(x[10], 30)

    def test_unsafe_pointers_to_uninitialized_list_items(self):
        # because this is testing unsafe operations, the test is
        # really just that we don't segfault!
        for _ in range(100):
            x = ListOf(TupleOf(int))()
            x.reserve(10)

            for i in range(x.reserved()):
                x.pointerUnsafe(i).initialize((i,))

            x.setSizeUnsafe(10)

        # now check that if we fail to set the size we'll leak the tuple
        aLeakedTuple = TupleOf(int)((1, 2, 3))
        x = ListOf(TupleOf(int))()
        x.reserve(1)
        x.pointerUnsafe(0).initialize(aLeakedTuple)
        x = None

        self.assertEqual(_types.refcount(aLeakedTuple), 2)

    def test_list_extend(self):
        LI = ListOf(int)
        LF = ListOf(float)

        li = LI([1, 2, 3])
        lf = LF([1.5, 2.5, 3.5])

        li.extend(li)
        self.assertEqual(li, [1, 2, 3, 1, 2, 3])

        lf.extend(lf)
        self.assertEqual(lf, [1.5, 2.5, 3.5, 1.5, 2.5, 3.5])

        lf.extend(li)
        self.assertEqual(lf, [1.5, 2.5, 3.5, 1.5, 2.5, 3.5, 1, 2, 3, 1, 2, 3])

        li = LI()
        li.extend(range(10))

        self.assertEqual(li, list(range(10)))

    def test_list_copy_operation_duplicates_list(self):
        T = ListOf(int)

        x = T([1, 2, 3])
        y = T(x)

        x[0] = 100

        self.assertNotEqual(y[0], 100)

    def test_list_and_tuple_conversion_to_numpy(self):
        for T in [ListOf(bool), TupleOf(bool)]:
            for arr in [
                    numpy.array([]),
                    numpy.array([0, 1, 2, 3, 4, 5]),
                    numpy.array([0, 1, 2, 3, 4, 5], 'int32'),
                    numpy.array([0, 1, 2, 3, 4, 5], 'int16'),
                    numpy.array([0, 1, 2, 3, 4, 5], 'bool')
            ]:
                self.assertEqual(T(arr), T(arr.tolist()))
                self.assertEqual(T(arr).toArray().tolist(), [bool(x) for x in arr.tolist()])

        for T in [ListOf(int), TupleOf(int)]:
            for arr in [
                    numpy.array([]),
                    numpy.array([1, 2, 3, 4, 5]),
                    numpy.array([1, 2, 3, 4, 5], 'int32'),
                    numpy.array([1, 2, 3, 4, 5], 'int16')
            ]:
                self.assertEqual(T(arr), T(arr.tolist()))
                self.assertEqual(T(arr).toArray().tolist(), arr.tolist())

        for T in [ListOf(float), TupleOf(float)]:
            for arr in [
                    numpy.array([]),
                    numpy.array([1, 2, 3, 4, 5], 'float64'),
                    numpy.array([1, 2, 3, 4, 5], 'float32')
            ]:
                self.assertEqual(T(arr), T(arr.tolist()))
                self.assertEqual(T(arr).toArray().tolist(), arr.tolist())

        self.assertEqual(str(ListOf(Int64)([1, 2, 3, 4]).toArray().dtype), 'int64')
        self.assertEqual(str(ListOf(Int32)([1, 2, 3, 4]).toArray().dtype), 'int32')
        self.assertEqual(str(ListOf(Int16)([1, 2, 3, 4]).toArray().dtype), 'int16')
        self.assertEqual(str(ListOf(Int8)([1, 2, 3, 4]).toArray().dtype), 'int8')

        self.assertEqual(str(ListOf(UInt64)([1, 2, 3, 4]).toArray().dtype), 'uint64')
        self.assertEqual(str(ListOf(UInt32)([1, 2, 3, 4]).toArray().dtype), 'uint32')
        self.assertEqual(str(ListOf(UInt16)([1, 2, 3, 4]).toArray().dtype), 'uint16')
        self.assertEqual(str(ListOf(UInt8)([1, 2, 3, 4]).toArray().dtype), 'uint8')

        self.assertEqual(str(ListOf(Float64)([1, 2, 3, 4]).toArray().dtype), 'float64')
        self.assertEqual(str(ListOf(Float32)([1, 2, 3, 4]).toArray().dtype), 'float32')

    def test_list_of_equality(self):
        x = ListOf(int)([1, 2, 3, 4])
        y = ListOf(int)([1, 2, 3, 5])

        self.assertEqual(x, x)
        self.assertNotEqual(x, y)

    def test_tuple_r_add(self):
        self.assertEqual(
            (1, 2, 4, 5, 6) + TupleOf(int)([1, 2]),
            (1, 2, 4, 5, 6, 1, 2)
        )

        self.assertEqual(
            [1, 2, 4, 5, 6] + TupleOf(int)([1, 2]),
            (1, 2, 4, 5, 6, 1, 2)
        )

        with self.assertRaises(TypeError):
            [1, 2, "hi", 5, 6] + TupleOf(int)([1, 2])

    def test_tuple_r_cmp(self):
        self.assertEqual(
            (1, 2, 3), TupleOf(int)([1, 2, 3])
        )

    def test_can_convert_numpy_scalars(self):
        self.assertEqual(OneOf(int, float)(numpy.int64(10)), 10)
        self.assertEqual(OneOf(int, float)(numpy.float64(10.5)), 10.5)

    def test_other_bitness_types(self):
        # verify we can cast around non-64-bit values in a way that matches numpy
        typeAndNumpyType = [
            (Bool, numpy.bool),
            (Int8, numpy.int8),
            (Int16, numpy.int16),
            (Int32, numpy.int32),
            (Int64, numpy.int64),
            (UInt8, numpy.uint8),
            (UInt16, numpy.uint16),
            (UInt32, numpy.uint32),
            (UInt64, numpy.uint64),
            (Float32, numpy.float32),
            (Float64, numpy.float64)
        ]

        for ourType, numpyType in typeAndNumpyType:
            for candValue in [-1, 0, 1, 10, 100, 1000, 100000, 10000000, 10000000000]:
                self.assertEqual(int(ourType(candValue)), int(numpyType(candValue)), (ourType, candValue))
                self.assertEqual(float(ourType(candValue)), float(numpyType(candValue)), (ourType, candValue))

            for ourType2, numpyType2 in typeAndNumpyType:
                zeroOrTwoFloatTypes = sum([1 if 'float' in str(t) else 0 for t in [numpyType, numpyType2]]) in [0, 2]

                if zeroOrTwoFloatTypes:
                    for candValue in [-1, 0, 1, 10, 100, 1000, 100000, 10000000, 10000000000]:
                        self.assertEqual(
                            int(ourType(ourType2(candValue))),
                            int(numpyType(numpyType2(candValue))),
                            (ourType, ourType2, candValue)
                        )
                        self.assertEqual(
                            float(ourType(ourType2(candValue))),
                            float(numpyType(numpyType2(candValue))),
                            (ourType, ourType2, candValue)
                        )
                else:
                    # we convert from float to int as c++, which is different than numpy, which clips
                    # floats in a bizarre way. e.g.
                    #  numpy.int16(numpy.int64(numpy.float64(10000000000)))
                    # is not
                    #  numpy.int16(numpy.float64(10000000000))
                    pass

    def test_other_bitness_types_operators(self):

        def add(x, y):
            return x+y

        def div(x, y):
            return x/y

        def mul(x, y):
            return x*y

        def sub(x, y):
            return x-y

        def bitand(x, y):
            return x&y

        def bitor(x, y):
            return x|y

        def bitxor(x, y):
            return x^y

        otherTypes = [Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float32, Float64]
        for t1 in otherTypes:
            for t2 in otherTypes:
                for op in [add, mul, div, sub, bitand, bitor, bitxor]:
                    if not ((t1.IsFloat or t2.IsFloat) and op in (bitand, bitor, bitxor)):
                        res = op(t1(10), t2(10))
                        resType = type(res)
                        resType = {bool: Bool, int: Int64, float: Float64}.get(resType, resType)

                        if t1.IsFloat and t2.IsFloat:
                            self.assertTrue(resType.IsFloat)
                            self.assertEqual(resType.Bits, max(t1.Bits, t2.Bits))
                            self.assertEqual(res, op(10, 10))
                        elif t1.IsFloat or t2.IsFloat:
                            self.assertTrue(resType.IsFloat)
                            self.assertEqual(resType.Bits, t1.Bits if t1.IsFloat else t2.Bits)
                            if t1.Bits > 1 and t2.Bits > 1:
                                self.assertEqual(res, op(10, 10))
                        elif t1 is Bool and t2 is Bool:
                            self.assertEqual(resType, Bool if op in (bitor, bitand, bitxor) else Int64 if op is not div else Float64)
                        else:
                            self.assertEqual(resType.Bits, max(t1.Bits, t2.Bits))

                            if op is not div:
                                self.assertEqual(resType.IsSignedInt, t1.IsSignedInt or t2.IsSignedInt)

                            if t1.Bits > 1 and t2.Bits > 1:
                                self.assertEqual(res, op(10, 10))

    def test_comparing_arbitrary_objects(self):
        x = TupleOf(object)(["a"])
        y = TupleOf(object)([1])

        with self.assertRaises(TypeError):
            x < y

        self.assertEqual(x, x)
        self.assertEqual(y, y)
        self.assertNotEqual(x, y)

    def test_list_of_indexing_with_numpy_ints(self):
        x = ListOf(ListOf(int))([[1, 2, 3], [4, 5, 6]])
        self.assertEqual(x[numpy.int64(0)][numpy.int64(0)], 1)

    def test_dispatch_tries_without_conversion_first(self):
        class ClassWithForcedConversion(Class):
            def f(self, x: float):
                return "float"

        class ClassWithBoth(Class):
            def f(self, x: float):
                return "float"

            def f(self, x: int):  # noqa: F811
                return "int"

            def f(self, x: bool):  # noqa: F811
                return "bool"

        # swap the order
        class ClassWithBoth2(Class):
            def f(self, x: bool):
                return "bool"

            def f(self, x: int):  # noqa: F811
                return "int"

            def f(self, x: float):  # noqa: F811
                return "float"

        self.assertEqual(ClassWithForcedConversion().f(10), "float")
        self.assertEqual(ClassWithForcedConversion().f(10.5), "float")
        self.assertEqual(ClassWithForcedConversion().f(True), "float")

        self.assertEqual(ClassWithBoth().f(10), "int")
        self.assertEqual(ClassWithBoth().f(10.5), "float")
        self.assertEqual(ClassWithBoth().f(True), "bool")

        self.assertEqual(ClassWithBoth2().f(10), "int")
        self.assertEqual(ClassWithBoth2().f(10.5), "float")
        self.assertEqual(ClassWithBoth2().f(True), "bool")

    def test_error_message_on_bad_dispatch(self):
        @Function
        def f(x: int):
            return x

        with self.assertRaisesRegex(TypeError, "str"):
            f("hi")

        with self.assertRaisesRegex(TypeError, "argname="):
            f(argname=1)

    def test_named_tuple_comparison(self):
        N = NamedTuple(x=OneOf(None, int), y=OneOf(None, int))

        class S(N):
            pass

        self.assertEqual(N(x=1, y=2), N(x=1, y=2))
        self.assertNotEqual(N(x=1, y=2), N(x=1, y=3))
        self.assertFalse(N(x=1, y=2) == N(x=1, y=3))

        self.assertEqual(S(x=1, y=2), S(x=1, y=2))
        self.assertNotEqual(S(x=1, y=2), S(x=1, y=3))
        self.assertFalse(S(x=1, y=2) == S(x=1, y=3))

    def test_const_dict_comparison_more(self):
        N = NamedTuple(x=OneOf(None, int), y=OneOf(None, int))
        D = ConstDict(str, N)

        n1 = N(x=1, y=2)
        n2 = N(x=1, y=3)

        self.assertEqual(D({'a': n1}), D({'a': n1}))
        self.assertNotEqual(D({'a': n1}), D({'a': n2}))

    def test_mutable_dict(self):
        T = Dict(int, int)

        d = T()

        self.assertEqual(len(d), 0)

        with self.assertRaises(KeyError):
            d[0]

        d[0] = 10

        self.assertEqual(len(d), 1)
        self.assertEqual(d[0], 10)

        d[0] = 20

        self.assertEqual(len(d), 1)
        self.assertEqual(d[0], 20)

        for i in range(2000):
            d[i] = i

            if i % 100 == 0 and i:
                for x in range(len(d)):
                    self.assertEqual(d[x], x)

        for i in range(2000):
            if i % 100 == 0 and i:
                for x in range(i):
                    assert x not in d

                for x in range(i, 2000):
                    self.assertEqual(d[x], x)

            del d[i]

        # verify that adding and removing elements doesn't leak memory
        usage = currentMemUsageMb()
        for i in range(1000000):
            d[0] = i
            del d[0]
        self.assertLess(currentMemUsageMb(), usage+1)

    def test_mutable_dict_fuzz(self):
        native_d = Dict(int, int)()
        py_d = {}

        for dictSize in [10, 100, 1000, 10000]:
            for i in range(100000):
                z = numpy.random.choice(dictSize)

                self.assertEqual(z in py_d, z in native_d)

                if i % 3 == 0 or (i % 1000) > 900:
                    if z in py_d:
                        del py_d[z]
                        del native_d[z]
                else:
                    py_d[z] = i
                    native_d[z] = i

            for i in range(dictSize):
                self.assertEqual(z in py_d, z in native_d)

    def test_mutable_dict_refcounts(self):
        native_d = Dict(str, ListOf(int))()
        i = ListOf(int)()

        native_d["a"] = i

        self.assertEqual(_types.refcount(i), 2)

        native_d["b"] = i

        self.assertEqual(_types.refcount(i), 3)

        del native_d["a"]

        self.assertEqual(_types.refcount(i), 2)

        native_d["b"] = ListOf(int)()

        self.assertEqual(_types.refcount(i), 1)

        native_d["b"] = i

        self.assertEqual(_types.refcount(i), 2)

        native_d = None

        self.assertEqual(_types.refcount(i), 1)

    def test_mutable_dict_create_many(self):
        for ct in range(100):
            d = Dict(int, int)()
            for i in range(ct):
                d[i] = i + 1

    def test_mutable_dict_methods(self):
        d = Dict(int, int)({i: i+1 for i in range(10)})

        self.assertEqual(list(d.keys()), list(range(10)))
        self.assertEqual(list(d.values()), list(range(1, 11)))
        self.assertEqual(list(d.items()), [(i, i+1) for i in range(10)])

        for i in range(10):
            self.assertEqual(d.get(i), i+1)
            self.assertEqual(d.get(i, None), i+1)

        self.assertEqual(d.get(1000), None)
        self.assertEqual(d.get(1000, 123), 123)

        with self.assertRaises(TypeError):
            self.assertEqual(d.get("1000"), None)

    def test_mutable_dict_iteration_order(self):
        d = Dict(int, int)()

        d[10] = 10
        d[1] = 1
        d[2] = 2

        self.assertEqual(list(d), [10, 1, 2])
        del d[1]
        self.assertEqual(list(d), [10, 2])

    def test_simplicity(self):
        isSimple = _types.isSimple

        self.assertTrue(isSimple(int))
        self.assertTrue(isSimple(Int32()))
        self.assertTrue(isSimple(Int16()))
        self.assertTrue(isSimple(Int8()))
        self.assertTrue(isSimple(UInt64()))
        self.assertTrue(isSimple(UInt32()))
        self.assertTrue(isSimple(UInt16()))
        self.assertTrue(isSimple(UInt8()))
        self.assertTrue(isSimple(str))
        self.assertTrue(isSimple(bytes))
        self.assertTrue(isSimple(bool))
        self.assertTrue(isSimple(float))

        class C(Class):
            pass

        self.assertFalse(isSimple(C))

        self.assertTrue(isSimple(ListOf(int)))
        self.assertFalse(isSimple(ListOf(C)))

        self.assertTrue(isSimple(TupleOf(int)))
        self.assertFalse(isSimple(TupleOf(C)))

        self.assertTrue(isSimple(ConstDict(int, int)))
        self.assertFalse(isSimple(ConstDict(C, int)))
        self.assertFalse(isSimple(ConstDict(int, C)))

        self.assertTrue(isSimple(Dict(int, int)))
        self.assertFalse(isSimple(Dict(C, int)))
        self.assertFalse(isSimple(Dict(int, C)))

        self.assertFalse(isSimple(Alternative("Alternative")))

        self.assertTrue(isSimple(NamedTuple(x=int)))
        self.assertFalse(isSimple(NamedTuple(x=C)))

        X = Forward("X")
        X = X.define(Alternative("X", X={'x': X}, Y={'i': int}))
        self.assertFalse(isSimple(X))
        self.assertFalse(isSimple(NamedTuple(x=X)))

        self.assertFalse(isSimple(OneOf(int, X)))
        self.assertTrue(isSimple(OneOf(int, float)))

    def test_oneof_picks_best_choice(self):
        T = OneOf(float, int, bool)

        self.assertIsInstance(T(1.5), float)
        self.assertIsInstance(T(1), int)
        self.assertIsInstance(T(True), bool)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 15 13:53:26 2020

@author: engels
"""

import glob
import numpy as np
import insect_tools
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("-d", "--directory", nargs='?', const='./',
                    help="directory of h5 files, if not ./")
parser.add_argument("-o", "--output", type=str,
                    help="""output *.csv value""")
args = parser.parse_args()

if args.directory is None:
    args.directory ='./'

if args.output is None:
    args.output = "merge.csv"

#%%
files = sorted(glob.glob( args.directory+"/*.csv" ))

print('Found %i csv files' % (len(files)))

# unfortunately extracting wabbit isosurfaces results in cluttered CSV files
d = []
for file in files:
    d.append( np.loadtxt(file, delimiter=',', skiprows=1) )

d = np.vstack(d)

insect_tools.write_csv_file( args.directory+"/"+args.output, d, header=None, sep=';')


from django.conf import settings
from analytics.utils import api_auth, api_client, api_method

import requests, uuid

def send_event(user_id, tid, event_category, event_action, event_label, event_value):
    auth = api_auth(settings.API_SERVICE_URL, settings.API_SERVICE_LOGIN, settings.API_SERVICE_PASSWORD)
    client = api_client(auth)
    user = api_method(client, settings.API_SERVICE_URL, ['subscriptions', 'subscribers', 'retrive', {'id': user_id}])

    if not user:
        user_uuid = user.uuid
    else:
        user_uuid = uuid.uuid4()

    data = {
        'v': 1,
        'tid': str(tid),
        'cid': str(user_uuid),
        't': 'event',
        'ec': event_category,
        'ea': event_action,
        'el': event_label,
        'ev': event_value
    }
    url = 'http://www.google-analytics.com/collect'
    requests.post(url, data)

    return None


import serial
import csv
import re

port = "/dev/cu.usbmodem14501"
rate = 9600

srl_Heartrate = serial.Serial(port,rate)
#srl_Heartrate.open()
#filename = input("Ardu_heartN.csv->")

myfile = open("Ardu_heart5.csv","w")
'''
while(1):
        val = srl_Heartrate.readline()
        byte_to_str = val.decode()
        dec = int(byte_to_str)
        myfile.write(dec)
        myfile.write('¥n')
        print(dec)
        #writer = csv.writer(file)
        #writer.writerow()

myfile.close()
'''
a = 0
try:                        # try:の部分にループ処理を書く
    while True:
        if a % 2 == 0 :

            val = srl_Heartrate.readline()
            byte_to_str = val.decode()
            #dec = int(byte_to_str)
            myfile.write(byte_to_str)
            print(byte_to_str)
            #writer = csv.writer(file)
            #writer.writerow()
        else:
            val = srl_Heartrate.readline()
            byte_to_str = val.decode()
            #dec = int(byte_to_str)
            print("time:s",byte_to_str)
        a+=1
except KeyboardInterrupt:   # exceptに例外処理を書く
    print('stop!')
    myfile.close()
    srl_Heartrate.close()

'''
while(1):
    val = srl_Heartrate.readline()
    byte_to_str = val.decode()
    dec = int(byte_to_str)
    print(dec)
    with open("Ardu_Heartrate.csv","w") as file:
        writer = csv.writer(file)
        writer
'''


from restapi import serializers
from fueled import models
from rest_framework import (
    viewsets,
    parsers,
    permissions,
    exceptions,
    mixins
)
from django.shortcuts import get_object_or_404
from django.db.models import Avg
from django.utils.translation import ugettext as _

class UserRestaurantUniqueMixin(object):
    """ Add this for views where user and restaurant must be unique together
        Make sure it is included before mixins.CreateModelMixin, as it overrides one of its methods
    """

    def perform_create(self, serializer):
        user = self.request.user
        restaurant_id = serializer.data['restaurant_id']
        if models.Visit.objects.filter(user=user, restaurant_id=restaurant_id).exists():
            raise exceptions.ValidationError(_('Object with such user_id and restaurant_id already exists'))
        serializer.save(user=self.request.user)

class RestaurantViewSet(viewsets.ReadOnlyModelViewSet):

    queryset = models.Restaurant.objects.all()
    serializer_class = serializers.RestaurantSerializer
    permission_classes = (permissions.IsAuthenticated,)

    def get_queryset(self):
        blacklisted = models.Visit.objects.filter(user=self.request.user, blacklisted=True)
        queryset = self.queryset.exclude(visit__in=blacklisted)
        queryset = queryset.annotate(rating=Avg('feedback__evaluation')).order_by('-rating')
        return queryset

class VisitViewSet(viewsets.GenericViewSet,
                   UserRestaurantUniqueMixin,
                   mixins.CreateModelMixin,
                   mixins.RetrieveModelMixin,
                   mixins.UpdateModelMixin):

    queryset = models.Visit.objects.all()
    serializer_class = serializers.VisitSerializer
    parser_classes = (parsers.JSONParser,)
    permission_classes = (permissions.IsAuthenticated,)
    lookup_field = 'restaurant_id'

    def get_object(self):
        queryset = self.filter_queryset(self.get_queryset())
        filter_kwargs = {'user': self.request.user, self.lookup_field: self.kwargs[self.lookup_field]}
        obj = get_object_or_404(queryset, **filter_kwargs)
        self.check_object_permissions(self.request, obj)
        return obj

class FeedbackViewSet(viewsets.GenericViewSet,
                      UserRestaurantUniqueMixin,
                      mixins.CreateModelMixin,
                      mixins.RetrieveModelMixin,
                      mixins.ListModelMixin):

    queryset = models.Feedback.objects.all()
    serializer_class = serializers.FeedbackSerializer
    parser_classes = (parsers.JSONParser,)
    authentication_classes = (permissions.IsAuthenticated,)
    lookup_field = 'restaurant_id'

    def get_queryset(self):
        queryset = self.queryset
        restaurant_id = self.request.query_params.get('restaurant_id')

        if restaurant_id is None:
            raise exceptions.ValidationError(_('Missing GET param restaurant_id'))

        try:
            restaurant_id = int(restaurant_id)
        except ValueError:
            raise exceptions.ValidationError(_('restaurant_id must be integer'))

        return queryset.filter(restaurant_id=restaurant_id)

class FeedbackCommentViewSet(viewsets.GenericViewSet,
                             mixins.CreateModelMixin,
                             mixins.ListModelMixin):

    queryset = models.FeedbackComment.objects.all()
    serializer_class = serializers.FeedbackCommentSerializer
    parser_classes = (parsers.JSONParser,)
    authentication_classes = (permissions.IsAuthenticated,)
    lookup_field = 'feedback_id'
    filter_fields = ('feedback_id',)

    def get_queryset(self):
        queryset = self.queryset
        feedback_id = self.request.query_params.get('feedback_id')

        if feedback_id is None:
            raise exceptions.ValidationError(_('Missing GET param feedback_id'))

        try:
            feedback_id = int(feedback_id)
        except ValueError:
            raise exceptions.ValidationError(_('feedback_id must be integer'))

        return queryset.filter(feedback_id=feedback_id)