from datetime import datetime, date
from typing import Collection, Any
from decimal import Decimal

from rdr_service import clock
from rdr_service.code_constants import CONSENT_FOR_STUDY_ENROLLMENT_MODULE, EMPLOYMENT_ZIPCODE_QUESTION_CODE, \
    PMI_SKIP_CODE, \
    STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE, ZIPCODE_QUESTION_CODE, DATE_OF_BIRTH_QUESTION_CODE, \
    CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE, EHR_CONSENT_QUESTION_CODE
from rdr_service.dao.consent_dao import ConsentDao
from rdr_service.etl.model.src_clean import SrcClean, Observation, PidRidMapping, Person, Measurement, Death, \
    EHRConsentStatus
from rdr_service.code_constants import CONSENT_FOR_STUDY_ENROLLMENT_MODULE, EMPLOYMENT_ZIPCODE_QUESTION_CODE, PMI_SKIP_CODE,\
    STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE, ZIPCODE_QUESTION_CODE, DATE_OF_BIRTH_QUESTION_CODE,\
    WEAR_CONSENT_QUESTION_CODE, WEAR_YES_ANSWER_CODE, WEAR_CONSENT_MODULE
from rdr_service.etl.model.src_clean import SrcClean, Observation, PidRidMapping, Person, Measurement, Death, \
    WearConsent
from rdr_service.model.code import Code
from rdr_service.model.consent_file import ConsentFile
from rdr_service.model.consent_response import ConsentResponse
from rdr_service.model.measurements import PhysicalMeasurements
from rdr_service.model.measurements import Measurement as RdrMeasurement
from rdr_service.model.participant import Participant
from rdr_service.model.deceased_report import DeceasedReport
from rdr_service.model.api_user import ApiUser
from rdr_service.dao.physical_measurements_dao import PhysicalMeasurementsDao
from rdr_service.dao.curation_etl_dao import CdrEtlRunHistoryDao, CdrEtlSurveyHistoryDao
from rdr_service.participant_enums import QuestionnaireResponseStatus, QuestionnaireResponseClassificationType, \
    PhysicalMeasurementsCollectType, OriginMeasurementUnit, DeceasedNotification, DeceasedReportStatus
from rdr_service.tools.tool_libs.curation import CurationExportClass
from tests.helpers.unittest_base import BaseTestCase
from tests.helpers.tool_test_mixin import ToolTestMixin
from tests import test_data
from rdr_service.clock import FakeClock

TIME = datetime(2000, 1, 10)
TIME_2 = datetime(2022, 5, 10)

WEAR_NO_ANSWER_CODE = "wear_no"

class CurationEtlTest(ToolTestMixin, BaseTestCase):
    def setUp(self):
        super(CurationEtlTest, self).setUp(with_consent_codes=True)
        self._setup_data()
        self.history_dao = CdrEtlRunHistoryDao()
        self.pm_dao = PhysicalMeasurementsDao()

    def _setup_data(self):
        self.participant = self.data_generator.create_database_participant()
        self.data_generator.create_database_participant_summary(
            participant=self.participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))

        self.module_code = self.data_generator.create_database_code(value='src_clean_test')
        self.question_code_list = []
        self.indexed_answers = []
        self.questionnaire = self.data_generator.create_database_questionnaire_history()
        for question_index in range(4):
            question_code = self.data_generator.create_database_code(value=f'q_{question_index}')
            self.question_code_list.append(question_code)
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=self.questionnaire.questionnaireId,
                questionnaireVersion=self.questionnaire.version,
                codeId=question_code.codeId
            )
            answer_code = self.data_generator.create_database_code(value=f'a_{question_index}')
            self.indexed_answers.append((question_index, 'valueCodeId', answer_code.codeId))

        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=self.questionnaire.questionnaireId,
            questionnaireVersion=self.questionnaire.version,
            codeId=self.module_code.codeId
        )

        self.questionnaire_response = self._setup_questionnaire_response(self.participant, self.questionnaire,
                                                                         indexed_answers=self.indexed_answers)

    def _setup_questionnaire_response(self, participant, questionnaire, authored=datetime(2020, 3, 15),
                                      created=datetime(2020, 3, 15), indexed_answers=None, ignored_answer_indexes=None,
                                      status=QuestionnaireResponseStatus.COMPLETED,
                                      classification_type=QuestionnaireResponseClassificationType.COMPLETE):

        if not ignored_answer_indexes:
            ignored_answer_indexes = []

        questionnaire_response = self.data_generator.create_database_questionnaire_response(
            participantId=participant.participantId,
            questionnaireId=questionnaire.questionnaireId,
            questionnaireVersion=questionnaire.version,
            authored=authored,
            created=created,
            status=status,
            classificationType=classification_type
        )

        if indexed_answers is None:
            # If no answers were specified then answer all questions with 'test answer'
            indexed_answers = [
                (question_index, 'valueString', 'test answer')
                for question_index in range(len(questionnaire.questions))
            ]

        for question_index, answer_field_name, answer_string in indexed_answers:
            question = questionnaire.questions[question_index]
            self.data_generator.create_database_questionnaire_response_answer(
                questionnaireResponseId=questionnaire_response.questionnaireResponseId,
                questionId=question.questionnaireQuestionId,
                **{answer_field_name: answer_string},
                ignore=question_index in ignored_answer_indexes
            )

        return questionnaire_response

    def _exists_in_src_clean(self, column_name:str, value:Any) -> bool:
        value_exists = self.session.query(
            getattr(SrcClean,column_name)
        ).filter(
            getattr(SrcClean,column_name) == value
        ).distinct().scalar()
        return bool(value_exists
                    )

    @staticmethod
    def run_cdm_data_generation(cutoff=None, vocabulary='gs://curation-vocabulary/aou_vocab_20220201/',
                                participant_origin='all', participant_list_file=None, include_surveys=None,
                                exclude_surveys=None, exclude_participants=None, omit_surveys=False,
                                omit_measurements=False, exclude_in_person_pm=False, exclude_remote_pm=False,
                                prep_bq=False):
        CurationEtlTest.run_tool(CurationExportClass, tool_args={
            'command': 'cdm-data',
            'cutoff': cutoff,
            'vocabulary': vocabulary,
            'participant_origin': participant_origin,
            'participant_list_file': participant_list_file,
            'include_surveys': include_surveys,
            'exclude_surveys': exclude_surveys,
            'exclude_participants': exclude_participants,
            'omit_surveys': omit_surveys,
            'omit_measurements': omit_measurements,
            "exclude_in_person_pm": exclude_in_person_pm,
            "exclude_remote_pm": exclude_remote_pm,
            "prep_bq": prep_bq
        })

    @staticmethod
    def run_exclude_code_command(operation, code_value, code_type):
        CurationEtlTest.run_tool(CurationExportClass, tool_args={
            'command': 'exclude-code',
            'operation': operation,
            'code_value': code_value,
            'code_type': code_type
        })

    def test_locking(self):
        """Make sure that building the CDM tables doesn't take exclusive locks"""

        # Take an exclusive lock on the participant, one of the records known to be part of the insert query
        self.session.query(Participant).filter(
            Participant.participantId == self.participant.participantId
        ).with_for_update().one()

        # This will time out if the tool tries to take an exclusive lock on the participant
        self.run_cdm_data_generation()

    def _src_clean_record_found_for_response(self, questionnaire_response_id):
        response_record = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == questionnaire_response_id
        ).one_or_none()
        return response_record is not None

    def test_latest_questionnaire_response_used(self):
        """The latest questionnaire response received for a module should be used"""
        # Note: this only applies to modules that shouldn't roll up answers (ConsentPII should be rolled up)

        # Create a questionnaire response that would be used instead of the default for the test suite
        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update'),
                (3, 'valueString', 'final answer')
            ],
            authored=datetime(2020, 5, 10),
            created=datetime(2020, 5, 10)
        )

        # Check that we are only be seeing the answers from the latest questionnaire response
        self.run_cdm_data_generation()
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update'
            elif question_index == 3:
                expected_answer = 'final answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)

    def _create_consent_questionnaire(self):
        module_code = self.session.query(Code).filter(Code.value == CONSENT_FOR_STUDY_ENROLLMENT_MODULE).one()
        consent_question_codes = [
            self.data_generator.create_database_code(value=f'consent_q_code_{question_index}')
            for question_index in range(4)
        ]
        consent_question_codes += self.session.query(Code).filter(Code.value.in_([
            STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE, DATE_OF_BIRTH_QUESTION_CODE
        ])).all()

        consent_questionnaire = self.data_generator.create_database_questionnaire_history()
        for consent_question_code in consent_question_codes:
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=consent_questionnaire.questionnaireId,
                questionnaireVersion=consent_questionnaire.version,
                codeId=consent_question_code.codeId
            )

        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=consent_questionnaire.questionnaireId,
            questionnaireVersion=consent_questionnaire.version,
            codeId=module_code.codeId
        )

        return consent_questionnaire

    def test_consent_response_answers_roll_up(self):
        """
        For the consent survey, all of the most recent answers for a code should be used
        even if they were in previous responses.
        """

        consent_questionnaire = self._create_consent_questionnaire()

        self._setup_questionnaire_response(self.participant, consent_questionnaire)
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            indexed_answers=[
                (1, 'valueString', 'NewLastName'),
                (3, 'valueString', 'new-email')
            ],
            authored=datetime(2020, 5, 1)
        )
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            indexed_answers=[
                (2, 'valueString', 'updated address'),
                (3, 'valueString', 'corrected-email')
            ],
            authored=datetime(2020, 8, 1)
        )

        # Check that the newest answer is in the src_clean, even if it wasn't from the latest response
        self.run_cdm_data_generation()
        for question_index, question in enumerate(consent_questionnaire.questions):
            expected_answer = 'test answer'
            if question_index == 1:
                expected_answer = 'NewLastName'
            elif question_index == 2:
                expected_answer = 'updated address'
            elif question_index == 3:
                expected_answer = 'corrected-email'

            # Since there was an initial response with an answer for every question, then every question
            # should have an answer in the export (even though partial responses updated some of them).
            # There also shouldn't be multiple answers from the participant for any of the survey
            # questions in the export.
            src_clean_answer_query = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one()
            self.assertEqual(expected_answer, src_clean_answer_query.value_string)

    def test_consent_address_roll_up(self):
        """
        For the consent survey, any answers for the first line of the street address should also
        override previous answers for StreetAddress2
        """

        consent_questionnaire = self._create_consent_questionnaire()

        # Set up a response that answers all the questions, including the two address lines
        self._setup_questionnaire_response(self.participant, consent_questionnaire)
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            authored=datetime(2020, 5, 1)
        )

        # Enter another response that just updates the first line of the street address
        expected_final_address = '42 Wallaby Way'
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            indexed_answers=[
                (4, 'valueString', expected_final_address)  # Assuming the 4th question is the first line of the address
            ],
            authored=datetime(2020, 8, 1)
        )

        # Check that the only address answer in src_clean is the updated line 1 for the street address (which will
        # also replace line 2 for the the first response)
        self.run_cdm_data_generation()

        # Load all src_clean rows for lines 1 and 2 of the address
        address_answers = self.session.query(SrcClean).join(
            Code, Code.codeId == SrcClean.question_code_id
        ).filter(
            Code.value.in_([STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE])
        ).all()

        # Make sure we only get the updated address and nothing from the original response
        self.assertEqual(
            1,
            len(address_answers),
            'The updated street address should overwrite the previous answers for line 1 and 2 of the address'
        )
        self.assertEqual(expected_final_address, address_answers[0].value_string)

    def test_in_progress_responses_are_filtered_out_of_export(self):
        """
        We will filter in-progress questionnaire responses out during the ETL process until the curation team is
        ready to start receiving them.
        """

        # Create an in-progress questionnaire response that should not appear in src_clean
        participant = self.data_generator.create_database_participant()
        self._setup_questionnaire_response(
            participant,
            self.questionnaire,
            status=QuestionnaireResponseStatus.IN_PROGRESS
        )
        self.run_cdm_data_generation()

        # Check that src_clean doesn't have any records for the in-progress questionnaire response
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.participant_id == participant.participantId
        ).all()
        self.assertEmpty(src_clean_answers)

    def test_later_in_progress_response_not_used(self):
        """
        Make sure later, in-progress responses don't make us filter out full and valid responses that should be used
        """

        # Create a questionnaire response that might be used instead of the default for the test suite
        in_progress_response = self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            authored=datetime(2020, 5, 10),
            created=datetime(2020, 5, 10),
            status=QuestionnaireResponseStatus.IN_PROGRESS
        )
        self.run_cdm_data_generation()

        # Make sure src_clean only has data from the full response
        in_progress_answers = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == in_progress_response.questionnaireResponseId
        ).all()
        self.assertEmpty(in_progress_answers)

        complete_answers = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == self.questionnaire_response.questionnaireResponseId
        ).all()
        self.assertNotEmpty(complete_answers)

    def test_duplicate_record_in_temp_questionnaire_response_filtered_out(self):
        """
        Test that duplicate record in cdm.tmp_questionnaire_response is not included in export
        """

        # Create a new questionnaire response
        participant = self.data_generator.create_database_participant()
        self._setup_questionnaire_response(
            participant,
            self.questionnaire
        )

        # Create another response that is a duplicate
        duplicate_response = self._setup_questionnaire_response(
            participant,
            self.questionnaire,
            classification_type=QuestionnaireResponseClassificationType.DUPLICATE
        )

        self.run_cdm_data_generation()

        # Make sure no answers from the duplicate response made it into SrcClean
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.participant_id == participant.participantId
        ).all()
        self.assertFalse(any(
            answer_record.questionnaire_response_id == duplicate_response.questionnaireResponseId
            for answer_record in src_clean_answers
        ))

    def test_invalid_record_in_temp_questionnaire_response_filtered_out(self):
        """
        Test that invalid record in cdm.tmp_questionnaire_response is not included in export
        """

        # Create a new questionnaire response
        participant = self.data_generator.create_database_participant()
        invalid_response = self._setup_questionnaire_response(
            participant,
            self.questionnaire,
            classification_type=QuestionnaireResponseClassificationType.INVALID
        )

        self.run_cdm_data_generation()

        # Make sure no answers from the response made it into SrcClean
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.participant_id == participant.participantId
        ).all()
        self.assertFalse(any(
            answer_record.questionnaire_response_id == invalid_response.questionnaireResponseId
            for answer_record in src_clean_answers
        ))

    def test_zip_code_maps_to_string_field(self):
        """
        There are some questionnaire responses that have the zip code transmitted to us in the valueInteger
        field. Curation is expecting zip codes to be exported to them as strings. This checks to make sure that
        they're mapped correctly.
        """

        # Two codes have used value_integer for transmitting the value
        employment_zipcode_code = self.data_generator.create_database_code(value=EMPLOYMENT_ZIPCODE_QUESTION_CODE)
        address_zipcode_code = self.data_generator.create_database_code(value=ZIPCODE_QUESTION_CODE)
        skip_code = self.data_generator.create_database_code(value=PMI_SKIP_CODE)

        # Create a questionnaire with zip code questions
        zip_code_questionnaire = self.data_generator.create_database_questionnaire_history()
        for index in range(2):  # Creating four questions to test PMI_SKIP and when the value_string is correctly used
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=zip_code_questionnaire.questionnaireId,
                questionnaireVersion=zip_code_questionnaire.version,
                codeId=employment_zipcode_code.codeId
            )
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=zip_code_questionnaire.questionnaireId,
                questionnaireVersion=zip_code_questionnaire.version,
                codeId=address_zipcode_code.codeId
            )
        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=zip_code_questionnaire.questionnaireId,
            questionnaireVersion=self.questionnaire.version,
            codeId=self.module_code.codeId
        )

        # Set up a response with zip code values transmitted in various ways
        expected_zip_code_answers = [
            (0, 'valueInteger', '90210'),
            (1, 'valueInteger', '12345'),
            (2, 'valueString', '12121'),
            (3, 'valueCodeId', skip_code.codeId),
        ]
        zip_code_response = self._setup_questionnaire_response(
            self.participant,
            zip_code_questionnaire,
            indexed_answers=expected_zip_code_answers
        )

        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == zip_code_response.questionnaireResponseId
        ).all()

        for index, answer_field, expected_value in expected_zip_code_answers:
            src_cln: SrcClean = src_clean_answers[index]
            if answer_field == 'valueCodeId':  # An answer of skipping the zip code
                self.assertEqual(expected_value, src_cln.value_code_id)
            else:
                self.assertEqual(expected_value, src_cln.value_string)
                self.assertIsNone(src_cln.value_number)

    def test_exclude_participants_age_under_18(self):
        """
        Curation team request to exclude participants that were under 18 at the time of study consent.
        """
        self._create_consent_questionnaire()
        self.run_cdm_data_generation(omit_measurements=True)
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(4, len(src_clean_answers))

        dob = datetime(1982, 1, 11)
        self.data_generator.create_database_participant_summary(participant=self.participant, dateOfBirth=dob)
        self.run_cdm_data_generation(omit_measurements=True)  # PM query errors out when no PIDs are supplied.
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(0, len(src_clean_answers))

    def test_etl_exclude_code(self):
        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(4, len(src_clean_answers))
        self.session.commit()

        exclude_question_str = self.question_code_list[0].value + ',' + self.question_code_list[1].value
        self.run_exclude_code_command('add', exclude_question_str, 'question')
        exclude_answer_str = 'a_' + str(self.indexed_answers[2][0])
        self.run_exclude_code_command('add', exclude_answer_str, 'answer')

        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(1, len(src_clean_answers))
        self.session.commit()

        self.run_exclude_code_command('add', self.module_code.value, 'module')
        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(0, len(src_clean_answers))

        self.clear_table_after_test('cdr_etl_survey_history')
        self.clear_table_after_test('cdr_etl_run_history')
        self.clear_table_after_test('cdr_excluded_code')

    def test_etl_include_exclude_survey_history(self):
        exclude_question_str = self.question_code_list[0].value + ',' + self.question_code_list[1].value
        self.run_exclude_code_command('add', exclude_question_str, 'question')
        exclude_answer_str = 'a_' + str(self.indexed_answers[2][0])
        self.run_exclude_code_command('add', exclude_answer_str, 'answer')

        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(1, len(src_clean_answers))
        self.session.commit()

        include_exclude_history_dao = CdrEtlSurveyHistoryDao()
        records = include_exclude_history_dao.get_all()
        self.assertEqual(len(records), 6)

        self.clear_table_after_test('cdr_etl_survey_history')
        self.clear_table_after_test('cdr_etl_run_history')
        self.clear_table_after_test('cdr_excluded_code')

    def test_etl_history(self):
        with FakeClock(TIME_2):
            self.run_cdm_data_generation(cutoff='2022-04-01')
        records = self.history_dao.get_all()
        self.assertEqual(len(records), 1)
        self.assertEqual(records[0].cutoffDate, date(2022, 4, 1))
        self.assertEqual(records[0].startTime, TIME_2)
        self.assertEqual(records[0].endTime, TIME_2)
        self.assertEqual(records[0].vocabularyPath, 'gs://curation-vocabulary/aou_vocab_20220201/')
        self.assertEqual(records[0].filterOptions['participant_origin'],'all')

        self.clear_table_after_test('cdr_etl_survey_history')
        self.clear_table_after_test('cdr_etl_run_history')
        self.clear_table_after_test('cdr_excluded_code')

    def test_cutoff_date(self):
        participant1 = self.data_generator.create_database_participant()
        # consent before cutoff 2022-04-01, should be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant1, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 2, 1),
            dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant1,
            self.questionnaire,
            authored=datetime(2022, 2, 1)
        )

        participant2 = self.data_generator.create_database_participant()
        # consent after cutoff 2022-04-01, should not be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant2, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 4, 20),
            dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant2,
            self.questionnaire,
            authored=datetime(2022, 4, 20)
        )

        participant3 = self.data_generator.create_database_participant()
        # withdrawal before cutoff 2022-04-01, should not be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant3, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 2, 1),
            withdrawalStatus=2, withdrawalAuthored=datetime(2022, 2, 20), dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant3,
            self.questionnaire,
            authored=datetime(2022, 2, 1)
        )

        participant4 = self.data_generator.create_database_participant()
        # withdrawal after cutoff 2022-04-01, should be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant4, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 2, 1),
            withdrawalStatus=2, withdrawalAuthored=datetime(2022, 4, 20), dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant4,
            self.questionnaire,
            authored=datetime(2022, 2, 1)
        )

        self.run_cdm_data_generation(cutoff='2022-04-01')

        src_clean_answers_p1 = self.session.query(SrcClean)\
            .filter(SrcClean.participant_id == participant1.participantId).all()
        self.assertEqual(4, len(src_clean_answers_p1))

        src_clean_answers_p2 = self.session.query(SrcClean) \
            .filter(SrcClean.participant_id == participant2.participantId).all()
        self.assertEqual(0, len(src_clean_answers_p2))

        src_clean_answers_p3 = self.session.query(SrcClean) \
            .filter(SrcClean.participant_id == participant3.participantId).all()
        self.assertEqual(0, len(src_clean_answers_p3))

        src_clean_answers_p4 = self.session.query(SrcClean) \
            .filter(SrcClean.participant_id == participant4.participantId).all()
        self.assertEqual(4, len(src_clean_answers_p4))

    def test_ignored_answers_are_marked_invalid(self):
        """
        Any answers that have the ignore field set to True should give the skip code
        (to be marked invalid in the finalization step)
        """
        questionnaire_response = self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (0, 'valueString', 'first_answer'),
                (1, 'valueString', 'ignored answer'),
                (2, 'valueBoolean', True),
                (3, 'valueDate', datetime.now())
            ],
            ignored_answer_indexes=[1, 2, 3]
        )

        # Check that we are only be seeing the answers from the latest questionnaire response
        self.run_cdm_data_generation()
        src_clean_answers: Collection[SrcClean] = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == questionnaire_response.questionnaireResponseId
        ).all()

        # Should be getting 4 records still (even though some of them are blank)
        self.assertEqual(4, len(src_clean_answers))
        for index, answer in enumerate(src_clean_answers):
            if index == 0:
                self.assertEqual('first_answer', answer.value_string)
            else:
                # Make sure none of the answer fields have an answer value
                self.assertFalse(any([
                    answer is not None
                    for answer in [
                        answer.value_string,
                        answer.value_number,
                        answer.value_date,
                        answer.value_boolean,
                        answer.value_code_id
                    ]
                ]))
                self.assertEqual(PMI_SKIP_CODE, answer.value_ppi_code)

    def test_cutoff_date_questionnaire_response_used(self):
        """The latest questionnaire response received before the cutoff date for a module should be used"""
        # Note: this only applies to modules that shouldn't roll up answers (ConsentPII should be rolled up)

        # Create a questionnaire response that would be used instead of the default for the test suite
        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update'),
                (3, 'valueString', 'intermediate answer')
            ],
            authored=datetime(2020, 5, 10),
            created=datetime(2020, 5, 10)
        )

        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update2'),
                (3, 'valueString', 'final answer')
            ],
            authored=datetime(2022, 5, 10),
            created=datetime(2022, 5, 10)
        )

        # Check that we are only seeing the answers from the questionnaire response before cutoff date
        self.run_cdm_data_generation(cutoff='2022-04-01')
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update'
            elif question_index == 3:
                expected_answer = 'intermediate answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)
        self.session.commit()

        # Returns the newest answers without cutoff
        self.run_cdm_data_generation()
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update2'
            elif question_index == 3:
                expected_answer = 'final answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)

    def test_exclude_profile_update_questionnaire_response(self):
        """Questionnaire Responses with PROFILE_UPDATE classification should be excluded"""

        # Create a questionnaire response that would be used instead of the default for the test suite
        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update'),
                (3, 'valueString', 'final answer')
            ],
            authored=datetime(2020, 6, 10),
            created=datetime(2020, 6, 10)
        )

        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'profile update'),
            ],
            authored=datetime(2021, 6, 10),
            created=datetime(2021, 6, 10),
            classification_type=QuestionnaireResponseClassificationType.PROFILE_UPDATE
        )

        # Check that we are only seeing the answers from the latest questionnaire response
        self.run_cdm_data_generation()
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update'
            elif question_index == 3:
                expected_answer = 'final answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)

    def test_participant_origin_filter(self):
        """ Test the participant origin filter selects the intended participants"""
        participant_ce = self.data_generator.create_database_participant(participantOrigin='careevolution')
        self.data_generator.create_database_participant_summary(
            participant=participant_ce,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
        self._setup_questionnaire_response(
            participant_ce,
            self.questionnaire
        )

        participant_vibrent = self.data_generator.create_database_participant(participantOrigin='vibrent')
        self.data_generator.create_database_participant_summary(
            participant=participant_vibrent,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
        self._setup_questionnaire_response(
            participant_vibrent,
            self.questionnaire
        )

        self.run_cdm_data_generation(participant_origin='careevolution')
        vibrent_ppt_exists = self._exists_in_src_clean('participant_id', participant_vibrent.participantId)
        ce_ppt_exists = self._exists_in_src_clean('participant_id', participant_ce.participantId)

        self.assertTrue(ce_ppt_exists)
        self.assertFalse(vibrent_ppt_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual('careevolution', run_history.filterOptions['participant_origin'])

        self.session.commit()

        self.run_cdm_data_generation(participant_origin='vibrent')
        vibrent_ppt_exists = self._exists_in_src_clean('participant_id', participant_vibrent.participantId)
        ce_ppt_exists = self._exists_in_src_clean('participant_id', participant_ce.participantId)

        self.assertFalse(ce_ppt_exists)
        self.assertTrue(vibrent_ppt_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual('vibrent', run_history.filterOptions['participant_origin'])

        self.session.commit()

        self.run_cdm_data_generation()
        vibrent_ppt_exists = self._exists_in_src_clean('participant_id', participant_vibrent.participantId)
        ce_ppt_exists = self._exists_in_src_clean('participant_id', participant_ce.participantId)

        self.assertTrue(ce_ppt_exists)
        self.assertTrue(vibrent_ppt_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual('all', run_history.filterOptions['participant_origin'])

    def test_participant_list(self):
        pids = list(range(10000,10010))
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
            self._setup_questionnaire_response(
                participant,
                self.questionnaire
            )

        self.run_cdm_data_generation(
            participant_origin=None,
            participant_list_file=test_data.data_path('test_curation_participant_list.txt')
        )

        test_participants = [10002, 10004, 10007]
        for pid in pids:
            pid_exists = self._exists_in_src_clean("participant_id", pid)
            if pid in test_participants:
                self.assertTrue(pid_exists)
            else:
                self.assertFalse(pid_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertIn('test-data/test_curation_participant_list.txt',
                      run_history.filterOptions['participant_list_file'])

    def _create_questionnaire(self, survey_name, question_code_list=None):
        module_code = self.data_generator.create_database_code(value=survey_name)
        question_codes = [
            self.data_generator.create_database_code(value=f'{survey_name}_q_code_{question_index}')
            for question_index in range(4)
        ]

        if question_code_list:
            question_codes += self.session.query(Code).filter(Code.value.in_([
                question_code_list
            ])).all()

        questionnaire = self.data_generator.create_database_questionnaire_history()
        for question_code in question_codes:
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=questionnaire.questionnaireId,
                questionnaireVersion=questionnaire.version,
                codeId=question_code.codeId
            )

        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=questionnaire.questionnaireId,
            questionnaireVersion=questionnaire.version,
            codeId=module_code.codeId
        )

        return questionnaire

    def test_include_surveys(self):
        survey_names = ['Q1', 'Q2', 'Q3', 'Q4']
        for survey_name in survey_names:
            questionnaire = self._create_questionnaire(survey_name)
            self._setup_questionnaire_response(self.participant, questionnaire)

        self.run_cdm_data_generation(include_surveys=test_data.data_path('test_curation_surveys.txt'))

        survey_results = []
        for survey_name in survey_names:
            survey_results.append((survey_name, self._exists_in_src_clean('survey_name', survey_name)))

        for result in survey_results:
            if result[0] in ['Q1', 'Q4']:
                self.assertFalse(result[1])
            elif result[0] in ['Q2', 'Q3']:
                self.assertTrue(result[1])

        last_run = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual(['Q2', 'Q3'], last_run.filterOptions['include_surveys'])

    def test_exclude_surveys(self):
        survey_names = ['Q1', 'Q2', 'Q3', 'Q4']
        for survey_name in survey_names:
            questionnaire = self._create_questionnaire(survey_name)
            self._setup_questionnaire_response(self.participant, questionnaire)

        self.run_cdm_data_generation(exclude_surveys=test_data.data_path('test_curation_surveys.txt'))

        survey_results = []
        for survey_name in survey_names:
            survey_results.append((survey_name, self._exists_in_src_clean('survey_name', survey_name)))

        for result in survey_results:
            if result[0] in ['Q1', 'Q4', 'src_clean_test']:
                self.assertTrue(result[1], f"{result[0]} expected in src_clean but not found")
            elif result[0] in ['Q2', 'Q3']:
                self.assertFalse(result[1])

        last_run = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual(['Q2', 'Q3'], last_run.filterOptions['exclude_surveys'])

    def test_exclude_participants(self):
        pids = list(range(10000,10010))
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
            self._setup_questionnaire_response(
                participant,
                self.questionnaire
            )

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_participants=test_data.data_path('test_curation_participant_list.txt')
        )

        excluded_participants = [10002, 10004, 10007]
        for pid in pids:
            pid_exists = self._exists_in_src_clean("participant_id", pid)
            if pid in excluded_participants:
                self.assertFalse(pid_exists)
            else:
                self.assertTrue(pid_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertIn('test-data/test_curation_participant_list.txt',
                         run_history.filterOptions['participant_exclude_file'])

    def test_survey_src_id(self):
        consent_questionnaire = self._create_consent_questionnaire()

        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            self.questionnaire
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        obs_src_id = self.session.query(
            Observation.src_id
        ).filter(
            Observation.person_id == participant.participantId
        ).first()[0]

        prm_src_id = self.session.query(
            PidRidMapping.src_id
        ).filter(
            PidRidMapping.person_id == participant.participantId
        ).first()[0]

        person_src_id = self.session.query(
            Person.src_id
        ).filter(
            Person.person_id == participant.participantId
        ).first()[0]

        self.assertEqual('test_portal', obs_src_id)
        self.assertEqual('test_portal', prm_src_id)
        self.assertEqual('test_portal', person_src_id)

    def _test_ce_src_id(self):
        # Care Evolution participant origin should be saved as 'ce' in src_id column
        consent_questionnaire = self._create_consent_questionnaire()

        participant = self.data_generator.create_database_participant(participantOrigin='careevolution')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            self.questionnaire
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        obs_src_id = self.session.query(
            Observation.src_id
        ).filter(
            Observation.person_id == participant.participantId
        ).first()[0]

        self.assertEqual('ce', obs_src_id)

    def _setup_pm(self, participant_id: int):
        """ Creates in-person and remote physical measurements for a participant_id"""
        resource = {"entry": [
            {"resource":
                 {"date": datetime.now().isoformat()}
             }
        ]}

        in_person_pm_data = {
            "physicalMeasurementsId": 1,
            "participantId": participant_id,
            "createdSiteId": 1,
            "finalizedSiteId": 2,
            "origin": 'hpro',
            "collectType": PhysicalMeasurementsCollectType.SITE,
            "originMeasurementUnit": OriginMeasurementUnit.UNSET
        }

        record: PhysicalMeasurements = PhysicalMeasurements(**in_person_pm_data)
        self.pm_dao.store_record_fhir_doc(record, resource)
        pm_record: PhysicalMeasurements = self.pm_dao.insert(record)

        meas_in_person_height = RdrMeasurement(
            measurementId=10001,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="height",
            measurementTime=datetime.now(),
            valueDecimal=162.0,
            valueUnit="cm",
        )
        meas_in_person_weight = RdrMeasurement(
            measurementId=10002,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="weight",
            measurementTime=datetime.now(),
            valueDecimal=63.0,
            valueUnit="kg",
        )
        self.session.add(meas_in_person_weight)
        self.session.add(meas_in_person_height)

        remote_pm_data = {
            "physicalMeasurementsId": 2,
            "participantId": participant_id,
            "createdSiteId": 1,
            "finalizedSiteId": 2,
            "origin": 'test-portal',
            "collectType": PhysicalMeasurementsCollectType.SELF_REPORTED,
            "originMeasurementUnit": OriginMeasurementUnit.UNSET
        }

        record: PhysicalMeasurements = PhysicalMeasurements(**remote_pm_data)
        self.pm_dao.store_record_fhir_doc(record, resource)
        pm_record: PhysicalMeasurements = self.pm_dao.insert(record)

        meas_remote_height = RdrMeasurement(
            measurementId=10005,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="height",
            measurementTime=datetime.now(),
            valueDecimal=165.0,
            valueUnit="cm",
        )
        meas_remote_weight = RdrMeasurement(
            measurementId=10006,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="weight",
            measurementTime=datetime.now(),
            valueDecimal=62.0,
            valueUnit="kg",
        )

        self.session.add(meas_remote_weight)
        self.session.add(meas_remote_height)
        self.session.commit()

    def test_pm_src_id(self):
        consent_questionnaire = self._create_consent_questionnaire()
        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )

        self._setup_pm(participant.participantId)

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        inperson_src_id = self.session.query(
            Measurement.src_id
        ).filter(
            Measurement.person_id == participant.participantId,
            Measurement.measurement_type_concept_id == 44818701
        ).first()[0]

        remote_src_id = self.session.query(
            Measurement.src_id
        ).filter(
            Measurement.person_id == participant.participantId,
            Measurement.measurement_type_concept_id == 32865
        ).first()[0]

        self.assertEqual('healthpro', inperson_src_id)
        self.assertEqual('test-portal', remote_src_id)

    def test_include_physical_measurements(self):
        consent_questionnaire = self._create_consent_questionnaire()
        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )
        self._setup_pm(participant.participantId)
        in_person_height = (44818701, 'height', Decimal('162.000000'), 'cm')
        in_person_weight = (44818701, 'weight', Decimal('63.000000'), 'kg')
        remote_height = (32865, 'height', Decimal('165.000000'), 'cm')
        remote_weight = (32865, 'weight', Decimal('62.000000'), 'kg')

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_in_person_pm=False,
            exclude_remote_pm=False
        )

        measurements = self.session.query(Measurement.measurement_type_concept_id,
                                          Measurement.measurement_source_value,
                                          Measurement.value_as_number,
                                          Measurement.unit_source_value).all()

        # Check if in-person and remote measurements are in table
        self.assertIn(in_person_height, measurements)
        self.assertIn(in_person_weight, measurements)
        self.assertIn(remote_height, measurements)
        self.assertIn(remote_weight, measurements)

        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_in_person_pm=True,
            exclude_remote_pm=False
        )

        measurements = self.session.query(Measurement.measurement_type_concept_id,
                                          Measurement.measurement_source_value,
                                          Measurement.value_as_number,
                                          Measurement.unit_source_value).all()

        # Only remote measurements should be present
        self.assertNotIn(in_person_height, measurements)
        self.assertNotIn(in_person_weight, measurements)
        self.assertIn(remote_height, measurements)
        self.assertIn(remote_weight, measurements)

        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_in_person_pm=False,
            exclude_remote_pm=True
        )

        measurements = self.session.query(Measurement.measurement_type_concept_id,
                                          Measurement.measurement_source_value,
                                          Measurement.value_as_number,
                                          Measurement.unit_source_value).all()

        # Only in-person measurements should be present
        self.assertIn(in_person_height, measurements)
        self.assertIn(in_person_weight, measurements)
        self.assertNotIn(remote_height, measurements)
        self.assertNotIn(remote_weight, measurements)

    def _exists_in_death_table(self, participant_id: int) -> bool:
        return bool(self.session.query(
            Death.person_id
        ).filter(
            Death.person_id == participant_id
        ).distinct().scalar())

    def test_death_table(self):
        # Create API User for DeceasedReport
        api_user = ApiUser(id=1, username='test', system='test')
        self.session.add(api_user)

        pids = list(range(10000, 10010))
        consent_questionnaire = self._create_consent_questionnaire()
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
            )
            self._setup_questionnaire_response(
                participant,
                consent_questionnaire,
                indexed_answers=[
                    (6, 'valueDate', datetime(1982, 1, 9))
                    # Assuming the 6th question is the date of birth
                ],
                authored=datetime(2020, 5, 1)
            )

        # Only approved deceased reports should be in Death
        deceased_reports = [
            DeceasedReport(
                participantId=pids[0],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.APPROVED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            ),
            DeceasedReport(
                participantId=pids[1],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.PENDING,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            ),
            DeceasedReport(
                participantId=pids[2],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.DENIED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            )
        ]

        for deceased_report in deceased_reports:
            self.session.add(deceased_report)
        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        self.assertTrue(self._exists_in_death_table(pids[0]))
        self.assertFalse(self._exists_in_death_table(pids[1]))
        self.assertFalse(self._exists_in_death_table(pids[2]))

    def test_death_table_cutoff(self):
        # Create API User for DeceasedReport
        api_user = ApiUser(id=1, username='test', system='test')
        self.session.add(api_user)

        pids = [10000, 10001]
        consent_questionnaire = self._create_consent_questionnaire()
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
            )
            self._setup_questionnaire_response(
                participant,
                consent_questionnaire,
                indexed_answers=[
                    (6, 'valueDate', datetime(1982, 1, 9))
                    # Assuming the 6th question is the date of birth
                ],
                authored=datetime(2020, 5, 1)
            )

        # Approved Deceased report authored after cutoff should not be in Death
        deceased_reports = [
            DeceasedReport(
                participantId=pids[0],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.APPROVED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            ),
            DeceasedReport(
                participantId=pids[1],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.APPROVED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 5, 1)
            )
        ]

        for deceased_report in deceased_reports:
            self.session.add(deceased_report)
        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
            cutoff='2023-04-01'
        )

        self.assertTrue(self._exists_in_death_table(pids[0]))
        self.assertFalse(self._exists_in_death_table(pids[1]))

    def test_ehr_consent_table(self):
        consent_dao = ConsentDao()
        self.data_generator.create_database_code(value=EHR_CONSENT_QUESTION_CODE)
        ehr_consent = self._create_questionnaire(CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE,
                                                        [EHR_CONSENT_QUESTION_CODE])
        ehr_yes = self.data_generator.create_database_code(value="Yes")
        ehr_no = self.data_generator.create_database_code(value="No")

        questionnaire_response_ids = []
        submission_date = datetime(2023, 4, 29, 0, 0)

        for pid in range(10000,10006):
            participant = self.data_generator.create_database_participant(participantId=pid,
                                                                          researchId=pid+100)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
            )
            if pid == 10000:
                self._setup_questionnaire_response(
                    participant,
                    ehr_consent,
                    indexed_answers=[
                        (4, 'valueCodeId', ehr_no.codeId)
                    ],
                    authored=submission_date
                )
            else:
                qr = self._setup_questionnaire_response(
                    participant,
                    ehr_consent,
                    indexed_answers=[
                        (4, 'valueCodeId', ehr_yes.codeId)
                    ],
                    authored=submission_date
                )
                questionnaire_response_ids.append(qr.questionnaireResponseId)

        consent_responses = [
            ConsentResponse(
                id=2,
                questionnaire_response_id=questionnaire_response_ids[1], # PID 10002
            ),
            ConsentResponse(
                id=3,
                questionnaire_response_id=questionnaire_response_ids[2], # PID 10003
            ),
            ConsentResponse(
                id=4,
                questionnaire_response_id=questionnaire_response_ids[3], # PID 10004
            ),
            ConsentResponse(
                id=5,
                questionnaire_response_id=questionnaire_response_ids[4], # PID 10005
            )
        ]
        for consent_response in consent_responses:
            self.session.add(consent_response)
        self.session.commit()

        with clock.FakeClock(datetime(2023, 5, 2)):
            consent_dao.insert(ConsentFile(
                    consent_response_id=2, # PID 10002
                    sync_status=2
                ))

        with clock.FakeClock(datetime(2023, 4, 30)):
            consent_files = [
                ConsentFile(
                    consent_response_id=3, # PID 10003
                    sync_status=2
                ),
                ConsentFile(
                    consent_response_id=4, # PID 10004
                    sync_status=1
                ),
            ]
            for consent_file in consent_files:
                consent_dao.insert(consent_file)

        self.run_cdm_data_generation(
            participant_origin='all',
            cutoff='2023-05-01'
        )

        expected_results = [
            (10000, 10100, 'SUBMITTED_NO', submission_date), # Submitted No answer
            (10001, 10101, 'SUBMITTED', submission_date), # Submitted Yes before validation implemented
            (10002, 10102, 'SUBMITTED_NOT_VALIDATED', submission_date), # Submitted Yes, not validated before cutoff date
            (10003, 10103, 'SUBMITTED', submission_date), # Submitted Yes, valid consent
            (10004, 10104, 'SUBMITTED_INVALID', submission_date), # Submitted Yes, invalid consent
            (10005, 10105, 'SUBMITTED_NOT_VALIDATED', submission_date) # Submitted yes, Consent Response exists, Consent file does not, not yet validated
        ]

        consent_table = self.session.query(
            EHRConsentStatus.person_id,
            EHRConsentStatus.research_id,
            EHRConsentStatus.consent_for_electronic_health_records,
            EHRConsentStatus.consent_for_electronic_health_records_authored
        ).all()

        for result in expected_results:
            self.assertIn(result, consent_table)

    def test_wear_consent_table(self):
        wear_yes_code = self.data_generator.create_database_code(value=WEAR_YES_ANSWER_CODE)
        wear_no_code = self.data_generator.create_database_code(value=WEAR_NO_ANSWER_CODE)
        self.data_generator.create_database_code(value=WEAR_CONSENT_QUESTION_CODE)

        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )

        wear_questionnaire = self._create_questionnaire(WEAR_CONSENT_MODULE, [WEAR_CONSENT_QUESTION_CODE])

        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_yes_code.codeId)
            ],
            authored=datetime(2020, 5, 1)
        )
        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_no_code.codeId)
            ],
            authored=datetime(2020, 6, 1)
        )
        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_yes_code.codeId)
            ],
            authored=datetime(2020, 7, 1)
        )
        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_no_code.codeId)
            ],
            authored=datetime(2020, 8, 1)
        )

        self.run_cdm_data_generation(
            participant_origin='all',
            cutoff='2020-07-15'
        )

        wear_table = self.session.query(
            WearConsent.person_id,
            WearConsent.authored,
            WearConsent.consent_status
        ).all()

        self.assertIn((participant.participantId, datetime(2020, 5, 1), WEAR_YES_ANSWER_CODE), wear_table)
        self.assertIn((participant.participantId, datetime(2020, 6, 1), WEAR_NO_ANSWER_CODE), wear_table)
        self.assertIn((participant.participantId, datetime(2020, 7, 1), WEAR_YES_ANSWER_CODE), wear_table)
        # Wear consent after cutoff date shouldn't be present.
        self.assertNotIn((participant.participantId, datetime(2020, 8, 1), WEAR_NO_ANSWER_CODE), wear_table)


from flask import Flask, render_template, Response, request
from picamera import PiCamera
from flask import jsonify,json
import numpy as np
from camera_pi import Camera

import sys
import wave
import contextlib
import time
import cv2
import pygame
import RPi.GPIO as GPIO

app = Flask(__name__)
cam = Camera()

@app.route('/', methods=['GET','POST'])
def index():
    """Video streaming home page."""

    return render_template('index.html')

@app.route('/button', methods=['GET','POST'])
def button():
    print("Coucou")
    return jsonify({})

def gen(camera):
    """Video streaming generator function."""
    while True:
        frame = camera.get_frame()
        yield (b'--frame\r\n'
               b'Content-Type: image/png\r\n\r\n' + frame + b'\r\n')

@app.route('/video_feed')
def video_feed():
    """Video streaming route. Put this in the src attribute of an img tag."""
    return Response(gen(cam),
                   mimetype='multipart/x-mixed-replace; boundary=frame')

@app.route('/pic', methods=['GET','POST'])
def pic():
    print("Pic taken")
    cam.save_frame()
    num_pic = cam.nbPic
    return jsonify({'num_pic' : str(num_pic)+ " pictures"})
#This application allows to play a sound
@app.route('/sound', methods=['GET','POST'])
def sound():
    pygame.init()
    pygame.mixer.music.load("output.wav")
    pygame.mixer.music.play()
    length=pygame.mixer.Sound('output.wav').get_length()
    time.sleep(length+10)
    return jsonify({})

#This application allows to choose a time where laser is turned on
@app.route('/switch', methods=['GET','POST'])
def switch():
    out1 = 40
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(out1,GPIO.OUT)
    val_switch = request.args.get('val_switch',0 , type = int)
    GPIO.output(out1,GPIO.HIGH)
    time.sleep(val_switch)
    value = GPIO.input(out1)
    print(value)
    GPIO.output(out1,GPIO.LOW)
    print(GPIO.input(out1))
    GPIO.cleanup()
    return jsonify({})

#This application allows to make a full turn of the servo of the ball launcher
@app.route('/shoot', methods=['GET','POST'])
def shoot():
    GPIO.cleanup()
    x =10
    print('shot')
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(36, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(36,50)
    pwm.start(x)
    time.sleep(0.4)
    GPIO.cleanup()
    return jsonify({})

#This application is useless but allows you tu use buttons to control a servo
@app.route('/servo', methods=['GET','POST'])
def servo():
    if request.method == 'POST':
        angle = request.form['angle']
    print(angle)

#This application allows to control the servo for the laser :set the right GPIO of RPi
@app.route('/servo_las', methods=['GET','POST'])
def servo_las():
    GPIO.cleanup()
    angle_las = request.args.get('angle_las',0 , type = int)
    print(angle_las)
    x = (-float(angle_las)+135)*12/180
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(38, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(38,50)
    pwm.start(0)
    time.sleep(1)
    pwm.ChangeDutyCycle(x)
    time.sleep(0.5)
    GPIO.cleanup()
    return jsonify({})

#This application allows to control the servo to move the ball launcher. Set the right pin
@app.route('/servo_ball', methods=['GET','POST'])
def servo_ball():
    GPIO.cleanup()
    angle_ball = request.args.get('angle_ball',0 , type = int)
    print(angle_ball)

    x =( int(angle_ball) +77)/18
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(32, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(32,50)
    pwm.start(x)
    time.sleep(0.5)
    GPIO.cleanup()
    return jsonify({})

# This application allows to control the stepper motor of the station . Set the right pin
@app.route('/step', methods=['GET','POST'])
def step():
    angle = request.args.get('angle',0 , type=int)
    print(angle)

    t = 0.02
    step_out1=37
    step_out2=35
    step_out3=33
    step_out4=31

    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(step_out1,GPIO.OUT)
    GPIO.setup(step_out2,GPIO.OUT)
    GPIO.setup(step_out3,GPIO.OUT)
    GPIO.setup(step_out4,GPIO.OUT)

    GPIO.output(step_out1,GPIO.LOW)
    GPIO.output(step_out2,GPIO.LOW)
    GPIO.output(step_out3,GPIO.LOW)
    GPIO.output(step_out4,GPIO.LOW)

    y=angle*25/180
    if y>0 and angle <= 360 :

         for i in range(0,int(y)):

            GPIO.output(step_out1,GPIO.HIGH)
            GPIO.output(step_out2,GPIO.LOW)
            GPIO.output(step_out3,GPIO.LOW)
            GPIO.output(step_out4,GPIO.LOW)
            time.sleep(t)

            GPIO.output(step_out1,GPIO.LOW)
            GPIO.output(step_out2,GPIO.LOW)
            GPIO.output(step_out3,GPIO.HIGH)
            GPIO.output(step_out4,GPIO.LOW)
            time.sleep(t)

            GPIO.output(step_out1,GPIO.LOW)
            GPIO.output(step_out2,GPIO.HIGH)
            GPIO.output(step_out3,GPIO.LOW)
            GPIO.output(step_out4,GPIO.LOW)
            time.sleep(t)

            GPIO.output(step_out1,GPIO.LOW)
            GPIO.output(step_out2,GPIO.LOW)
            GPIO.output(step_out3,GPIO.LOW)
            GPIO.output(step_out4,GPIO.HIGH)
            time.sleep(t)

    if y<0 and abs(angle) <= 360:
        for i in range(0,-int(y)):

            GPIO.output(step_out1,GPIO.LOW)
            GPIO.output(step_out2,GPIO.LOW)
            GPIO.output(step_out3,GPIO.LOW)
            GPIO.output(step_out4,GPIO.HIGH)
            time.sleep(t)

            GPIO.output(step_out1,GPIO.LOW)
            GPIO.output(step_out2,GPIO.HIGH)
            GPIO.output(step_out3,GPIO.LOW)
            GPIO.output(step_out4,GPIO.LOW)
            time.sleep(t)

            GPIO.output(step_out1,GPIO.LOW)
            GPIO.output(step_out2,GPIO.LOW)
            GPIO.output(step_out3,GPIO.HIGH)
            GPIO.output(step_out4,GPIO.LOW)
            time.sleep(t)

            GPIO.output(step_out1,GPIO.HIGH)
            GPIO.output(step_out2,GPIO.LOW)
            GPIO.output(step_out3,GPIO.LOW)
            GPIO.output(step_out4,GPIO.LOW)
            time.sleep(t)

    return jsonify({})

#@app.route('/5')
def move5():
    GPIO.cleanup()
    angle = 5
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/10')
def move10():
    GPIO.cleanup()
    angle = 10
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/15')
def move15():
    GPIO.cleanup()
    angle = 15
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/20')
def move20():
    GPIO.cleanup()
    angle = 20
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/25')
def move25():
    GPIO.cleanup()
    angle = 25
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/30')
def move30():
    GPIO.cleanup()
    angle = 30
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/35')
def move35():
    GPIO.cleanup()
    angle = 35
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/40')
def move40():
    GPIO.cleanup()
    angle = 40
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/45')
def move45():
    GPIO.cleanup()
    angle = 45
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l5')
def movel5():
    GPIO.cleanup()
    angle = -5
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l10')
def movel10():
    GPIO.cleanup()
    angle = -10
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l15')
def movel15():
    GPIO.cleanup()
    angle = -15
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l20')
def movel20():
    GPIO.cleanup()
    angle = -20
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l25')
def movel25():
    GPIO.cleanup()
    angle = -25
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l30')
def movel30():
    GPIO.cleanup()
    angle = -30
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l35')
def movel35():
    GPIO.cleanup()
    angle = -35
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l40')
def movel40():
    GPIO.cleanup()
    angle = -40
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/l45')
def movel45():
    GPIO.cleanup()
    angle = -45
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})
@app.route('/l0')
def movel0():
    GPIO.cleanup()
    angle = -0
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

@app.route('/0')
def move0():
    GPIO.cleanup()
    angle = 0
    x = (int(angle) +72)/18
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(26, GPIO.OUT)
    GPIO.setwarnings(False)
    pwm=GPIO.PWM(26,50)
    pwm.start(x)
    time.sleep(1)
    return jsonify({})

if __name__ == '__main__':
    app.run(host='0.0.0.0', port =80, debug=True, threaded=True)


from django.urls import path

from .views import *

urlpatterns = [
    path("", index_dao, name="index_dao"),
    path("course/", course, name="courses"),  # страница курсы
    path("<str:page>/", other_page, name="other"),  # выход на другие статичные страницы сайта
    path("accounts/login/", DaoLoginView.as_view(), name="login"),  # страница входа пользователя
    path("accounts/profile/", profile, name="profile"),  # страница личного кабинета пользователя
    path("accounts/logout/", DaoLogoutView.as_view(), name="logout"),  # выход пользователя

    # страница завершения регистрации
    path("accounts/register/done/", DaoRegisterDoneView.as_view(), name="register_done"),
    path("accounts/register/", DaoRegisterUserView.as_view(), name="register"),  # страница регистрации пользователя
    path("accounts/register/activate/<str:sign>/", user_activate, name="register_activate"),  # активация пользователя
]


from collections.abc import Sequence
from pathlib import Path, PurePath
from typing import Any, Dict, List, Optional, Union

import numpy as np
import pandas as pd

datasource_default_types = ["train", "test"]

def check_datasource_type(datasource_type: str):
    if datasource_type in datasource_default_types:
        return datasource_type
    raise TypeError(f"Unknown datasource {datasource_type}.")

def check_dir_exists(path: Union[str, Path]) -> Path:
    if isinstance(path, str):
        path = Path(path)
    if path.is_dir():
        return path
    raise FileExistsError(f"Can't find directory at {path}")

def get_image_paths(image_dir: Path):
    return [img_path for img_path in image_dir.rglob("*/*.*")]

def get_labels_from_image_paths(image_paths: List[Path]) -> Dict[str, str]:
    labels = {
        image_path.name: image_path.parent.name for image_path in image_paths
    }
    return labels

def check_labels_and_predictions(
    labels: Union[Dict, Sequence], image_names: List[str] = None
) -> Optional[Dict[str, Any]]:
    if labels is None:
        return None

    if isinstance(labels, dict):
        assert set(image_names) == set(labels.keys())
        return labels

    if (
        isinstance(labels, Sequence)
        or isinstance(labels, np.ndarray)
        or isinstance(labels, pd.Series)
    ) and not isinstance(labels, str):
        labels_dict = dict(zip(image_names, labels))
        return labels_dict

    raise TypeError(f"Unsupported labels type: {type(labels)}.")

def convert_to_path(image_paths: Sequence):
    sample = image_paths[0]

    if isinstance(image_paths[0], str):
        return [Path(path) for path in image_paths]

    if isinstance(image_paths[0], PurePath):
        return image_paths

    raise TypeError(f"Unsupported image path type: {type(sample)}.")


# Copyright (c) OpenMMLab. All rights reserved.
import torch

from mmdet3d.core import bbox3d2result, build_prior_generator
from mmdet3d.core.bbox.structures.utils import get_proj_mat_by_coord_type
from mmdet3d.models.fusion_layers.point_fusion import point_sample
from mmdet.models.detectors import BaseDetector
from ..builder import DETECTORS, build_backbone, build_head, build_neck

@DETECTORS.register_module()
class ImVoxelNet(BaseDetector):
    r"""`ImVoxelNet <https://arxiv.org/abs/2106.01178>`_.

    Args:
        backbone (dict): Config of the backbone.
        neck (dict): Config of the 2d neck.
        neck_3d (dict): Config of the 3d neck.
        bbox_head (dict): Config of the head.
        prior_generator (dict): Config of the prior generator.
        n_voxels (tuple[int]): Number of voxels for x, y, and z axis.
        coord_type (str): The type of coordinates of points cloud:
            'DEPTH', 'LIDAR', or 'CAMERA'.
        train_cfg (dict, optional): Config for train stage. Defaults to None.
        test_cfg (dict, optional): Config for test stage. Defaults to None.
        init_cfg (dict, optional): Config for weight initialization.
            Defaults to None.
        pretrained (str, optional): Deprecated initialization parameter.
            Defaults to None.
    """

    def __init__(self,
                 backbone,
                 neck,
                 neck_3d,
                 bbox_head,
                 prior_generator,
                 n_voxels,
                 coord_type,
                 train_cfg=None,
                 test_cfg=None,
                 init_cfg=None,
                 pretrained=None):
        super().__init__(init_cfg=init_cfg)
        self.backbone = build_backbone(backbone)
        self.neck = build_neck(neck)
        self.neck_3d = build_neck(neck_3d)
        bbox_head.update(train_cfg=train_cfg)
        bbox_head.update(test_cfg=test_cfg)
        self.bbox_head = build_head(bbox_head)
        self.n_voxels = n_voxels
        self.coord_type = coord_type
        self.prior_generator = build_prior_generator(prior_generator)
        self.train_cfg = train_cfg
        self.test_cfg = test_cfg

    def extract_feat(self, img, img_metas):
        """Extract 3d features from the backbone -> fpn -> 3d projection.

        -> 3d neck -> bbox_head.

        Args:
            img (torch.Tensor): Input images of shape (N, C_in, H, W).
            img_metas (list): Image metas.

        Returns:
            Tuple:
             - torch.Tensor: Features of shape (N, C_out, N_x, N_y, N_z).
             - torch.Tensor: Valid mask of shape (N, 1, N_x, N_y, N_z).
        """
        x = self.backbone(img)
        x = self.neck(x)[0]
        points = self.prior_generator.grid_anchors([self.n_voxels[::-1]],
                                                   device=img.device)[0][:, :3]
        volumes, valid_preds = [], []
        for feature, img_meta in zip(x, img_metas):
            img_scale_factor = (
                points.new_tensor(img_meta['scale_factor'][:2])
                if 'scale_factor' in img_meta.keys() else 1)
            img_flip = img_meta['flip'] if 'flip' in img_meta.keys() else False
            img_crop_offset = (
                points.new_tensor(img_meta['img_crop_offset'])
                if 'img_crop_offset' in img_meta.keys() else 0)
            proj_mat = points.new_tensor(
                get_proj_mat_by_coord_type(img_meta, self.coord_type))
            volume = point_sample(
                img_meta,
                img_features=feature[None, ...],
                points=points,
                proj_mat=points.new_tensor(proj_mat),
                coord_type=self.coord_type,
                img_scale_factor=img_scale_factor,
                img_crop_offset=img_crop_offset,
                img_flip=img_flip,
                img_pad_shape=img.shape[-2:],
                img_shape=img_meta['img_shape'][:2],
                aligned=False)
            volumes.append(
                volume.reshape(self.n_voxels[::-1] + [-1]).permute(3, 2, 1, 0))
            valid_preds.append(
                ~torch.all(volumes[-1] == 0, dim=0, keepdim=True))
        x = torch.stack(volumes)
        x = self.neck_3d(x)
        x = self.bbox_head(x)
        return x, torch.stack(valid_preds).float()

    def forward_train(self, img, img_metas, gt_bboxes_3d, gt_labels_3d,
                      **kwargs):
        """Forward of training.

        Args:
            img (torch.Tensor): Input images of shape (N, C_in, H, W).
            img_metas (list): Image metas.
            gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): gt bboxes of each batch.
            gt_labels_3d (list[torch.Tensor]): gt class labels of each batch.

        Returns:
            dict[str, torch.Tensor]: A dictionary of loss components.
        """
        x, valid_preds = self.extract_feat(img, img_metas)
        # For indoor datasets ImVoxelNet uses ImVoxelHead that handles
        # mask of visible voxels.
        if self.coord_type == 'DEPTH':
            x += (valid_preds, )
        losses = self.bbox_head.loss(*x, gt_bboxes_3d, gt_labels_3d, img_metas)
        return losses

    def forward_test(self, img, img_metas, **kwargs):
        """Forward of testing.

        Args:
            img (torch.Tensor): Input images of shape (N, C_in, H, W).
            img_metas (list): Image metas.

        Returns:
            list[dict]: Predicted 3d boxes.
        """
        # not supporting aug_test for now
        return self.simple_test(img, img_metas)

    def simple_test(self, img, img_metas):
        """Test without augmentations.

        Args:
            img (torch.Tensor): Input images of shape (N, C_in, H, W).
            img_metas (list): Image metas.

        Returns:
            list[dict]: Predicted 3d boxes.
        """
        x, valid_preds = self.extract_feat(img, img_metas)
        # For indoor datasets ImVoxelNet uses ImVoxelHead that handles
        # mask of visible voxels.
        if self.coord_type == 'DEPTH':
            x += (valid_preds, )
        bbox_list = self.bbox_head.get_bboxes(*x, img_metas)
        bbox_results = [
            bbox3d2result(det_bboxes, det_scores, det_labels)
            for det_bboxes, det_scores, det_labels in bbox_list
        ]
        return bbox_results

    def aug_test(self, imgs, img_metas, **kwargs):
        """Test with augmentations.

        Args:
            imgs (list[torch.Tensor]): Input images of shape (N, C_in, H, W).
            img_metas (list): Image metas.

        Returns:
            list[dict]: Predicted 3d boxes.
        """
        raise NotImplementedError


import sqlite3 as sql
from sqlite3 import Error
import os

tables_nonpets = ('expense', 'images', 'meds', 'stats', 'vet')

def create_connection(db_file="pets.db"):
    """ create a database connection to the SQLite database
        specified by the db_file
    :param db_file: database file
    :return: Connection object or None
    """
    try:
        conn = sql.connect(db_file)
        return conn
    except Error as e:
        print(e)

    return None

def insert_record(tablename, values_):
    conn = create_connection()
    c = conn.cursor()

    keys = ','.join(values_.keys())
    question_marks = ','.join(list('?'*len(values_)))
    data_ = tuple(values_.values())

    c.execute('INSERT INTO '+tablename+' ('+keys+') VALUES ('+question_marks+')', data_)
    conn.commit()
    conn.close()

def delete_records(tablename, column_, value_):
    # when deleting a pet all related records in other tables will be deleted
    conn = create_connection()
    c = conn.cursor()

    c.execute('DELETE FROM '+ tablename +' WHERE ' + column_ + ' = ' + value_)

    for t in tables_nonpets:
        c.execute('DELETE FROM '+ t +' WHERE id = ' + value_)

    conn.commit()
    conn.close()

def delete_record_from_table(tablename, column_, value_):
    # for non-pets tables --> pass the record id / to delete all records for pet, delete from pets table
    conn = create_connection()
    c = conn.cursor()

    c.execute('DELETE FROM '+ tablename +' WHERE ' + column_ + ' = ' + value_)

    conn.commit()
    conn.close()

def find_records(tablename, pet_id):
    conn = create_connection()
    c = conn.cursor()

    if tablename == 'pets':
        c.execute('SELECT * FROM '+ tablename +' WHERE id = ' + str(pet_id))
    else:
        c.execute('SELECT * FROM '+ tablename +' WHERE id = ' + str(pet_id) + ' ORDER BY date DESC')
    results = c.fetchall()
    conn.close()
    return results

# test = ""
# field_ = ""
# data_to_enter = {}
# continue_ = ""

# test = input("start? ")

#find data
print(find_records('vet', 7))

#delete data
# while test != "N":
#     table_ = input("Which Table to delete from? ")
#     column_ = input("Which Column - use id? ")
#     value_ = input ("record to delete matches: ")

#     delete_record(table_, column_, value_)

#     test = input("again? ")

#add data
# while test != 'n':
#     tablename = input("Table name: ")

#     while continue_ != "y":
#         print("new field record --------- N to fin")
#         field_ = input("table column: ")
#         value_ = input("input value: ")
#         data_to_enter[field_] = value_
#         continue_ = input("done? ")

#     keys = ','.join(data_to_enter.keys())
#     question_marks = ','.join(list('?'*len(data_to_enter)))
#     values = tuple(data_to_enter.values())

#     print('INSERT INTO '+tablename+' ('+keys+') VALUES ('+question_marks+')', values)
#     insert_record(tablename, data_to_enter)
#     test = input("more? ")


import pygame
from clock import Clock
from constants import WIDTH, HEIGHT

WIN = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption('Analog Clock v1.0 by AW')
FPS = 60 # minimum 1

def main():
    run = True
    clock = pygame.time.Clock()
    clock_face = Clock(WIN)

    while run:
        clock.tick(FPS)

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False

            if event.type == pygame.MOUSEBUTTONDOWN:
                LMB, MMB, RMB = pygame.mouse.get_pressed(3)
                if LMB:
                    print(clock_face.get_hours())
                elif MMB:
                    print(clock_face.get_mins())
                elif RMB:
                    print(clock_face.get_secs())

        clock_face.update()

    pygame.quit()

main()


import optuna

if optuna.type_checking.TYPE_CHECKING:
    from typing import Dict  # NOQA
    from typing import Optional  # NOQA

try:
    from pytorch_lightning.callbacks import EarlyStopping
    _available = True
except (ImportError, SyntaxError) as e:
    # SyntaxError is raised with Python versions below 3.6 since PyTorch Lightning does not
    # support them.
    _import_error = e
    # PyTorchLightningPruningCallback is disabled because PyTorch Lightning is not available.
    _available = False
    EarlyStopping = object

class PyTorchLightningPruningCallback(EarlyStopping):
    """PyTorch Lightning callback to prune unpromising trials.

    Example:

        Add a pruning callback which observes validation accuracy.

        .. code::

            trainer.pytorch_lightning.Trainer(
                early_stop_callback=PyTorchLightningPruningCallback(trial, monitor='avg_val_acc'))

    Args:
        trial:
            A :class:`~optuna.trial.Trial` corresponding to the current evaluation of the
            objective function.
        monitor:
            An evaluation metric for pruning, e.g., ``val_loss`` or
            ``val_acc``. The metrics are obtained from the returned dictionaries from e.g.
            ``pytorch_lightning.LightningModule.training_step`` or
            ``pytorch_lightning.LightningModule.validation_end`` and the names thus depend on
            how this dictionary is formatted.
    """

    def __init__(self, trial, monitor):
        # type: (optuna.trial.Trial, str) -> None

        super(PyTorchLightningPruningCallback, self).__init__()

        _check_pytorch_lightning_availability()

        self._trial = trial
        self._monitor = monitor

    def on_epoch_end(self, epoch, logs=None):
        # type: (int, Optional[Dict[str, float]]) -> None

        logs = logs or {}
        current_score = logs.get(self._monitor)
        if current_score is None:
            return
        self._trial.report(current_score, step=epoch)
        if self._trial.should_prune():
            message = "Trial was pruned at epoch {}.".format(epoch)
            raise optuna.exceptions.TrialPruned(message)

def _check_pytorch_lightning_availability():
    # type: () -> None

    if not _available:
        raise ImportError(
            'PyTorch Lightning is not available. Please install PyTorch Lightning to use this '
            'feature. PyTorch Lightning can be installed by executing `$ pip install '
            'pytorch-lightning`. For further information, please refer to the installation guide '
            'of PyTorch Lightining. (The actual import error is as follows: '
            + str(_import_error) + ')')


##################################################
#                                                #
# File by 0Stormy                                #
# Github: https://github.com/Kinda-Stormy/tfm    #
#                                                #
##################################################

# Packages
import os
import platform
import keyboard
import time
import cursor
import subprocess
from multiprocessing import Process
from colorama import Fore, Back, Style

# Functions \/

# Draw the topbar
def drawTopBar():
    dirStringLen = len(currentDirectory + ' | Press H For help')
    print(f'{Back.CYAN}{currentDirectory} | Press H For help' + ' ' * (terminalX - dirStringLen) + Style.RESET_ALL)

# Create tempfile
def createTempfile():

    try:
        with open('tfm', 'w') as f:
            f.write(os.path.expanduser("~"))
    except:
        print("Can't open tempfile")
# Read tempfile
def readTempfile():
    try:
        with open('tfm', 'r') as f:
            global currentDirectory
            currentDirectory = f.read()
    except:
        print("Can't open tempfile")

# Clear the screen
def clearScreen():
    if platform.system() == 'Windows':
        os.system('cls')
    else:
        os.system('clear')

# Draw contents
def drawContents():
    folderContents = os.listdir(currentDirectory)
    listLength = len(folderContents)
    terminalX = os.get_terminal_size().columns
    int(terminalX)

    for i in range(listLength):
        if os.path.isfile(f'{currentDirectory}/{folderContents[i]}') == False:
            print(Fore.YELLOW, folderContents[i], Style.RESET_ALL)
        else:
            print(Fore.LIGHTBLUE_EX, folderContents[i], Style.RESET_ALL)

        if i == (terminalY - 4):
            break

# Main loop
def mainLoop():
    while True:
        if keyboard.is_pressed("d"):
            global haultLoop
            haultLoop = True
            subprocess.run(['python', 'cd.py'])
            haultLoop = False

            if haultLoop == True:
                while haultLoop == True:
                    time.sleep(0.1)
        clearScreen()
        readTempfile()
        cursor.hide()
        global terminalX
        global terminalY
        terminalY = os.get_terminal_size().lines
        terminalX = os.get_terminal_size().columns
        int(terminalX)
        int(terminalY)
        drawTopBar()
        drawContents()
        time.sleep(0.2)

# -------------------- #

createTempfile()

mainLoop()


"""Configuration."""
import numpy as np
from sklearn.linear_model import LinearRegression, RidgeCV

class Config:
    """Configuration."""

    #: Extra features and how to create them.
    EXTRA_FEATURES = (
        ('1/workers', lambda df: 1/df.workers),
        ('log(input/workers)', lambda df: np.log2(df.input/df.workers)),
        ('log(input)', lambda df: np.log2(df.input)),
        ('log(workers)', lambda df: np.log2(df.workers)),
        ('log(duration_ms)', lambda df: np.log2(df.duration_ms)),
    )

    #: Features for log regression.
    LOG_FEATURES = ['log(input)', 'log(workers)']
    #: What we are trying to predict in log regression.
    LOG_Y = 'log(duration_ms)'

    #: Features for regular regression.
    FEATURES = ['input', 'workers', '1/workers', 'log(input/workers)',
                'log(input)', 'log(workers)']
    #: What we are trying to predict in regular regression.
    Y = 'duration_ms'

    #: Polynomial expansion degree.
    DEGREE = 2
    #: Columns that should not be expanded.
    NO_EXPANSION = ['application', 'set', 'duration_ms', 'log(duration_ms)']

    #: linear models
    LINEAR_MODELS = (LinearRegression(),
                     RidgeCV(normalize=True, alphas=(0.01, 0.1, 1, 3, 10)))

    #: Features to remove because of long time required for processing.
    #: Factors must be sorted.
    DEL_FEATURES = [
        'input * log(input/workers)',
        # 'input * log(input)',
        # 'input * log(workers)',
        '1/workers * workers',
        'log(input/workers) * workers',
        # 'log(input) * workers',
        # 'log(workers) * workers',
        '1/workers * log(input/workers)',
        # '1/workers * log(input)',
        '1/workers * log(workers)',
        'log(input) * log(input/workers)',
        'log(input/workers) * log(workers)',
    ]