import datetime
from functools import partial

from django.test.utils import override_settings
from django_webtest import WebTest
from model_bakery import baker

from evap.evaluation.models import (
    Answer,
    Contribution,
    Evaluation,
    Question,
    Questionnaire,
    QuestionType,
    RatingAnswerCounter,
    Semester,
    TextAnswer,
    UserProfile,
    VoteTimestamp,
)
from evap.evaluation.tests.tools import FuzzyInt, WebTestWith200Check, render_pages
from evap.student.tools import answer_field_id
from evap.student.views import SUCCESS_MAGIC_STRING

class TestStudentIndexView(WebTestWith200Check):
    url = "/student/"

    @classmethod
    def setUpTestData(cls):
        # View is only visible to users participating in at least one evaluation.
        cls.user = baker.make(UserProfile, email="student@institution.example.com")
        baker.make(Evaluation, participants=[cls.user])

        cls.test_users = [cls.user]

    def test_num_queries_is_constant(self):
        semester1 = baker.make(Semester)
        semester2 = baker.make(Semester, participations_are_archived=True)

        for semester in [semester1, semester2]:
            evaluations = baker.make(
                Evaluation,
                course__semester=semester,
                state=Evaluation.State.PUBLISHED,
                _quantity=100,
                _bulk_create=True,
            )
            participations = [Evaluation.participants.through(evaluation=e, userprofile=self.user) for e in evaluations]
            Evaluation.participants.through.objects.bulk_create(participations)

        with self.assertNumQueries(FuzzyInt(0, 100)):
            self.app.get(self.url, user=self.user)

@override_settings(INSTITUTION_EMAIL_DOMAINS=["example.com"])
class TestVoteView(WebTest):
    render_pages_url = "/student/vote/PK"

    @classmethod
    def setUpTestData(cls):
        cls.voting_user1 = baker.make(UserProfile, email="voting_user1@institution.example.com")
        cls.voting_user2 = baker.make(UserProfile, email="voting_user2@institution.example.com")
        cls.contributor1 = baker.make(UserProfile, email="contributor1@institution.example.com")
        cls.contributor2 = baker.make(UserProfile, email="contributor2@institution.example.com")

        cls.evaluation = baker.make(
            Evaluation,
            participants=[cls.voting_user1, cls.voting_user2, cls.contributor1],
            state=Evaluation.State.IN_EVALUATION,
        )
        cls.url = f"/student/vote/{cls.evaluation.pk}"

        cls.top_general_questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.TOP)
        cls.bottom_general_questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.BOTTOM)
        cls.contributor_questionnaire = baker.make(Questionnaire, type=Questionnaire.Type.CONTRIBUTOR)

        cls.contributor_heading_question = baker.make(
            Question, questionnaire=cls.contributor_questionnaire, order=0, type=QuestionType.HEADING
        )
        cls.contributor_text_question = baker.make(
            Question, questionnaire=cls.contributor_questionnaire, order=1, type=QuestionType.TEXT
        )
        cls.contributor_likert_question = baker.make(
            Question, questionnaire=cls.contributor_questionnaire, order=2, type=QuestionType.POSITIVE_LIKERT
        )

        cls.top_heading_question = baker.make(
            Question, questionnaire=cls.top_general_questionnaire, order=0, type=QuestionType.HEADING
        )
        cls.top_text_question = baker.make(
            Question, questionnaire=cls.top_general_questionnaire, order=1, type=QuestionType.TEXT
        )
        cls.top_likert_question = baker.make(
            Question, questionnaire=cls.top_general_questionnaire, order=2, type=QuestionType.POSITIVE_LIKERT
        )
        cls.top_grade_question = baker.make(
            Question, questionnaire=cls.top_general_questionnaire, order=3, type=QuestionType.GRADE
        )

        cls.bottom_heading_question = baker.make(
            Question, questionnaire=cls.bottom_general_questionnaire, order=0, type=QuestionType.HEADING
        )
        cls.bottom_text_question = baker.make(
            Question, questionnaire=cls.bottom_general_questionnaire, order=1, type=QuestionType.TEXT
        )
        cls.bottom_likert_question = baker.make(
            Question, questionnaire=cls.bottom_general_questionnaire, order=2, type=QuestionType.POSITIVE_LIKERT
        )
        cls.bottom_grade_question = baker.make(
            Question, questionnaire=cls.bottom_general_questionnaire, order=3, type=QuestionType.GRADE
        )

        cls.contribution1 = baker.make(
            Contribution,
            contributor=cls.contributor1,
            questionnaires=[cls.contributor_questionnaire],
            evaluation=cls.evaluation,
        )
        cls.contribution2 = baker.make(
            Contribution,
            contributor=cls.contributor2,
            questionnaires=[cls.contributor_questionnaire],
            evaluation=cls.evaluation,
        )

        cls.evaluation.general_contribution.questionnaires.set(
            [cls.top_general_questionnaire, cls.bottom_general_questionnaire]
        )

    @render_pages
    def render_pages(self):
        with_confirmation = self.app.get(self.url, user=self.voting_user1)
        submit_errors = with_confirmation.forms["student-vote-form"].submit()

        with override_settings(SMALL_COURSE_SIZE=2):
            normal = self.app.get(self.url, user=self.voting_user1)

        return {
            "with_textanswer_publish_confirmation": with_confirmation.content,
            "submit_errors": submit_errors.content,
            "normal": normal.content,
        }

    def test_question_ordering(self):
        page = self.app.get(self.url, user=self.voting_user1, status=200)

        top_heading_index = page.body.decode().index(self.top_heading_question.text)
        top_text_index = page.body.decode().index(self.top_text_question.text)

        contributor_heading_index = page.body.decode().index(self.contributor_heading_question.text)
        contributor_likert_index = page.body.decode().index(self.contributor_likert_question.text)

        bottom_heading_index = page.body.decode().index(self.bottom_heading_question.text)
        bottom_grade_index = page.body.decode().index(self.bottom_grade_question.text)

        self.assertTrue(
            top_heading_index
            < top_text_index
            < contributor_heading_index
            < contributor_likert_index
            < bottom_heading_index
            < bottom_grade_index
        )

    def fill_form(self, form, fill_general_complete=True, fill_contributors_complete=True):
        contribution = self.evaluation.general_contribution
        questionnaire = self.top_general_questionnaire
        form[answer_field_id(contribution, questionnaire, self.top_text_question)] = "some text"
        form[answer_field_id(contribution, questionnaire, self.top_grade_question)] = 3
        form[answer_field_id(contribution, questionnaire, self.top_likert_question)] = 1
        form[
            answer_field_id(contribution, questionnaire, self.top_likert_question, additional_textanswer=True)
        ] = "some additional text"

        questionnaire = self.bottom_general_questionnaire
        form[answer_field_id(contribution, questionnaire, self.bottom_text_question)] = "some bottom text"
        form[answer_field_id(contribution, questionnaire, self.bottom_grade_question)] = 4
        if fill_general_complete:
            form[answer_field_id(contribution, questionnaire, self.bottom_likert_question)] = 2

        contribution = self.contribution1
        questionnaire = self.contributor_questionnaire
        form[answer_field_id(contribution, questionnaire, self.contributor_text_question)] = "some other text"
        form[answer_field_id(contribution, questionnaire, self.contributor_likert_question)] = 4
        form[
            answer_field_id(contribution, questionnaire, self.contributor_likert_question, additional_textanswer=True)
        ] = "some other additional text"

        contribution = self.contribution2
        form[answer_field_id(contribution, questionnaire, self.contributor_text_question)] = "some more text"
        if fill_contributors_complete:
            form[answer_field_id(contribution, questionnaire, self.contributor_likert_question)] = 2

    def test_incomplete_general_vote_form(self):
        """
        Submits a student vote, verifies that an error message is displayed if not all general rating questions have
        been answered and that all given answers stay selected/filled.
        """
        page = self.app.get(self.url, user=self.voting_user1.email, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form, fill_general_complete=False)
        response = form.submit(status=200)
        self.assertIn("vote for all rating questions", response)

        form = page.forms["student-vote-form"]

        field_id = partial(answer_field_id, self.evaluation.general_contribution, self.top_general_questionnaire)
        self.assertEqual(form[field_id(self.top_text_question)].value, "some text")
        self.assertEqual(form[field_id(self.top_likert_question)].value, "1")
        self.assertEqual(
            form[field_id(self.top_likert_question, additional_textanswer=True)].value, "some additional text"
        )
        self.assertEqual(form[field_id(self.top_grade_question)].value, "3")

        field_id = partial(answer_field_id, self.evaluation.general_contribution, self.bottom_general_questionnaire)
        self.assertEqual(form[field_id(self.bottom_text_question)].value, "some bottom text")
        self.assertEqual(form[field_id(self.bottom_grade_question)].value, "4")

        field_id = partial(answer_field_id, self.contribution1, self.contributor_questionnaire)
        self.assertEqual(form[field_id(self.contributor_text_question)].value, "some other text")
        self.assertEqual(form[field_id(self.contributor_likert_question)].value, "4")
        self.assertEqual(
            form[field_id(self.contributor_likert_question, additional_textanswer=True)].value,
            "some other additional text",
        )

        field_id = partial(answer_field_id, self.contribution2, self.contributor_questionnaire)
        self.assertEqual(form[field_id(self.contributor_text_question)].value, "some more text")
        self.assertEqual(form[field_id(self.contributor_likert_question)].value, "2")

    def test_incomplete_contributors_vote_form(self):
        """
        Submits a student vote, verifies that an error message is displayed if not all rating questions about
        contributors have been answered and that all given answers stay selected/filled.
        """
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form, fill_contributors_complete=False)
        response = form.submit(status=200)
        self.assertIn("vote for all rating questions", response)

        form = page.forms["student-vote-form"]

        field_id = partial(answer_field_id, self.evaluation.general_contribution, self.top_general_questionnaire)
        self.assertEqual(form[field_id(self.top_text_question)].value, "some text")
        self.assertEqual(form[field_id(self.top_likert_question)].value, "1")
        self.assertEqual(
            form[field_id(self.top_likert_question, additional_textanswer=True)].value, "some additional text"
        )
        self.assertEqual(form[field_id(self.top_grade_question)].value, "3")

        field_id = partial(answer_field_id, self.evaluation.general_contribution, self.bottom_general_questionnaire)
        self.assertEqual(form[field_id(self.bottom_text_question)].value, "some bottom text")
        self.assertEqual(form[field_id(self.bottom_likert_question)].value, "2")
        self.assertEqual(form[field_id(self.bottom_grade_question)].value, "4")

        field_id = partial(answer_field_id, self.contribution1, self.contributor_questionnaire)
        self.assertEqual(form[field_id(self.contributor_text_question)].value, "some other text")
        self.assertEqual(form[field_id(self.contributor_likert_question)].value, "4")
        self.assertEqual(
            form[field_id(self.contributor_likert_question, additional_textanswer=True)].value,
            "some other additional text",
        )

        field_id = partial(answer_field_id, self.contribution2, self.contributor_questionnaire)
        self.assertEqual(form[field_id(self.contributor_text_question)].value, "some more text")

    def test_answer(self):
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form)
        response = form.submit()
        self.assertEqual(SUCCESS_MAGIC_STRING, response.body.decode())

        page = self.app.get(self.url, user=self.voting_user2, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form)
        response = form.submit()
        self.assertEqual(SUCCESS_MAGIC_STRING, response.body.decode())

        self.assertEqual(len(TextAnswer.objects.all()), 12)
        self.assertEqual(len(RatingAnswerCounter.objects.all()), 6)

        self.assertEqual(RatingAnswerCounter.objects.filter(question=self.top_likert_question).count(), 1)
        self.assertEqual(RatingAnswerCounter.objects.get(question=self.top_likert_question).answer, 1)

        self.assertEqual(RatingAnswerCounter.objects.filter(question=self.top_grade_question).count(), 1)
        self.assertEqual(RatingAnswerCounter.objects.get(question=self.top_grade_question).answer, 3)

        self.assertEqual(RatingAnswerCounter.objects.filter(question=self.bottom_likert_question).count(), 1)
        self.assertEqual(RatingAnswerCounter.objects.get(question=self.bottom_likert_question).answer, 2)

        self.assertEqual(RatingAnswerCounter.objects.filter(question=self.bottom_grade_question).count(), 1)
        self.assertEqual(RatingAnswerCounter.objects.get(question=self.bottom_grade_question).answer, 4)

        self.assertEqual(RatingAnswerCounter.objects.filter(question=self.contributor_likert_question).count(), 2)
        self.assertEqual(
            RatingAnswerCounter.objects.get(
                question=self.contributor_likert_question, contribution=self.contribution1
            ).answer,
            4,
        )
        self.assertEqual(
            RatingAnswerCounter.objects.get(
                question=self.contributor_likert_question, contribution=self.contribution2
            ).answer,
            2,
        )

        self.assertEqual(TextAnswer.objects.filter(question=self.top_text_question).count(), 2)
        self.assertEqual(TextAnswer.objects.filter(question=self.top_likert_question).count(), 2)
        self.assertEqual(TextAnswer.objects.filter(question=self.bottom_text_question).count(), 2)
        self.assertEqual(TextAnswer.objects.filter(question=self.contributor_text_question).count(), 4)
        self.assertEqual(TextAnswer.objects.filter(question=self.contributor_likert_question).count(), 2)

        self.assertEqual(
            TextAnswer.objects.filter(question=self.top_text_question)[0].contribution,
            self.evaluation.general_contribution,
        )
        self.assertEqual(
            TextAnswer.objects.filter(question=self.top_text_question)[1].contribution,
            self.evaluation.general_contribution,
        )

        answers = TextAnswer.objects.filter(
            question=self.contributor_text_question, contribution=self.contribution1
        ).values_list("answer", flat=True)
        self.assertEqual(list(answers), ["some other text"] * 2)

        answers = TextAnswer.objects.filter(
            question=self.contributor_likert_question, contribution=self.contribution1
        ).values_list("answer", flat=True)
        self.assertEqual(list(answers), ["some other additional text"] * 2)

        answers = TextAnswer.objects.filter(
            question=self.contributor_text_question, contribution=self.contribution2
        ).values_list("answer", flat=True)
        self.assertEqual(list(answers), ["some more text"] * 2)

        answers = TextAnswer.objects.filter(
            question=self.top_text_question, contribution=self.evaluation.general_contribution
        ).values_list("answer", flat=True)
        self.assertEqual(list(answers), ["some text"] * 2)

        answers = TextAnswer.objects.filter(
            question=self.top_likert_question, contribution=self.evaluation.general_contribution
        ).values_list("answer", flat=True)
        self.assertEqual(list(answers), ["some additional text"] * 2)

        answers = TextAnswer.objects.filter(
            question=self.bottom_text_question, contribution=self.evaluation.general_contribution
        ).values_list("answer", flat=True)
        self.assertEqual(list(answers), ["some bottom text"] * 2)

    def test_vote_timestamp(self):
        time_before = datetime.datetime.now()
        timestamps_before = VoteTimestamp.objects.count()
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form)
        form.submit()
        self.assertEqual(VoteTimestamp.objects.count(), timestamps_before + 1)
        time = VoteTimestamp.objects.latest("timestamp").timestamp
        self.assertTrue(time_before < time < datetime.datetime.now())

    def test_user_cannot_vote_multiple_times(self):
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form)
        form.submit()

        self.app.get(self.url, user=self.voting_user1, status=403)

    def test_user_cannot_vote_for_themselves(self):
        response = self.app.get(self.url, user=self.contributor1, status=200)

        for contributor, __, __, __, __ in response.context["contributor_form_groups"]:
            self.assertNotEqual(
                contributor, self.contributor1, "Contributor should not see the questionnaire about themselves"
            )

        response = self.app.get(self.url, user=self.voting_user1, status=200)
        self.assertTrue(
            any(
                contributor == self.contributor1
                for contributor, __, __, __, __ in response.context["contributor_form_groups"]
            ),
            "Regular students should see the questionnaire about a contributor",
        )

    def test_user_logged_out(self):
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form)

        page = page.forms["logout-form"].submit(status=302)

        response = form.submit(status=302)
        self.assertNotIn(SUCCESS_MAGIC_STRING, response)

    def test_midterm_evaluation_warning(self):
        evaluation_warning = "The results of this evaluation will be published while the course is still running."
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        self.assertNotIn(evaluation_warning, page)

        self.evaluation.is_midterm_evaluation = True
        self.evaluation.save()

        page = self.app.get(self.url, user=self.voting_user1, status=200)
        self.assertIn(evaluation_warning, page)

    @override_settings(SMALL_COURSE_SIZE=5)
    def test_small_evaluation_size_warning_shown(self):
        small_evaluation_size_warning = "Only a small number of people can take part in this evaluation."
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        self.assertIn(small_evaluation_size_warning, page)

    @override_settings(SMALL_COURSE_SIZE=2)
    def test_small_evaluation_size_warning_not_shown(self):
        small_evaluation_size_warning = "Only a small number of people can take part in this evaluation."
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        self.assertNotIn(small_evaluation_size_warning, page)

    def helper_test_answer_publish_confirmation(self, form_element):
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        form = page.forms["student-vote-form"]
        self.fill_form(form)
        if form_element:
            form[form_element] = True
        response = form.submit()
        self.assertEqual(SUCCESS_MAGIC_STRING, response.body.decode())
        evaluation = Evaluation.objects.get(pk=self.evaluation.pk)
        if form_element:
            self.assertTrue(evaluation.can_publish_text_results)
        else:
            self.assertFalse(evaluation.can_publish_text_results)

    def test_user_checked_top_textanswer_publish_confirmation(self):
        self.helper_test_answer_publish_confirmation("text_results_publish_confirmation_top")

    def test_user_checked_bottom_textanswer_publish_confirmation(self):
        self.helper_test_answer_publish_confirmation("text_results_publish_confirmation_bottom")

    def test_user_did_not_check_textanswer_publish_confirmation(self):
        self.helper_test_answer_publish_confirmation(None)

    def test_textanswer_visibility_is_shown(self):
        page = self.app.get(self.url, user=self.voting_user1, status=200)
        self.assertRegex(
            page.body.decode(),
            r"can be seen by:<br />\s*{}".format(self.contributor1.full_name.replace("(", "\\(").replace(")", "\\)")),
        )

    def test_xmin_of_all_answers_is_updated(self):
        page = self.app.get(self.url, user=self.voting_user1)
        form = page.forms["student-vote-form"]
        self.fill_form(form)
        form.submit()

        page = self.app.get(self.url, user=self.voting_user2)
        form = page.forms["student-vote-form"]
        self.fill_form(form)
        form[
            answer_field_id(
                self.evaluation.general_contribution, self.top_general_questionnaire, self.top_grade_question
            )
        ] = 2
        form.submit()

        self.assertEqual(
            set(Answer.__subclasses__()),
            {RatingAnswerCounter, TextAnswer},
            "This test requires an update if a new answer type is added. Also, when adding a new answer type, "
            "the new table should probably also be vacuumed and clustered -- see and update "
            "https://github.com/e-valuation/EvaP/wiki/Installation#database-vacuuming-and-clustering",
        )

        query = RatingAnswerCounter.objects.raw("SELECT id, xmin FROM evaluation_ratinganswercounter")
        rating_answer_xmins = [row.xmin for row in query]
        self.assertTrue(all(xmin == rating_answer_xmins[0] for xmin in rating_answer_xmins))

        query = TextAnswer.objects.raw("SELECT id, xmin FROM evaluation_textanswer")
        text_answer_xmins = [row.xmin for row in query]
        self.assertTrue(all(xmin == text_answer_xmins[0] for xmin in text_answer_xmins))


import os

import numpy as np

import pandas as pd

import lightgbm

import warnings

warnings.filterwarnings('ignore')

from IPython.display import FileLink

from sklearn.metrics import roc_auc_score

from fastai.structured import proc_df, train_cats

from sklearn.model_selection import train_test_split

print(os.listdir("../input"))
dtypes = {

        'MachineIdentifier':                                    'category',

        'ProductName':                                          'category',

        'EngineVersion':                                        'category',

        'AppVersion':                                           'category',

        'AvSigVersion':                                         'category',

        'IsBeta':                                               'int8',

        'RtpStateBitfield':                                     'float16',

        'IsSxsPassiveMode':                                     'int8',

        'DefaultBrowsersIdentifier':                            'float16',

        'AVProductStatesIdentifier':                            'float32',

        'AVProductsInstalled':                                  'float16',

        'AVProductsEnabled':                                    'float16',

        'HasTpm':                                               'int8',

        'CountryIdentifier':                                    'int16',

        'CityIdentifier':                                       'float32',

        'OrganizationIdentifier':                               'float16',

        'GeoNameIdentifier':                                    'float16',

        'LocaleEnglishNameIdentifier':                          'int8',

        'Platform':                                             'category',

        'Processor':                                            'category',

        'OsVer':                                                'category',

        'OsBuild':                                              'int16',

        'OsSuite':                                              'int16',

        'OsPlatformSubRelease':                                 'category',

        'OsBuildLab':                                           'category',

        'SkuEdition':                                           'category',

        'IsProtected':                                          'float16',

        'AutoSampleOptIn':                                      'int8',

        'PuaMode':                                              'category',

        'SMode':                                                'float16',

        'IeVerIdentifier':                                      'float16',

        'SmartScreen':                                          'category',

        'Firewall':                                             'float16',

        'UacLuaenable':                                         'float32',

        'Census_MDC2FormFactor':                                'category',

        'Census_DeviceFamily':                                  'category',

        'Census_OEMNameIdentifier':                             'float16',

        'Census_OEMModelIdentifier':                            'float32',

        'Census_ProcessorCoreCount':                            'float16',

        'Census_ProcessorManufacturerIdentifier':               'float16',

        'Census_ProcessorModelIdentifier':                      'float16',

        'Census_ProcessorClass':                                'category',

        'Census_PrimaryDiskTotalCapacity':                      'float32',

        'Census_PrimaryDiskTypeName':                           'category',

        'Census_SystemVolumeTotalCapacity':                     'float32',

        'Census_HasOpticalDiskDrive':                           'int8',

        'Census_TotalPhysicalRAM':                              'float32',

        'Census_ChassisTypeName':                               'category',

        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',

        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',

        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',

        'Census_PowerPlatformRoleName':                         'category',

        'Census_InternalBatteryType':                           'category',

        'Census_InternalBatteryNumberOfCharges':                'float32',

        'Census_OSVersion':                                     'category',

        'Census_OSArchitecture':                                'category',

        'Census_OSBranch':                                      'category',

        'Census_OSBuildNumber':                                 'int16',

        'Census_OSBuildRevision':                               'int32',

        'Census_OSEdition':                                     'category',

        'Census_OSSkuName':                                     'category',

        'Census_OSInstallTypeName':                             'category',

        'Census_OSInstallLanguageIdentifier':                   'float16',

        'Census_OSUILocaleIdentifier':                          'int16',

        'Census_OSWUAutoUpdateOptionsName':                     'category',

        'Census_IsPortableOperatingSystem':                     'int8',

        'Census_GenuineStateName':                              'category',

        'Census_ActivationChannel':                             'category',

        'Census_IsFlightingInternal':                           'float16',

        'Census_IsFlightsDisabled':                             'float16',

        'Census_FlightRing':                                    'category',

        'Census_ThresholdOptIn':                                'float16',

        'Census_FirmwareManufacturerIdentifier':                'float16',

        'Census_FirmwareVersionIdentifier':                     'float32',

        'Census_IsSecureBootEnabled':                           'int8',

        'Census_IsWIMBootEnabled':                              'float16',

        'Census_IsVirtualDevice':                               'float16',

        'Census_IsTouchEnabled':                                'int8',

        'Census_IsPenCapable':                                  'int8',

        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',

        'Wdft_IsGamer':                                         'float16',

        'Wdft_RegionIdentifier':                                'float16',

        'HasDetections':                                        'int8'

        }
imp_col = ['AvSigVersion', 'CountryIdentifier', 'MachineIdentifier','Census_SystemVolumeTotalCapacity', 'Census_FirmwareVersionIdentifier',

           'CityIdentifier', 'Census_OEMModelIdentifier','AVProductStatesIdentifier', 'LocaleEnglishNameIdentifier','Census_OSVersion',

          'AppVersion', 'Census_OSBuildRevision', 'Census_OSUILocaleIdentifier', 'Census_PrimaryDiskTotalCapacity','OsBuildLab',

           'SmartScreen', 'EngineVersion', 'Census_TotalPhysicalRAM', 'Census_ChassisTypeName', 'Census_OSEdition', 'Census_MDC2FormFactor',

          'Census_OSBranch', 'Census_InternalBatteryNumberOfCharges', 'Census_OSSkuName', 'Census_ActivationChannel',

           'Census_OSInstallTypeName', 'HasDetections']

imp_col_test = ['AvSigVersion', 'CountryIdentifier', 'MachineIdentifier','Census_SystemVolumeTotalCapacity', 'Census_FirmwareVersionIdentifier',

           'CityIdentifier', 'Census_OEMModelIdentifier','AVProductStatesIdentifier', 'LocaleEnglishNameIdentifier','Census_OSVersion',

          'AppVersion', 'Census_OSBuildRevision', 'Census_OSUILocaleIdentifier', 'Census_PrimaryDiskTotalCapacity','OsBuildLab',

           'SmartScreen', 'EngineVersion', 'Census_TotalPhysicalRAM', 'Census_ChassisTypeName', 'Census_OSEdition', 'Census_MDC2FormFactor',

          'Census_OSBranch', 'Census_InternalBatteryNumberOfCharges', 'Census_OSSkuName', 'Census_ActivationChannel',

           'Census_OSInstallTypeName']

imp_col2 = ['AvSigVersion', 'CountryIdentifier', 'MachineIdentifier','Census_SystemVolumeTotalCapacity', 'Census_FirmwareVersionIdentifier',

           'CityIdentifier', 'Census_OEMModelIdentifier','AVProductStatesIdentifier', 'LocaleEnglishNameIdentifier','Census_OSVersion',

          'AppVersion', 'Census_OSBuildRevision', 'Census_OSUILocaleIdentifier', 'Census_PrimaryDiskTotalCapacity','OsBuildLab',

           'SmartScreen', 'EngineVersion', 'Census_TotalPhysicalRAM', 'Census_ChassisTypeName', 'Census_OSEdition', 'Census_MDC2FormFactor',

          'Census_OSBranch', 'Census_InternalBatteryNumberOfCharges', 'Census_OSSkuName', 'Census_ActivationChannel_Volume:GVLK',

           'Census_OSInstallTypeName_UUPUpgrade']
df = df_raw[imp_col]

train_cats(df)

X = X[imp_col2]; X.shape
X_trn, X_vld, y_trn, y_vld = train_test_split(X, y, test_size =0.0)
params = {}

params['learning_rate'] = 0.3

params['boosting_type'] = 'gbdt'

params['objective'] = 'binary'

params['metric'] = 'binary_logloss'

params['sub_feature'] = 0.5

params['num_leaves'] = 2**12-1

params['min_data'] = 50

params['max_depth'] = -1

train_data = lightgbm.Dataset(X_trn, label=y_trn)
del(df_raw, df)
del(X, X_trn, X_vld, y_trn, y_vld)
test = df_test[imp_col_test]
train_cats(test)

X = X_test[imp_col2]; X.shape
preds= clf.predict(X)
submission_data = test[['MachineIdentifier']]
submission_data['HasDetections'] = preds
submission_data.to_csv('sub.csv', index=False)
submission_data.head()
FileLink('sub.csv')


import bisect
from collections import defaultdict

class MajorityChecker(object):

    def __init__(self, arr):
        """
        :type arr: List[int]
        """
        self.d = defaultdict(list)
        for i, j in enumerate(arr):
            self.d[j].append(i)

    def query(self, left, right, threshold):
        """
        :type left: int
        :type right: int
        :type threshold: int
        :rtype: int
        """
        for k, v in self.d.items():
            if len(v) < threshold:
                continue
            left_index = bisect.bisect_left(v, left)
            right_index = bisect.bisect_right(v, right)
            if right_index - left_index >= threshold:
                return k
        return -1

# Your MajorityChecker object will be instantiated and called as such:
# obj = MajorityChecker(arr)
# param_1 = obj.query(left,right,threshold)

python/Binary Search/Online Majority Element In Subarray/Online Majority Element In Subarray.py


from ai2_kit.core.script import BashTemplate, BashStep, BashScript
from ai2_kit.core.artifact import Artifact, ArtifactDict
from ai2_kit.core.log import get_logger
from ai2_kit.core.job import gather_jobs
from ai2_kit.core.util import list_split, merge_dict

from typing import List, Optional
from pydantic import BaseModel
from dataclasses import dataclass

import numpy as np
import ase.io
import copy
import os
import re

from .iface import BaseCllContext, ICllExploreOutput
from .constant import DEFAULT_LASP_IN, DEFAULT_LAMMPS_TEMPLATE_FOR_DP_SSW, MODEL_DEVI_OUT
from .data import artifacts_to_ase_atoms, DataFormat

logger = get_logger(__name__)

class CllLaspInputConfig(BaseModel):
    class Potential(BaseModel):
        class LammpsPotential(BaseModel):
            input_template: str = DEFAULT_LAMMPS_TEMPLATE_FOR_DP_SSW

        lammps: Optional[LammpsPotential] = None  # currently only lammps is supported

    input_template: dict
    """
    Input template for LASP
    """
    potential: Potential

    system_files: List[str]
    """
    Initial system files to explore
    """

class CllLaspContextConfig(BaseModel):
    lasp_cmd: str = 'lasp'
    script_template: BashTemplate
    concurrency: int = 5

@dataclass
class CllLaspInput:
    config: CllLaspInputConfig
    type_map: List[str]
    mass_map: List[float]
    models: List[Artifact]
    new_system_files: List[Artifact]

@dataclass
class CllLaspContext(BaseCllContext):
    config: CllLaspContextConfig

@dataclass
class CllLaspOutput(ICllExploreOutput):
    output_dirs: List[Artifact]

    def get_model_devi_dataset(self) -> List[Artifact]:
        return self.output_dirs

async def cll_lasp(input: CllLaspInput, ctx: CllLaspContext):
    executor = ctx.resource_manager.default_executor

    # setup workspace
    work_dir = os.path.join(executor.work_dir, ctx.path_prefix)
    tasks_dir, = executor.setup_workspace(work_dir, ['tasks'])

    # resolve artifacts
    if len(input.new_system_files) > 0:
        systems = input.new_system_files
    else:
        systems = ctx.resource_manager.resolve_artifacts(input.config.system_files)

    # setup configuration
    lasp_in_data = merge_dict(copy.deepcopy(DEFAULT_LASP_IN), input.config.input_template)
    lasp_in_data['explore_type'] = 'ssw'
    lasp_in_data['SSW.output'] = 'T'

    lammps_input_template = None
    if input.config.potential.lammps is not None:
        lasp_in_data['potential'] = 'lammps'
        lammps_input_template = input.config.potential.lammps.input_template
    else:
        raise ValueError('At least one potential should be specified.')

    # make task dirs
    task_dirs = executor.run_python_fn(make_lasp_task_dirs)(
        systems=[a.to_dict() for a in systems],
        type_map=input.type_map, mass_map=input.mass_map,
        lasp_in_data=lasp_in_data,
        dp_models=[m.url for m in input.models],
        lammps_input_template=lammps_input_template,
        base_dir=tasks_dir,
    )

    # generate steps
    lasp_cmd = f'{ctx.config.lasp_cmd} 1 > lasp.out > lasp.err'
    steps = []
    for task_dir in task_dirs :
        steps.append(BashStep(cwd=task_dir['url'], cmd=lasp_cmd, checkpoint='lasp'))

    # submit jobs by the number of concurrency
    jobs = []
    for i, steps_group in enumerate(list_split(steps, ctx.config.concurrency)):
        if not steps_group:
            continue
        script = BashScript(
            template=ctx.config.script_template,
            steps=steps_group,
        )
        job = executor.submit(script.render(), cwd=tasks_dir,
                              checkpoint_key=f'queue-job/lasp/{i}:{tasks_dir}')
        jobs.append(job)
    await gather_jobs(jobs, max_tries=2)

    # process outputs
    executor.run_python_fn(process_lasp_outputs)(task_dirs=[a['url'] for a in task_dirs])

    output_dirs = [
        Artifact.of(
            url=task_dir['url'],
            executor=executor.name,
            format=DataFormat.LASP_LAMMPS_OUT_DIR,
            attrs={ **task_dir['attrs'],  'structures_file': 'structures.xyz'},
        ) for task_dir in task_dirs]  # type: ignore
    return CllLaspOutput(output_dirs=output_dirs)

def __export_remote_functions():

    from string import Template

    class LammpsInputTemplate(Template):
        delimiter = '$$'

    def make_lasp_task_dirs(systems: List[ArtifactDict],
                            lasp_in_data: dict,
                            base_dir: str,
                            type_map: List[str],
                            mass_map: List[float],
                            dp_models: List[str],
                            lammps_input_template: Optional[str],
                            ) -> List[ArtifactDict]:
        input_data = artifacts_to_ase_atoms(systems, type_map=type_map)

        i, task_dirs = 0, []  # TODO: why i is not generated from the loop?
        for artifact, atoms in input_data:
            # create task_dir
            task_dir = os.path.join(base_dir, f'task_{i:06}' )
            os.makedirs(task_dir, exist_ok=True)
            # create lasp.in
            lasp_in_text =  '\n'.join([f'{k:32} {v}' for k, v in lasp_in_data.items()])
            with open(os.path.join(task_dir, 'lasp.in'), 'w', encoding='utf-8') as f:
                f.write(lasp_in_text)
            # create lasp.str
            ase.io.write(os.path.join(task_dir, 'lasp.str'), atoms, format='dmol-arc')
            if lammps_input_template is not None:
                # create lammps.data
                data_file = os.path.join(task_dir, 'lammps.data')
                ase.io.write(data_file, atoms, format='lammps-data', specorder=type_map)  # type: ignore
                # create lammps input: in.simple, PS: Its LASP to blame for the name
                read_data_section = '\n'.join([
                    f"read_data {data_file}",
                    *(f"mass {i+1} {m}" for i, m in enumerate(mass_map))
                ])
                force_field_section = '\n'.join([
                    f"pair_style deepmd {' '.join(dp_models)} out_file {MODEL_DEVI_OUT}",
                    f"pair_coeff * *"
                ])
                lammps_input = LammpsInputTemplate(lammps_input_template).substitute(
                    read_data_section=read_data_section,
                    force_field_section=force_field_section,
                )
                lammps_input_file = os.path.join(task_dir, 'in.simple')
                with open(lammps_input_file, 'w', encoding='utf-8') as f:
                    f.write(lammps_input)
            else:
                raise ValueError('At least one potential should be specified.')
            # the `source` field is required as model_devi will use it to update init structures
            task_dirs.append({'url': task_dir,
                              'attrs': {**artifact['attrs'], 'source': artifact['url']}})

            i += 1  # TODO: refactor this
        return task_dirs

    def process_lasp_output(task_dir: str, file_name='structures.xyz'):
        """
        Align lasp output with model_devi records.

        As allstr.arc contains all the structures generated by LASP,
        we need to use the result of lasp.out for alignment.

        The following code is copy from ChecMate, may need to be refactored
        """
        all_str_file = os.path.join(task_dir, 'allstr.arc')
        lasp_out_file = os.path.join(task_dir, 'lasp.out')
        all_strs = ase.io.read(all_str_file, ':', format='dmol-arc')

        with open(all_str_file, "r") as f:
            all_qs = list((round(float(line[:73].strip().split()[-2]),6) for line in f.readlines() if "Energy" in line))
        with open(lasp_out_file, "r") as f:
            lines = f.readlines()
            traj_qs = list((round(float(line[:73].strip().split()[2]),6) for line in lines if "Energy,force" in line))
            traj_es = list((round(float(line[:73].strip().split()[1]),6) for line in lines if "Energy,force" in line))

        traj_strs = []
        for i, q in enumerate(all_qs):
            # FIXME: using fuzzy match to align data may have problem in some corner cases
            if len(traj_qs) > 0 and np.isclose(q, traj_qs[0], rtol=0, atol=0.0001):
                all_strs[i].info['ssw_energy'] = traj_es[len(traj_strs)]  # type: ignore
                traj_strs.append(all_strs[i])
                traj_qs.pop(0)
        # write trajectory to file
        ase.io.write(os.path.join(task_dir, file_name), traj_strs, format='extxyz')
        # edit model_devi.out
        model_devi_file = os.path.join(task_dir, MODEL_DEVI_OUT)
        lines = []
        with open(model_devi_file, "r") as f:
            for i, line in enumerate(f):
                if i > 0:
                    # replace step 0 with step i so that it can be aligned with structures
                    line = re.sub(r'^\s+\d+', f'{i-1:>12} ', line)   #
                lines.append(line)
        with open(model_devi_file, "w") as f:
            f.writelines(lines)

    def process_lasp_outputs(task_dirs: List[str], workers: int = 4):
        import joblib
        joblib.Parallel(n_jobs=workers)(
            joblib.delayed(process_lasp_output)(task_dir)
            for task_dir in task_dirs
        )

    return (
        make_lasp_task_dirs,
        process_lasp_outputs,
    )

(
    make_lasp_task_dirs,
    process_lasp_outputs,
) = __export_remote_functions()


from django.contrib import messages
from django.contrib.auth.decorators import login_required

from django.shortcuts import render

# Create your views here.

from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render

# Create your views here.
from product.models import Comment, CommentForm

def index(request):

    return HttpResponse("Product Page")

@login_required(login_url='/login')
def add(request, id):
    url = request.META.get('HTTP_REFERER')
    if request.method == 'POST':
        form = CommentForm(request.POST)
        if form.is_valid():
            current_user = request.user

            data = Comment()
            data.user_id = current_user.id
            data.product_id = id
            data.subject = request.user.username
            data.comment = form.cleaned_data['comment']
            #data.rate = form.cleaned_data['rate']
            data.ip = request.META.get('REMOTE_ADDR')
            data.save()
            messages.success(request, "Yorumunuz Başarılı bir şekilde eklendi")
            #url = request.META.get('HTTP_REFERER')
            return HttpResponseRedirect(url)
        else:
            messages.warning(request, "Yorumunuz Eklenemedi :( ")
    return HttpResponseRedirect(url)


import logging

logging.basicConfig(level="INFO")
logger = logging.getLogger(__name__)

from utils.shadowgen import generator
HETEROGENEITY = [1.0]  # Homogeneous
NUM_MACHINES = 400
SPEC_RANGE = [(50,100)]
MAGNITUDE = 'giga'

SYSTEM_OUTPUT_PATH = 'recipes/routput/basic_spec-{0}.json'.format(
    NUM_MACHINES
)
system_config_path = generator.generate_system_machines(
    SYSTEM_OUTPUT_PATH, NUM_MACHINES, MAGNITUDE, HETEROGENEITY, SPEC_RANGE
)
logger.info(system_config_path)


"""
======================
Tests for Ecoli Master
======================
"""

import os
import numpy as np
import pytest
import warnings

from vivarium.core.engine import Engine
from vivarium.core.control import run_library_cli

from ecoli.plots.snapshots import plot_snapshots, format_snapshot_data
from ecoli.plots.snapshots_video import make_video
from ecoli.composites.ecoli_configs import ECOLI_DEFAULT_PROCESSES, ECOLI_DEFAULT_TOPOLOGY
from ecoli.experiments.ecoli_master_sim import EcoliSim, CONFIG_DIR_PATH

@pytest.mark.slow
def test_division(
        agent_id='0',
        total_time=4
):
    """tests that a cell can be divided and keep running"""

    # get initial mass from Ecoli composer
    sim = EcoliSim.from_file()
    sim.config['initial_state_file'] = 'vivecoli_t2000'
    sim.config['divide'] = True
    sim.config['division_variable'] = ('listeners', 'mass', 'dry_mass')
    sim.config['agent_id'] = agent_id
    sim.config['total_time'] = total_time
    # Ensure unique molecules are emitted
    sim.config['emit_unique'] = True
    sim.build_ecoli()

    # Set threshold so that mother divides after first timestep
    sim.generated_initial_state['agents']['0']['division_threshold'] = 572.2
    sim.run()

    # retrieve output
    output = sim.ecoli_experiment.emitter.get_data()
    # metabolism runs and greatly changes molecule counts after
    # previoius emit but before division, so ignore metabolites
    metabolite_idx = sim.ecoli_experiment.state['agents']['00'][
        'ecoli-metabolism'].value.metabolite_idx
    # ignore trna charging molecules, which also change before division
    polypep_elong = sim.ecoli_experiment.state['agents']['00']['process'][
        'ecoli-polypeptide-elongation'].value[0]
    trna_charging_idx = polypep_elong.charging_molecule_idx
    ppgpp_idx = np.array([polypep_elong.ppgpp_idx])
    # ignore inactive ribosomes, which also change before division
    polypep_init = sim.ecoli_experiment.state['agents']['00']['process'][
        'ecoli-polypeptide-initiation'].value[0]
    ribosome_idx = np.array([polypep_init.ribosome30S_idx,
                             polypep_init.ribosome50S_idx])
    ignore_idx = np.concatenate([metabolite_idx, trna_charging_idx,
                                 ppgpp_idx, ribosome_idx])
    mother_state = next(iter(output[1]['agents'].values()))
    mother_bulk = np.delete(mother_state['bulk'], ignore_idx)
    daughter_states = list(output[2]['agents'].values())
    daughter_bulk = [np.delete(ds['bulk'], ignore_idx)
        for ds in daughter_states]

    # compare the counts of bulk molecules between the mother and daughters
    # this is not exact because the mother grew slightly in the timestep
    # after its last emit but before being split into two daughter cells
    assert np.allclose(mother_bulk, np.array(daughter_bulk[0])
        + np.array(daughter_bulk[1]), atol=50)

    # compare the counts of unique molecules between the mother and daughters
    for name, mols in mother_state['unique'].items():
        d1_state = daughter_states[0]['unique'][name]
        d2_state = daughter_states[1]['unique'][name]
        m_state = np.array(mols)
        entryState_col = np.where(np.array(
            d1_state.dtype.names) == '_entryState')[0]
        n_mother = m_state[entryState_col].sum()
        n_daughter = d1_state['_entryState'].sum() + \
            d2_state['_entryState'].sum()
        if name == 'chromosome_domain':
            # Chromosome domain 0 is lost after division because
            # it has been fully split into child domains 1 and 2
            n_daughter += 1
        assert np.isclose(n_mother, n_daughter, rtol=0.01), \
            f'{name}: mother has {n_mother}, daughters have {n_daughter}'
        # Assert that no unique mol is in both daughters
        assert not (set(d1_state['unique_index']) &
            set(d2_state['unique_index']))

    # asserts
    final_agents = output[total_time]['agents'].keys()
    print(f"initial agent id: {agent_id}")
    print(f"final agent ids: {final_agents}")
    assert len(final_agents) == 2

def test_division_topology():
    """test that the topology is correctly dividing"""
    timestep = 2
    agent_id = '0'
    # get initial mass from Ecoli composer
    sim = EcoliSim.from_file()
    sim.config['seed'] = 1
    sim.config['initial_state_file'] = 'vivecoli_t2000'
    sim.config['divide'] = True
    sim.config['agent_id'] = agent_id
    sim.build_ecoli()

    # make the experiment
    experiment_config = {
        'processes': sim.ecoli.processes,
        'steps': sim.ecoli.steps,
        'flow': sim.ecoli.flow,
        'topology': sim.ecoli.topology,
        'initial_state': sim.generated_initial_state,
    }

    # Since unique numpy updater is an class method, internal
    # deepcopying in vivarium-core causes this warning to appear
    warnings.filterwarnings("ignore",
        message="Incompatible schema assignment at ")
    sim.ecoli_experiment = Engine(**experiment_config)

    # Only emit designated stores
    sim.ecoli_experiment.state.set_emit_values([tuple()], False)
    sim.ecoli_experiment.state.set_emit_values(
        sim.config['emit_paths'],
        True,
    )
    # Divide immediately
    sim.ecoli_experiment.state['agents']['0']['divide'] = True

    # Clean up unnecessary references
    sim.generated_initial_state = None
    sim.ecoli_experiment.initial_state = None

    full_topology = sim.ecoli_experiment.state.get_topology()
    mother_topology = full_topology['agents'][agent_id].copy()

    # update one time step at a time until division
    while len(full_topology['agents']) <= 1:
        sim.ecoli_experiment.update(timestep)
        full_topology = sim.ecoli_experiment.state.get_topology()
    sim.ecoli_experiment.end()

    # assert that the daughter topologies are the same as the mother topology
    daughter_ids = list(full_topology['agents'].keys())
    for daughter_id in daughter_ids:
        daughter_topology = full_topology['agents'][daughter_id]
        assert daughter_topology == mother_topology

def test_ecoli_generate():
    sim = EcoliSim.from_file()
    sim.divide = False
    sim.build_ecoli()

    # asserts to ecoli_composite['processes'] and ecoli_composite['topology']
    assert all('_requester' in k or
               '_evolver' in k or
               'allocator' in k or
               'unique_update' in k or
               k == 'division' or
               k == 'mark_d_period' or
               isinstance(v, ECOLI_DEFAULT_PROCESSES[k])
               for k, v in sim.ecoli['steps'].items())
    ignore_keys = ['request', 'process', 'allocate', 'first_update', 'global_time']
    for k, v in sim.ecoli['topology'].items():
        proc_name = k.split('_requester')[0].split('_evolver')[0]
        # Clock topology is not registered in registry
        if proc_name == 'clock':
            pass
        elif proc_name in ECOLI_DEFAULT_TOPOLOGY:
            if '_requester' in k or '_evolver' in k:
                # Ignore partitioning and first_update keys
                for ignore_key in ignore_keys:
                    v.pop(ignore_key, None)
            assert ECOLI_DEFAULT_TOPOLOGY[proc_name] == v

def test_lattice_lysis(plot=False):
    """
    Run plots:
    '''
    > python ecoli/composites/ecoli_master_tests.py -n 4 -o plot=True
    '''

    ANTIBIOTIC_KEY = 'nitrocefin'
    PUMP_KEY = 'TRANS-CPLX-201[s]'
    PORIN_KEY = 'porin'
    BETA_LACTAMASE_KEY = 'EG10040-MONOMER[p]'

    TODO: connect glucose! through local_field
    """
    sim = EcoliSim.from_file(CONFIG_DIR_PATH + 'lysis.json')
    sim.total_time = 10
    sim.process_configs.update({'global_clock': {'time_step': 2}})
    sim.build_ecoli()
    # Add beta-lactam to bulk store
    initial_state = sim.generated_initial_state
    zero_mass = (0,) * 9
    initial_state['agents']['0']['bulk'] = np.append(
        initial_state['agents']['0']['bulk'], np.array(
            [('beta-lactam[p]', 0) + zero_mass,
             ('hydrolyzed-beta-lactam[p]', 0) + zero_mass],
            dtype=initial_state['agents']['0']['bulk'].dtype))
    sim.run()
    data = sim.query()

    if plot:
        plot_spatial_snapshots(data, sim, experiment_dir='ecoli_lysis')

def plot_spatial_snapshots(data, sim, experiment_dir='ecoli_test'):
    out_dir = os.path.join('out', 'experiments', experiment_dir)
    os.makedirs(out_dir, exist_ok=True)

    bounds = sim.config['spatial_environment_config']['multibody']['bounds']

    # format the data for plot_snapshots
    agents, fields = format_snapshot_data(data)

    plot_snapshots(
        bounds,
        agents=agents,
        fields=fields,
        n_snapshots=5,
        out_dir=out_dir,
        filename=f"snapshots")

    # make snapshot video
    make_video(
        data,
        bounds,
        plot_type='fields',
        out_dir=out_dir,
        filename='video',
    )

test_library = {
    '1': test_division,
    '2': test_division_topology,
    '3': test_ecoli_generate,
    '4': test_lattice_lysis,
}

# run experiments in test_library from the command line with:
# python ecoli/composites/ecoli_master_tests.py -n [experiment id]
if __name__ == '__main__':
    run_library_cli(test_library)


from __future__ import print_function
from time import time

from fenics import *
from mshr import *
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.pyplot as plt
import numpy as np

channel = Rectangle(Point(0, 0), Point(2.2, .41))
cylinder1 = Circle(Point(.2, .2), .05)
cylinder2 = Circle(Point(1.2, .2), .05)
domain = channel - cylinder1 - cylinder2
mesh_sizes = [4, 8, 16, 32, 64, 128, 256]

times = []
iter_counts = []

for mesh_size in mesh_sizes:
    mesh = generate_mesh(domain, mesh_size)

    inflow = 'near(x[0], 0)'
    outflow = 'near(x[0], 2.2)'
    walls = 'near(x[1], 0) || near(x[1], .41)'
    cylinder1 = 'on_boundary && x[0] > .1 && x[0] < .3 && x[1] > .1 && x[1] < .3'
    cylinder2 = 'on_boundary && x[0] > 1.1 && x[0] < 1.3 && x[1] > .1 && x[1] < .3'
    inflow_profile = '-4. * 1.5 * x[1] * (.41 - x[1]) / .41 / .41', '0'

    V1 = VectorElement('Lagrange', mesh.ufl_cell(), 2)
    Q1 = FiniteElement('Lagrange', mesh.ufl_cell(), 1)
    W = FunctionSpace(mesh, V1 * Q1)

    bcu_inflow = DirichletBC(W.sub(0), Expression(inflow_profile, degree=2), inflow)
    bcu_walls = DirichletBC(W.sub(0), Constant((0, 0)), walls)
    bcp_outflow = DirichletBC(W.sub(1), Constant(0), outflow)
    bcu_cylinder1 = DirichletBC(W.sub(0), Constant((0, 0)), cylinder1)
    bcu_cylinder2 = DirichletBC(W.sub(0), Constant((0, 0)), cylinder2)
    bcs = [bcu_inflow, bcu_walls, bcu_cylinder1, bcu_cylinder2, bcp_outflow]

    u, p = TrialFunctions(W)
    v, q = TestFunctions(W)
    f = Constant((0., 0.))

    a = inner(grad(u), grad(v)) * dx + div(v) * p * dx + q * div(u) * dx
    L = inner(f, v) * dx
    b = inner(grad(u), grad(v)) * dx + p * q * dx

    A, bb = assemble_system(a, L, bcs)
    P, btmp = assemble_system(b, L, bcs)

    solver = KrylovSolver('minres', 'amg')
    solver.set_operators(A, P)

    U = Function(W)

    start = time()
    num_iter = solver.solve(U.vector(), bb)
    times.append(time() - start)
    iter_counts.append(num_iter)

    print(f'grid size: {mesh_size}, time: {times[-1]}, iterations: {iter_counts[-1]}\n')

    u, p = U.split()

    p1 = plot(u)
    plt.colorbar(p1)
    plt.savefig('plot_u.png')
    plt.close()

    p2 = plot(p)
    plt.colorbar(p2)
    plt.savefig('plot_p.png')
    plt.close()


import tkinter as tk
from tkinter import *
from tkinter import ttk
from tkinter.ttk import *
from googletrans import LANGUAGES
from translate import Translator

root = tk.Tk()

src_text = Text(root, font="arial 10", height=11, wrap=WORD, padx=5, pady=5, width=60)
src_text.place(x=50, y=120)

des_text = Text(root, font="arial 10", height=11, wrap=WORD, padx=5, pady=5, width=60)
des_text.place(x=600, y=120)

class Lang_Translator(Frame):
	def __init__(self, master=None):
		super().__init__(master)
		self.pack()
		self.label()

	src_text = Text(root)
	des_text = Text(root)

	def label(self):
		Label(root,text="Language Translator", font="italic 20").pack(side="top")
		Label(root,text="Select Language", font="arial 15").place(x=30, y=60)
		Label(root,text="Select Language", font="arial 15").place(x=600, y=60)

lang = list(LANGUAGES.values())
src_lang = ttk.Combobox(root, values=lang, width=30)
src_lang.place(x=200, y=63)
src_lang.set("english")

des_lang = ttk.Combobox(root, values=lang, width=30)
des_lang.place(x=770, y=63)
des_lang.set("marathi")

def trans():
	translator2 = Translator(to_lang = des_lang.get())
	translated = translator2.translate(src_text.get(0.0, END))
	des_text.insert(0.0, translated)

def clear_all() :
    des_text.delete(0.0, END)
    src_text.delete(0.0, END)

button_1 = Button(root, text="Translate" , command=trans).place(x=510, y=160)
button_2 = Button(root, text="Clear", command=clear_all).place(x=510, y=220)

app = Lang_Translator(master=root)
app.master.title('Translator')
app.master.geometry('1080x350')
app.mainloop()


"""Robust metrics to evaluate performance of copy number estimates.
"""
from __future__ import absolute_import, division, print_function
from builtins import zip

import logging

import numpy as np
import pandas as pd
from scipy import stats

from .descriptives import (biweight_midvariance, interquartile_range,
                           median_absolute_deviation)

def do_metrics(cnarrs, segments=None, skip_low=False):
    """Compute coverage deviations and other metrics for self-evaluation."""
    # Catch if passed args are single CopyNumArrays instead of lists
    from .cnary import CopyNumArray as CNA
    if isinstance(cnarrs, CNA):
        cnarrs = [cnarrs]
    if isinstance(segments, CNA):
        segments = [segments]
    elif segments is None:
        segments = [None]
    else:
        segments = list(segments)
    if skip_low:
        cnarrs = (cna.drop_low_coverage() for cna in cnarrs)
    rows = ((cna.meta.get("filename", cna.sample_id),
             len(seg) if seg is not None else '-'
            ) + ests_of_scale(cna.residuals(seg))
            for cna, seg in zip_repeater(cnarrs, segments))
    colnames = ["sample", "segments", "stdev", "mad", "iqr", "bivar"]
    return pd.DataFrame.from_records(rows, columns=colnames)

def zip_repeater(iterable, repeatable):
    """Repeat a single segmentation to match the number of copy ratio inputs"""
    rpt_len = len(repeatable)
    if rpt_len == 1:
        rpt = repeatable[0]
        for it in iterable:
            yield it, rpt
    else:
        i = -1
        for i, (it, rpt) in enumerate(zip(iterable, repeatable)):
            yield it, rpt
        # Require lengths to match
        if i + 1 != rpt_len:
            raise ValueError("""Number of unsegmented and segmented input files
                             did not match (%d vs. %d)""" % (i, rpt_len))

def ests_of_scale(deviations):
    """Estimators of scale: standard deviation, MAD, biweight midvariance.

    Calculates all of these values for an array of deviations and returns them
    as a tuple.
    """
    std = np.std(deviations, dtype=np.float64)
    mad = median_absolute_deviation(deviations)
    iqr = interquartile_range(deviations)
    biw = biweight_midvariance(deviations)
    return (std, mad, iqr, biw)

def segment_mean(cnarr, skip_low=False):
    """Weighted average of bin log2 values."""
    if skip_low:
        cnarr = cnarr.drop_low_coverage()
    if len(cnarr) == 0:
        return np.nan
    if 'weight' in cnarr:
        return np.average(cnarr['log2'], weights=cnarr['weight'])
    return cnarr['log2'].mean()

# Intervals

def confidence_interval_bootstrap(bins, alpha, bootstraps=100, smoothed=True):
    """Confidence interval for segment mean log2 value, estimated by bootstrap."""
    if not 0 < alpha < 1:
        raise ValueError("alpha must be between 0 and 1; got %s" % alpha)
    if bootstraps <= 2 / alpha:
        new_boots = int(np.ceil(2 / alpha))
        logging.warn("%d bootstraps not enough to estimate CI alpha level %f; "
                     "increasing to %d", bootstraps, alpha, new_boots)
        bootstraps = new_boots
    # Bootstrap for CI
    k = len(bins)
    if k < 2:
        return np.array([bins["log2"][0], bins["log2"][0]])

    np.random.seed(0xA5EED)
    rand_indices = np.random.randint(0, k, (bootstraps, k))
    samples = [bins.data.take(idx) for idx in rand_indices]
    if smoothed:
        # samples = _smooth_samples(bins, samples, alpha)
        pass
    # Recalculate segment means
    bootstrap_dist = np.array([segment_mean(samp) for samp in samples])
    alphas = np.array([alpha / 2, 1 - alpha / 2])
    if not smoothed:
        # alphas = _bca_correct_alpha(bins, bootstrap_dist, alphas)
        pass
    ci = np.percentile(bootstrap_dist, list(100 * alphas))
    return ci

def _smooth_samples(bins, samples, alpha):
    k = len(bins)
    # Essentially, resample from a kernel density estimate of the data
    # instead of the original data.
    # Estimate KDE bandwidth (Polansky 1995)
    resids = bins['log2'] - bins['log2'].mean()
    s_hat = 1/k * (resids**2).sum()  # sigma^2 = E[X-theta]^2
    y_hat = 1/k * abs((resids**3).sum())  # gamma = E[X-theta]^3
    z = stats.norm.ppf(alpha / 2)  # or alpha?
    bw = k**(-1/4) * np.sqrt(y_hat*(z**2 + 2) / (3*s_hat*z))
    # NB: or, Silverman's Rule for KDE bandwidth (roughly):
    # std = interquartile_range(bins['log2']) / 1.34
    # bw = std * (k*3/4) ** (-1/5)
    if bw > 0:
        samples = [samp.assign(log2=lambda x:
                                x['log2'] + bw * np.random.randn(k))
                    for samp in samples]
        logging.debug("Smoothing worked for this segment (bw=%s)", bw)
    else:
        logging.debug("Smoothing not needed for this segment (bw=%s)", bw)
    return samples

def _bca_correct_alpha(bins, bootstrap_dist, alphas):
    # BCa correction (Efron 1987, "Better Bootstrap Confidence Intervals")
    # http://www.tandfonline.com/doi/abs/10.1080/01621459.1987.10478410
    # Ported from R package "bootstrap" function "bcanon"
    n_boots = len(bootstrap_dist)
    orig_mean = segment_mean(bins)
    logging.warn("boot samples less: %s / %s",
                 (bootstrap_dist < orig_mean).sum(),
                 n_boots)
    n_boots_below = (bootstrap_dist < orig_mean).sum()
    if n_boots_below == 0:
        logging.warn("boots mean %s, orig mean %s",
                     bootstrap_dist.mean(), orig_mean)
    else:
        logging.warn("boot samples less: %s / %s",
                     n_boots_below, n_boots)
    z0 = stats.norm.ppf((bootstrap_dist < orig_mean).sum() / n_boots)
    zalpha = stats.norm.ppf(alphas)
    # Jackknife influence values
    u = np.array([segment_mean(bins.concat([bins[:i], bins[i+1:]]))
                    for i in range(len(bins))])
    uu = u.mean() - u
    acc = (u**3).sum() / (6 * (uu**2).sum()**1.5)
    alphas = stats.norm.cdf(z0 + (z0 + zalpha)
                                    / (1 - acc * (z0 + zalpha)))
    logging.warn("New alphas: %s -- via z0=%s, za=%s, acc=%s",
                    alphas, z0, zalpha, acc)
    if not 0 < alphas[0] < 1 and 0 < alphas[1] < 1:
        raise ValueError("CI alphas should be in (0,1); got %s" % alphas)
    return alphas

def prediction_interval(bins, alpha):
    """Prediction interval, estimated by percentiles."""
    pct_lo = 100 * alpha / 2
    pct_hi = 100 * (1 - alpha / 2)
    # ENH: weighted percentile
    return np.percentile(bins['log2'], [pct_lo, pct_hi])


import json

import click
import requests

files_to_download = [
    "building_properties.json",
    "district_system_properties.json",
    "electrical_connector_properties.json",
    "electrical_junction_properties.json",
    "region_properties.json",
    "site_properties.json",
    "thermal_connector_properties.json",
    "thermal_junction_properties.json",
]

baseurl = "https://raw.githubusercontent.com/urbanopt/urbanopt-geojson-gem/develop/lib/urbanopt/geojson/schema/"  # noqa

@click.command()
@click.argument('schema', required=False)
def update_schemas(schema):
    for f in files_to_download:
        click.echo(f"Downloading schema: {f}")
        response = requests.get(f"{baseurl}/{f}")
        save_path = f"geojson_modelica_translator/geojson/data/schemas/{f}"
        with open(save_path, "w") as outf:
            json.dump(response.json(), outf, indent=2)

        print("Note that the [unused] fields will have been overwritten by this operation. It is recommended to "
              "open the previous version and new version in a diff tool to copy over the [unused] tags.")


import cStringIO
import json
import logging
import optparse
import os
import shutil
import subprocess
import urllib
import urllib2
import zipfile

_LOGGER = logging.getLogger(os.path.basename(__file__))

_SRC_DIR = os.path.abspath(
    os.path.join(os.path.dirname(__file__), '../..'))
_VERSION_FILE = os.path.join(_SRC_DIR, 'syzygy/VERSION')
_BINARIES_DIR = os.path.join(_SRC_DIR, 'syzygy/binaries')
_EXE_DIR = os.path.join(_BINARIES_DIR, 'exe')

_SYZYGY_WATERFALL_URL = 'http://build.chromium.org/p/client.syzygy'
_SYZYGY_OFFICIAL = 'Syzygy Official'

_SYZYGY_ARCHIVE_URL = ('http://syzygy-archive.commondatastorage.googleapis.com/'
    'builds/official/%(revision)d')

def _Shell(*cmd, **kw):
  """Runs cmd, returns the results from Popen(cmd).communicate()."""
  _LOGGER.info('Executing %s.', cmd)
  prog = subprocess.Popen(cmd, shell=True, **kw)

  stdout, stderr = prog.communicate()
  if prog.returncode != 0:
    raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode))
  return (stdout, stderr)

def _Download(url):
  """Downloads the given URL and returns the contents as a string."""
  response = urllib2.urlopen(url)
  if response.code != 200:
    raise RuntimeError('Failed to download "%s".' % url)
  return response.read()

def _QueryWaterfall(path):
  """Queries the JSON API of the Syzygy waterfall."""
  url = _SYZYGY_WATERFALL_URL + '/json' + path
  data = _Download(url)
  return json.loads(data)

def _GetLastOfficialBuildRevision():
  """Query the Syzygy waterfall to get the SVN revision associated with the
  last successful official build.
  """
  # First make sure the builder doesn't have any pending builds and is idle.
  builders = _QueryWaterfall('/builders')
  if builders[_SYZYGY_OFFICIAL]['pendingBuilds'] > 0:
    raise RuntimeError('There are pending official builds.')
  if builders[_SYZYGY_OFFICIAL]['state'] != 'idle':
    raise RuntimeError('An official build is in progress.')

  # Get the information from the last build and make sure it passed before
  # extracting the revision number.
  build = _QueryWaterfall('/builders/%s/builds/-1' %
      urllib.quote(_SYZYGY_OFFICIAL))
  if 'successful' not in build['text']:
    raise RuntimeError('Last official build failed.')
  return int(build['sourceStamp']['revision'])

def main():
  option_parser = optparse.OptionParser()
  option_parser.add_option(
      '--revision', type="int",
      help='The SVN revision associated with the release build. '
           'If omitted, the SVN revision of the last successful official '
           'build will be used.')
  options, args = option_parser.parse_args()
  if args:
    option_parser.error('Unexpected arguments: %s' % args)

  # Enable info logging.
  logging.basicConfig(level=logging.INFO)

  # Get the SVN revision associated with the archived binaries to use.
  if options.revision is not None:
    revision = options.revision
  else:
    revision = _GetLastOfficialBuildRevision()
    _LOGGER.info('Using official build at revision %d.' % revision)

  # And build the corresponding archive URL.
  archive_url = _SYZYGY_ARCHIVE_URL % { 'revision': revision }
  benchmark_url = archive_url + '/benchmark.zip'
  binaries_url = archive_url + '/binaries.zip'

  # Download the archives.
  _LOGGER.info('Retrieving benchmark archive at "%s".', benchmark_url)
  benchmark_data = _Download(benchmark_url)
  _LOGGER.info('Retrieving binaries archive at "%s".', binaries_url)
  binaries_data = _Download(binaries_url)

  # Create a new feature branch off the master branch for the release
  # before we start changing any files.
  _LOGGER.info('Creating a release-binaries feature branch.')
  _Shell('git', 'checkout', '-b', 'release-binaries', 'master')

  # Clean out the output directories.
  shutil.rmtree(_BINARIES_DIR, True)
  os.makedirs(_BINARIES_DIR)
  os.makedirs(_EXE_DIR)

  # Extract the contents of the benchmark archive to the binaries directory.
  _LOGGER.info('Unzipping benchmark archive.')
  archive = zipfile.ZipFile(cStringIO.StringIO(benchmark_data))
  archive.extractall(_BINARIES_DIR)

  # Extract the binaries archives to the exe directory.
  _LOGGER.info('Unzipping binaries archive.')
  archive = zipfile.ZipFile(cStringIO.StringIO(binaries_data))
  archive.extractall(_EXE_DIR)

  # Add all the new files to the repo.
  _LOGGER.info('Committing release files.')
  # Update any changed or deleted files.
  _Shell('git', 'add', '-u', _BINARIES_DIR)
  # Add new files.
  _Shell('git', 'add', _BINARIES_DIR)

  # Now commit and upload the new binaries.
  message = 'Checking in version %d release binaries.' % revision
  _Shell('git', 'commit', '-m', message)
  _Shell('git', 'cl', 'upload', '-t', message)

if __name__ == '__main__':
  main()


import streamlit as st
from database import add_data

cities = ["Bengaluru", "Chennai", "Mangaluru"]
availability_choice = ["Yes", "No"]

def create_columns(train_no='', train_name='', train_type='', source=[], destination=[], availability=availability_choice, disabled=False):
    c1, c2, c3 = st.columns(3)
    with c1:
        train_no = st.text_input("Train No:", train_no, disabled)
        source = st.selectbox("Source:", source)
    with c2:
        train_name = st.text_input("Train Name:", train_name)
        destination = st.selectbox("Destination:", destination)
    with c3:
        train_type = st.text_input("Train Type:", train_type)
        availability = st.selectbox("Avaiability:", availability)

    return [train_no, train_name, train_type, source, destination, availability]

def create():
    res = create_columns(source=cities, destination=cities, availability=availability_choice)

    train_no = res[0]
    train_name = res[1]
    train_type = res[2]
    source = res[3]
    destination = res[4]
    availability = res[5]

    if st.button("Add Train"):
        add_data(train_no, train_name, train_type, source, destination, availability)
        st.success(f"Successfully added train: {train_no} - {train_name}")


import logging
import time

from pathlib import Path

from django.core.management import BaseCommand
from concurrent.futures import Future, ThreadPoolExecutor

from zds.tutorialv2.models.database import PublicationEvent
from zds.tutorialv2.publication_utils import PublicatorRegistry

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    help = 'Launch a watchdog that generate all exported formats (epub, pdf...) files without blocking request handling'

    def handle(self, *args, **options):
        with ThreadPoolExecutor(1) as executor:
            try:
                while True:
                    Command.launch_publicators(executor)
                    time.sleep(10)
            except KeyboardInterrupt:
                executor.shutdown(wait=False)

    @staticmethod
    def get_callback_of(publication_event: PublicationEvent):
        def callback(future: Future):
            if future.cancelled() or future.exception():
                publication_event.state_of_processing = 'FAILURE'
                if future.exception():
                    logger.error('error while producing %s of %s', publication_event.format_requested,
                                 publication_event.published_object.title(), exc_info=future.exception())
            elif future.done():
                publication_event.state_of_processing = 'SUCCESS'

            publication_event.save()
        return callback

    @staticmethod
    def launch_publicators(executor):
        query_set = PublicationEvent.objects.select_related('published_object', 'published_object__content',
                                                            'published_object__content__image') \
                                            .filter(state_of_processing='REQUESTED')

        for publication_event in query_set.iterator():
            logger.info('Export %s -- format=%s', publication_event.published_object.title(),
                        publication_event.format_requested)
            content = publication_event.published_object
            publicator = PublicatorRegistry.get(publication_event.format_requested)

            extra_content_dir = content.get_extra_contents_directory()
            building_extra_content_path = Path(str(Path(extra_content_dir).parent) + '__building',
                                               'extra_contents', content.content_public_slug)
            if not building_extra_content_path.exists():
                building_extra_content_path.mkdir(parents=True)
            base_name = str(building_extra_content_path)
            md_file_path = base_name + '.md'

            publication_event.state_of_processing = 'RUNNING'
            publication_event.save()
            future = executor.submit(publicator.publish, md_file_path, base_name)
            future.add_done_callback(Command.get_callback_of(publication_event))


###   Nome do programa: prep_dia_FORTUNA01
###
###   O seguinte programa le os arquivos da interface FORTUNA01, de um dia especifico, realiza validacoes dos campos e grava os registros na tabela final
###
###   Tambem verifica a duplicidade dos registros, tanto dentro do proprio arquivo quanto em relacao a tabela final
###
###   Os registros rejeitados na validacao serao gravados no diretorio de rejeicoes no HDFS
###
###   Os erros geram registros na tabela de log com a data de ocorrencia

from pyspark import SparkContext, SparkConf
from pyspark.sql import SQLContext, HiveContext, Row
from pyspark.sql.types import *
from pyspark import SparkFiles
import sys

conf = SparkConf()

#conf.setMaster("yarn-client")

conf.setAppName("Valida FORTUNA01")

sc = SparkContext(conf = conf)
sqlContext = HiveContext(sc)

# Seta parametros

data_processa = sys.argv[2]+"000000"
conf_path = sys.argv[1]
data_arquivo = sys.argv[4]

# Adiciona arquivos de configuracao

sc.addPyFile(conf_path+"/conf_prep_dia_FORTUNA01.py")
sc.addPyFile(conf_path+"/conf_geral.py")
sc.addPyFile(conf_path+"/valida_lib.py")

from conf_prep_dia_FORTUNA01 import *

for chave in sqlContext_conf_list:
	sqlContext.setConf(chave, sqlContext_conf_list[chave])

arquivo = arquivo.replace("dataArquivo",data_arquivo)

dt_execucao = str(datetime.now()).split(".")[0]
dt_execucao_formatada = dt_execucao.replace("-","").replace(":","").replace(" ","").split(".")[0]

##Registra na tabela de log e grava no diretorio de rejeicao erros encontrados

def registra_erros(rej_rdd, nome_campo, tipo_erro):
	n_rej = rej_rdd.count()
	if n_rej > 0:
		if tipo_erro == "REGISTRO_ANTIGO":
			mensagem = str(n_rej) + " Registros Antigos"
		elif tipo_erro == "REGISTRO_FORA_TEMPO_GUARDA":
			mensagem = str(n_rej) + " Registros fora do tempo de guarda"
		else:
			mensagem = str(n_rej) + " Registros com o campo "+nome_campo+" invalido"
		grava_log("ERRO",cod_programa + cod_erro[tipo_erro],mensagem) #tp_ocorrencia, cod_ocorrencia, mensagem
		grava_rej(rej_rdd, dir_rejeitado+arquivo+"."+dt_execucao_formatada+"."+cod_programa + cod_erro[tipo_erro])

##Realiza a gravacao de um registro na tabela de log

def grava_log(tp_ocorrencia,cod_ocorrencia,mensagem):
	dh_ocorrencia = dt_execucao
	nome_arquivo = arquivo
	dt = str(dh_ocorrencia)[:10]
	dup_rdd = sc.parallelize([[dh_ocorrencia, tp_ocorrencia, cod_ocorrencia, modulo, mensagem, nome_arquivo, cod_programa, dt]])
	dup_df = sqlContext.createDataFrame(dup_rdd).withColumnRenamed("_7","cod_programa").withColumnRenamed("_8","dt")
	dup_df.write.mode("append").partitionBy("cod_programa","dt").saveAsTable(db_prepago+".TB_LOG", format="parquet")

fieldsLog = [StructField(field_name, StringType(), True) for field_name in schemaLog.split(';')]

fieldsLog[0].dataType = TimestampType()

schemaLog = StructType(fieldsLog)

#config.read(SparkFiles.get(conf_path))

dir_arquivo = dir_processamento+arquivo

# Le arquivo do HDFS

fortuna01 = sc.textFile(dir_arquivo)

##Define a estrutura e dos tipos dos campos da tabela final

schemaString = "DH_EXTRACAO;MSISDN;VALOR_FACE;DT_SOLICITACAO_RECARGA;NSU_REQUISICAO;ID_RECARGA;DT_EFETIVACAO_RECARGA;PRODUTO_RECARGA;CANAL_RECARGA;SUB_CANAL;SALDO;CD_PRODUTO;CD_CAMPANHA;CD_TIPO_PRODUTO;TIPO_RECARGA;CD_PONTO_VENDA;COD_DISTRIBUIDOR_SAP;ORGANIZACAO_VENDA;SETOR_ATIVIDADE;CANAL_DISTRIBUICAO;NUM_FATURA;DATA_HORA_FATURA;CNPJ_EMISSOR;Flag_Aceite_Upsell;Flag_Upsell_habilitado;Valor_Original_Recarga;Meio_de_Pagamento;Meio_de_Acesso_pagador;NU_SERIE;NU_VOUCHER;CD_CELL_ID;CD_AREA_CELL_ID;ID_PLANO_FAT;NUMERO_PEDIDO;DATA_SOLIC_ORDEM_VENDA;DT_EXECUCAO;dt"

fields = [StructField(field_name, StringType(), True) for field_name in schemaString.split(';')]

fields[0].dataType = TimestampType()
fields[2].dataType = DecimalType(10,2)
fields[3].dataType = TimestampType()
fields[6].dataType = TimestampType()
fields[10].dataType = DecimalType(10,2)
fields[21].dataType = TimestampType()
#fields[32].dataType = DecimalType(10,2) antigo valor fatura
fields[34].dataType = TimestampType()
#fields[35].dataType = TimestampType()

schema = StructType(fields)

def for_map(p):
	try:
		return (valida_data(p[0]), p[1], valida_valor(p[2]), valida_data(p[3]), p[4], p[5], valida_data(p[6]), p[7], p[8], p[9], valida_valor(p[10]), p[11], p[12], p[13], p[14], p[15], p[16], p[17], p[18], p[19], p[20], valida_dt_fatura(p[21]), p[22], p[23], p[24], p[25], p[26], p[27], remove_zeros_serie(p[28]), p[29], p[30], p[31], "", "", valida_data("19600101000000"), dt_execucao, str(valida_data(p[6]))[:10])
	except IndexError:
		linha = ""
		for campo in p:
			linha += campo+";"
		return [linha] #precisa ser array por conta da funcao str_rej

##Mapeia os campos para o formato da tabela e inicia as validacoes

fortuna01_temp = fortuna01.map(lambda k: k.split(";")).map(for_map).cache()

fortuna01_rej_estrutura = fortuna01_temp.filter(lambda x: len(x) <= 1)

registra_erros(fortuna01_rej_estrutura," ","ESTRUTURA_INVALIDA")

fortuna01_ace = fortuna01_temp.filter(lambda x : len(x) > 1)

fortuna01_rej_dh_extracao = fortuna01_ace.filter(lambda x: type(x[0]) is unicode)

registra_erros(fortuna01_rej_dh_extracao,"DH_EXTRACAO","DH_EXTRACAO_INVALIDA")

fortuna01_ace = fortuna01_ace.filter(lambda x : type(x[0]) is not unicode)

fortuna01_rej_msisdn = fortuna01_ace.filter(lambda x : x[1] == "")

registra_erros(fortuna01_rej_msisdn,"MSISDN","MSISDN_INVALIDO")

fortuna01_ace = fortuna01_ace.filter(lambda x : x[1] != "")

fortuna01_rej_vl_face = fortuna01_ace.filter(lambda x: type(x[2]) is unicode)

registra_erros(fortuna01_rej_vl_face,"VALOR_FACE","VALOR_FACE_INVALIDO")

fortuna01_ace = fortuna01_ace.filter(lambda x : type(x[2]) is not unicode)

fortuna01_rej_id_recarga = fortuna01_ace.filter(lambda x : x[5] == "")

registra_erros(fortuna01_rej_id_recarga,"ID_RECARGA","ID_RECARGA_INVALIDO")

fortuna01_ace = fortuna01_ace.filter(lambda x : x[5] != "")

fortuna01_rej_dt_efetivacao_recarga = fortuna01_ace.filter(lambda x: type(x[6]) is unicode)

registra_erros(fortuna01_rej_dt_efetivacao_recarga,"DT_EFETIVACAO_RECARGA","DT_EFETIVACAO_RECARGA_INVALIDA")

fortuna01_ace = fortuna01_ace.filter(lambda x : type(x[6]) is not unicode)

fortuna01_rej_tempo_guarda = fortuna01_ace.filter(lambda x: x[6] < valida_data(data_processa) - timedelta(days=int(tempo_guarda)))

registra_erros(fortuna01_rej_tempo_guarda,"DT_EFETIVACAO_RECARGA","REGISTRO_FORA_TEMPO_GUARDA")

fortuna01_ace = fortuna01_ace.filter(lambda x : x[6] >= valida_data(data_processa) - timedelta(days=int(tempo_guarda)))

fortuna01_rej_antigos = fortuna01_ace.filter(lambda x: x[6] < valida_data(data_producao))

registra_erros(fortuna01_rej_antigos,"DT_EFETIVACAO_RECARGA","REGISTRO_ANTIGO")

fortuna01_ace = fortuna01_ace.filter(lambda x : x[6] >= valida_data(data_producao))

fortuna01_rej_saldo = fortuna01_ace.filter(lambda x: type(x[10]) is unicode)

registra_erros(fortuna01_rej_saldo,"SALDO","SALDO_INVALIDO")

fortuna01_ace = fortuna01_ace.filter(lambda x : type(x[10]) is not unicode)

fortuna01_rej_serie = fortuna01_ace.filter(lambda x: x[28][:4] == "erro")

registra_erros(fortuna01_rej_serie,"NU_SERIE","SERIE_INVALIDA")

fortuna01_ace = fortuna01_ace.filter(lambda x : x[28][:4] != "erro")

fortuna01_rej_tipo_recarga = fortuna01_ace.filter(lambda x : x[14] not in ("FIS", "BAT", "ELT"))

registra_erros(fortuna01_rej_tipo_recarga,"TIPO_RECARGA","TIPO_RECARGA_INVALIDO")

fortuna01_ace = fortuna01_ace.filter(lambda x : x[14] in ("FIS", "BAT", "ELT"))

#fortuna01_rej_cod_distribuidor_sap = fortuna01_ace.filter(lambda x : x[16] == "")

#registra_erros(fortuna01_rej_cod_distribuidor_sap,"COD_DISTRIBUIDOR_SAP","106016")

#fortuna01_ace = fortuna01_ace.filter(lambda x : x[16] != "")

#fortuna01_rej_organizacao_venda = fortuna01_ace.filter(lambda x : x[17] == "")

#registra_erros(fortuna01_rej_organizacao_venda,"ORGANIZACAO_VENDA","105017")

#fortuna01_ace = fortuna01_ace.filter(lambda x : x[17] != "")

#fortuna01_rej_setor_atividade = fortuna01_ace.filter(lambda x : x[18] == "")

#registra_erros(fortuna01_rej_setor_atividade,"SETOR_ATIVIDADE","105018")

#fortuna01_ace = fortuna01_ace.filter(lambda x : x[18] != "")

#fortuna01_rej_canal_distribuicao = fortuna01_ace.filter(lambda x : x[19] == "")

#registra_erros(fortuna01_rej_canal_distribuicao,"CANAL_DISTRIBUICAO","105019")

#fortuna01_ace = fortuna01_ace.filter(lambda x : x[19] != "")

# Cria dataframe a partir de registros nao rejeitados

fortuna01_df = sqlContext.createDataFrame(fortuna01_ace, schema).cache()

##Agrupa registros por id_recarga e dt_efetivacao_recarga para verificar duplicidade

windowSpec = Window.partitionBy(fortuna01_df['ID_RECARGA'], fortuna01_df['DT_EFETIVACAO_RECARGA'])

fortuna01_window = fortuna01_df.select([func.rowNumber().over(windowSpec).alias("row_number")]+[name for name in schemaString.split(";")])

fortuna01_dup = fortuna01_window.filter("row_number <> 1").select([name for name in schemaString.split(";")])

fortuna01_df_uniq = fortuna01_window.filter("row_number = 1").select([name for name in schemaString.split(";")])

#fortuna01_old = sqlContext.sql("select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as teste from STG_PP_RECARGA_CV union select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as teste from STG_PP_RECARGA_FIS union select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as teste from STG_PP_RECARGA_REC union select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as teste from STG_PP_RECARGA_PS").withColumnRenamed("teste", "testeJoin")

# Le registros que ja estavam em tabela para verificar duplicidade

fortuna01_old_cv = sqlContext.sql("select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as testeJoin from "+db_stg+".STG_PP_RECARGA_CV")
fortuna01_old_fis = sqlContext.sql("select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as testeJoin from "+db_stg+".STG_PP_RECARGA_FIS")
fortuna01_old_ps = sqlContext.sql("select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as testeJoin from "+db_stg+".STG_PP_RECARGA_PS")
fortuna01_old_rec = sqlContext.sql("select ID_RECARGA, DT_EFETIVACAO_RECARGA, ID_RECARGA as testeJoin from "+db_stg+".STG_PP_RECARGA_REC")

fortuna01_old = fortuna01_old_cv.unionAll(fortuna01_old_fis).unionAll(fortuna01_old_ps).unionAll(fortuna01_old_rec)

#fortuna01_old = fortuna01_old.cache()
#fortuna01_old.count()

fortuna01_join = fortuna01_df_uniq.join(fortuna01_old, ["ID_RECARGA", "DT_EFETIVACAO_RECARGA"], "leftouter")

fortuna01_dup2 = fortuna01_join.filter("testejoin is not null")

schema_final = "DH_EXTRACAO,MSISDN,VALOR_FACE,DT_SOLICITACAO_RECARGA,NSU_REQUISICAO,ID_RECARGA,DT_EFETIVACAO_RECARGA,PRODUTO_RECARGA,CANAL_RECARGA,SUB_CANAL,SALDO,CD_PRODUTO,CD_CAMPANHA,CD_TIPO_PRODUTO,TIPO_RECARGA,CD_PONTO_VENDA,COD_DISTRIBUIDOR_SAP,ORGANIZACAO_VENDA,SETOR_ATIVIDADE,CANAL_DISTRIBUICAO,FLAG_ACEITE_UPSELL,FLAG_UPSELL_HABILITADO,Valor_Original_Recarga,Meio_de_Pagamento,Meio_de_Acesso_pagador,CD_CELL_ID,CD_AREA_CELL_ID,ID_PLANO_FAT,NUMERO_PEDIDO,DATA_SOLIC_ORDEM_VENDA,NUM_FATURA,DATA_HORA_FATURA,CNPJ_EMISSOR,NU_SERIE,NU_VOUCHER,DT_EXECUCAO,dt"

fortuna01_df_final = fortuna01_join.filter("testeJoin is null").select([name for name in schema_final.split(",")])

fortuna01_duplicados = fortuna01_dup.unionAll(fortuna01_dup2.select([name for name in schema_final.split(",")]))

n_dup = fortuna01_duplicados.count()
if n_dup > 0:
    mensagem = str(n_dup) + " registros duplicados rejeitados"
    grava_log("ERRO",cod_programa + cod_erro["REGISTRO_DUPLICADO"],mensagem) #tp_ocorrencia, cod_ocorrencia, mensagem
    grava_rej(fortuna01_duplicados, dir_rejeitado + arquivo +"."+ dt_execucao_formatada +"."+ cod_programa + cod_erro["REGISTRO_DUPLICADO"])

#fortuna01_df_final.registerTempTable("fortuna01_df_final")

#!#!#!#adiciona bolso principal. alterar com a chegada do novo bolso

produto_recarga = sqlContext.sql("select cd_produto_recarga as PRODUTO_RECARGA, cd_direito from {db}.tb_produto_recarga where cd_direito in {cds_direito}".format(db = db_prepago,cds_direito = str(cd_direito_franquia+cd_direito_principal).replace("[","(").replace("]",")").format(db = db_prepago))).dropDuplicates(['PRODUTO_RECARGA'])

#assumindo que todas as recargas sao do bolso principal ou de franquia

fortuna01_df_final_join = fortuna01_df_final.join(produto_recarga,["PRODUTO_RECARGA"],"left_outer")

fortuna01_df_final_join.registerTempTable("fortuna01_df_final_join")

fortuna01_df_final = sqlContext.sql("select DH_EXTRACAO,MSISDN,VALOR_FACE,DT_SOLICITACAO_RECARGA,NSU_REQUISICAO,ID_RECARGA,DT_EFETIVACAO_RECARGA,PRODUTO_RECARGA,CANAL_RECARGA,SUB_CANAL,SALDO,CD_PRODUTO,CD_CAMPANHA,CD_TIPO_PRODUTO,TIPO_RECARGA,CD_PONTO_VENDA,COD_DISTRIBUIDOR_SAP,ORGANIZACAO_VENDA,SETOR_ATIVIDADE,CANAL_DISTRIBUICAO,FLAG_ACEITE_UPSELL,FLAG_UPSELL_HABILITADO,Valor_Original_Recarga,Meio_de_Pagamento,Meio_de_Acesso_pagador,CD_CELL_ID,CD_AREA_CELL_ID,ID_PLANO_FAT,NUMERO_PEDIDO,DATA_SOLIC_ORDEM_VENDA,NUM_FATURA,DATA_HORA_FATURA,CNPJ_EMISSOR,NU_SERIE,NU_VOUCHER,if(CD_DIREITO IN {cd_direito_franquia},2,1) AS ID_BOLSO,DT_EXECUCAO,dt from fortuna01_df_final_join".format(cd_direito_franquia = str(cd_direito_franquia).replace("[","(").replace("]",")")))

fortuna01_recarga = sqlContext.sql("select DH_EXTRACAO,MSISDN,VALOR_FACE,DT_SOLICITACAO_RECARGA,NSU_REQUISICAO,ID_RECARGA,DT_EFETIVACAO_RECARGA,PRODUTO_RECARGA,CANAL_RECARGA,SUB_CANAL,SALDO,CD_PRODUTO,CD_CAMPANHA,CD_TIPO_PRODUTO,TIPO_RECARGA,CD_PONTO_VENDA,COD_DISTRIBUIDOR_SAP,ORGANIZACAO_VENDA,SETOR_ATIVIDADE,CANAL_DISTRIBUICAO,FLAG_ACEITE_UPSELL,FLAG_UPSELL_HABILITADO,Valor_Original_Recarga as VALOR_ORIGINAL,Meio_de_Pagamento as MEIO_PAGAMENTO,Meio_de_Acesso_pagador as MSISDN_PAGADOR,CD_CELL_ID,CD_AREA_CELL_ID,NUM_FATURA as NUMERO_FATURA,DATA_HORA_FATURA,CNPJ_EMISSOR,NU_SERIE,NU_VOUCHER,if(CD_DIREITO IN {cd_direito_franquia},2,1) as ID_BOLSO,dt_execucao, dt from fortuna01_df_final_join".format(cd_direito_franquia = str(cd_direito_franquia).replace("[","(").replace("]",")")))

##Seleciona a estrutura final pelo tipo_recarga

fortuna01_df_bat = fortuna01_df_final.filter("TIPO_RECARGA = 'BAT'")

fortuna01_df_fis = fortuna01_df_final.filter("TIPO_RECARGA = 'FIS'")

fortuna01_df_elt = fortuna01_df_final.filter("TIPO_RECARGA = 'ELT'")

schema_bolso = "DH_EXTRACAO,MSISDN,VALOR_FACE,DT_SOLICITACAO_RECARGA,NSU_REQUISICAO,ID_RECARGA,DT_EFETIVACAO_RECARGA,PRODUTO_RECARGA,CANAL_RECARGA,SUB_CANAL,SALDO,CD_PRODUTO,CD_CAMPANHA,CD_TIPO_PRODUTO,TIPO_RECARGA,CD_PONTO_VENDA,COD_DISTRIBUIDOR_SAP,ORGANIZACAO_VENDA,SETOR_ATIVIDADE,CANAL_DISTRIBUICAO,FLAG_ACEITE_UPSELL,FLAG_UPSELL_HABILITADO,Valor_Original_Recarga,Meio_de_Pagamento,Meio_de_Acesso_pagador,CD_CELL_ID,CD_AREA_CELL_ID,ID_PLANO_FAT,NUMERO_PEDIDO,DATA_SOLIC_ORDEM_VENDA,NUM_FATURA,DATA_HORA_FATURA,CNPJ_EMISSOR,NU_SERIE,NU_VOUCHER,ID_BOLSO,DT_EXECUCAO,dt"

sap03_df = sqlContext.sql("select CD_DISTRIBUIDOR_SAP as COD_DISTRIBUIDOR_SAP, CD_CANAL_SAP as CANAL_DISTRIBUICAO, CD_SETOR_ATIVIDADE as SETOR_ATIVIDADE, CD_ORGANIZACAO_VENDA as ORGANIZACAO_VENDA, empresa as testeJoin from {db}.tb_distribuidores where modelo_de_contrato = 'Prest.Serv.Com Limit' or modelo_de_contrato = 'Prest.Serv.Sem Limit'".format(db = db_prepago))

fortuna01_df_elt_join = fortuna01_df_elt.join(sap03_df,["COD_DISTRIBUIDOR_SAP","CANAL_DISTRIBUICAO","SETOR_ATIVIDADE", "ORGANIZACAO_VENDA"],"left_outer")

# Separa recargas eletronicas em prestacao de servico e compra e venda

fortuna01_df_prest = fortuna01_df_elt_join.filter("testeJoin is not null").select([name for name in schema_bolso.split(",")])

fortuna01_df_cv = fortuna01_df_elt_join.filter("testeJoin is null").select([name for name in schema_bolso.split(",")])

##Os dados resultantes sao armazenados na tabela final

fortuna01_df_prest.write.mode("append").partitionBy("dt").saveAsTable(db_stg+".STG_PP_RECARGA_PS", format="parquet")

fortuna01_df_fis.write.mode("append").partitionBy("dt").saveAsTable(db_stg+".STG_PP_RECARGA_FIS", format="parquet")

fortuna01_df_bat.write.mode("append").partitionBy("dt").saveAsTable(db_stg+".STG_PP_RECARGA_REC", format="parquet")

fortuna01_df_cv.write.mode("append").partitionBy("dt").saveAsTable(db_stg+".STG_PP_RECARGA_CV", format="parquet")

fortuna01_recarga.write.mode("append").partitionBy("dt").saveAsTable(db_prepago+".TB_RECARGA", format="parquet")


from utils import get_db_collection
from bson.objectid import ObjectId
from Category import Category

COLLECTION_NAME = "Categories"

class CategoryAPI():
    #[input]  cat object
    #[return] _id of inserted category
    def insert(self, cat):
        collection = get_db_collection(COLLECTION_NAME)
        srchRlt = collection.find_one(cat.toQuery())
        if srchRlt is None:
            cat_id = collection.insert_one(cat.toQuery()).inserted_id
        else:
            cat_id = srchRlt['_id']
        return cat_id

    #[input]  cat object
    #[return] updated result
    def update(self, cat):
        collection = get_db_collection(COLLECTION_NAME)
        if cat.cat_id is None:
            print("Cat. ID cannot be None")
            return None
        elif collection.find_one({"_id": ObjectId(cat.cat_id)}) is None:
            print("Cat. ID does not exist, please create new cat. instead")
            return None
        else:
            return collection.update_one({ "_id": ObjectId(cat.cat_id)}, {'$set':cat.toQuery()})

    #[input]  cat id
    #[return] cat object
    def get(self, cat_id):
        collection = get_db_collection(COLLECTION_NAME)
        srchRlt = collection.find_one({"_id": ObjectId(cat_id)})
        if srchRlt == None:
            print("The Category corresponding the input cat. id does not exist")
            return None
        else:
            cat = Category(srchRlt["catName"], srchRlt["_id"])
            return cat

    #[input]  None
    #[return] list of cat object
    def list(self):
        collection = get_db_collection(COLLECTION_NAME)
        results = collection.find({})
        cats = []
        for result in results:
            cat = Category(result["catName"], result["_id"])
            cats.append(cat)
        return cats

    #[input]  cat id
    #[return] deleted result
    def delete(self, cat_id):
        collection = get_db_collection(COLLECTION_NAME)
        srchRlt = collection.find_one({"_id": ObjectId(cat_id)})
        if srchRlt is None:
            print("The Category corresponding the input cat. id does not exist")
            return None
        else:
            result = collection.delete_one({"_id": ObjectId(cat_id)})
            return result