from os import name
import os
from django.conf import settings
from django import http
from django.db import reset_queries
from django.http import JsonResponse
from django.contrib.auth.models import PermissionManager, User
from django.contrib import messages
from django.http import response
from django.http.response import Http404, HttpResponse
from django.shortcuts import redirect, render
from .models import Assignment, Comments, Results, Student, Class, Teacher, Grade, ClassMaterials, \
    ClassMaterialsChapter, ClassMaterialsModule, Files, Times, SubmittedAssignments, Quiz, Answers, Question, \
    EnrollementsAwaiting
import json
from django.http import HttpResponse
from datetime import datetime, time, timezone, timedelta
import datetime
import re

def index(request):
    if request.user.is_authenticated:
        time = ''
        try:
            us = Student.objects.get(user=request.user)
            classes = us.className.all()
        except:
            us = Teacher.objects.get(user=request.user)
            classes = us.class_set.all()
        dai = []
        sched = {}
        # for days in Times.get_all_days():
        #     for cl in classes:
        #         if len(cl.times_set.filter(day=days[0])) > 0:
        #             if days[0] not in dai:
        #                 # print(days[0], ': ')
        #                 dai.append(days[0])
        #                 sched[days[0]] = [{cl.name: cl.times_set.filter(day=days[0])[0].time}]
        #             else:
        #                 sched[days[0]].append({cl.name: cl.times_set.filter(day=days[0])[0].time})
        #                 pass
        for days in Times.get_all_days():
            times = Times.objects.filter(day=days[0])
            for time in times:
                if us == time.class_name.teacher or us in time.class_name.student_set.all():
                    print(days[0], time.class_name, time.time)
                    if days[0] not in dai:
                        dai.append(days[0])
                        sched[days[0]] = [{time.class_name: time.time}]
                    else:
                        sched[days[0]].append({time.class_name: time.time})
                        pass
        # print(sched)
        # for s in sched:
        #     print(s, sched[s])
        #     # for time in cl.times_set.all():
        #     #     print('class: ', time.class_name, 'day: ', time.day, 'time: ', time.time)
        grades = []
        gr_int = []
        if (request.user.groups.filter(name='Students').exists()):
            for class_name in classes:
                try:
                    grade = Grade.objects.get(studentName=us, className=class_name)
                    grades.append(grade)
                    gr_int.append(round((grade.grade / grade.number_of_total_grades) * 100, 2))
                except:
                    pass
        elif (request.user.groups.filter(name='Teachers').exists()):
            grades = 'not_student'
            # print('You are not a student')
            # messages.warning(request, "You cannot perform this action for the following reason(s): You are not a student'")
        if len(grades) == 0:
            grades = None

        print(grades)

        ctx = {'days': dai, 'sched': sched, 'classes': classes, 'grades': grades, 'gr_int': gr_int}
        return render(request, 'class/index.html', ctx)
    else:
        return redirect('index:login')

def class_info(request, class_name):
    if (request.user.groups.filter(name='Teachers').exists()):
        is_teacher = True
    else:
        is_teacher = False
    ctx = {}
    try:
        try:
            className = Class.objects.get(name=class_name)
        except Class.DoesNotExist:
            messages.error(request, "Class Does not exist")
            print('Class Does not exist')
            return redirect('class:index')
        try:
            student = Student.objects.get(user=request.user)
            cl = student.className.all()
            for c in cl:
                if class_name in c.name:
                    student_class = True
        except:
            student_class = False
            pass
        try:
            className.teacher.user == request.user
        except:
            return redirect('class:index')
        if className.teacher.user == request.user or student_class:
            pass
        else:
            messages.warning(request, "you do not have access")
            print('you do not have access')
            return redirect('class:index')
        class_assignments = Assignment.objects.filter(class_name=className)
        status_list = []
        for cl in class_assignments:
            try:
                stat = cl.submittedassignments_set.get(submitted_by=request.user)
                status_list.append(stat.status)
            except:
                if cl.due_date < datetime.date.today():
                    status_list.append('missed')
                else:
                    status_list.append('awaiting submission')

        chapters = ClassMaterialsChapter.objects.filter(className=className).order_by('-date_added')
        chap_mod = {}
        for chapter in chapters:
            module = ClassMaterialsModule.objects.filter(chapter=chapter)
            # print('chapter: ', chapter, ' module: ', module)
            chap_mod[chapter] = module
        # print(chap_mod)

        # for c in chap_mod:
        #     for m in chap_mod[c]:
        #         print(m.files_set.all())
        quizes = Quiz.objects.filter(class_name=className)
        quizes_list = []
        # print(Student.objects.filter(className=className))
        for quiz in quizes:
            timezone_offset = + 2.0
            tzinfo = timezone(timedelta(hours=timezone_offset))
            if quiz.max_time_to_take > datetime.datetime.now(tzinfo) and len(
                    quiz.results_set.filter(user=request.user)) == 0:
                quizes_list.append(quiz)
            for student in Student.objects.filter(className=className):
                if quiz.max_time_to_take < datetime.datetime.now(tzinfo) and len(
                        quiz.results_set.filter(user=student.user)) == 0:
                    Results(user=student.user, quiz=quiz, score=0).save()
                    number_of_total_grades = int(quiz.points)
                    gr = 0
                    try:
                        grade = Grade.objects.get(className=className, studentName=student)
                        number_of_total_grades = grade.number_of_total_grades + int(quiz.points)
                        gr = grade.grade + 0
                    except Grade.DoesNotExist:
                        grade = Grade(className=className, studentName=student)
                        grade.save()
                        print(grade.studentName)
                    Grade.objects.select_for_update().filter(className=className, studentName=student).update(
                        number_of_total_grades=number_of_total_grades, grade=gr)

        assignments = Assignment.objects.filter(class_name=className)
        assignments_list = []
        # print(Student.objects.filter(className=className))
        date_now = datetime.date(datetime.datetime.now().year, datetime.datetime.now().month,
                                 datetime.datetime.now().day)
        for assignment in assignments:
            for student in Student.objects.filter(className=className):
                if assignment.due_date < date_now and len(SubmittedAssignments.objects.filter(assignment=assignment,
                                                                                submitted_by=student.user)) == 0 and assignment.noted:
                    SubmittedAssignments(assignment=assignment, submitted_by=student.user, grade=0,
                                         status="graded").save()
                    number_of_total_grades = int(assignment.points)
                    gr = 0
                    try:
                        grade = Grade.objects.get(className=className, studentName=student)
                        number_of_total_grades = grade.number_of_total_grades + int(assignment.points)
                        gr = grade.grade + 0
                    except Grade.DoesNotExist:
                        grade = Grade(className=className, studentName=student)
                        grade.save()
                        print(grade.studentName)
                    Grade.objects.select_for_update().filter(className=className, studentName=student).update(
                        number_of_total_grades=number_of_total_grades, grade=gr)

        comments = Comments.objects.filter(class_name=className, parent=None)
        if request.method == 'POST':
            parent_obj = None
            new_comment = request.POST.get('new_comment')
            try:
                parent_id = request.POST.get('parent_id')
            except:
                parent_id = None

            if parent_id:
                parent_qs = Comments.objects.get(id=parent_id)
                if parent_qs:
                    print(parent_qs)
                    parent_obj = parent_qs

            print('parent_id', parent_id)
            new_c = Comments(author=request.user, text=new_comment, class_name=className, parent=parent_obj)
            new_c.save()
            return redirect('class:class_info', class_name)

        ctx = {'className': className, 'chap_mod': chap_mod, 'is_teacher': is_teacher, 'comments': comments,
               'class_assignments': class_assignments, 'status_list': status_list, 'quizes': quizes_list}
    except Class.DoesNotExist:
        ctx = {}
        messages.error(request, "Class Does not exist")
        print('class Does not exist')
    return render(request, 'class/class.html', ctx)

def add_new_mat(request):
    try:
        teacher = Teacher.objects.get(user=request.user)
        classes = Class.objects.filter(teacher=teacher)
        ctx = {'classes': classes}
        return render(request, 'class/add_new_material.html', ctx)
    except:
        return redirect('class:index')

def getdetails(request):
    data = {}
    if request.is_ajax():
        class_id = request.POST.get('class_name')
        chapter_id = request.POST.get('chapter')

        title = request.POST.get('name')
        url = request.POST.get('url_name')
        module_id = request.POST.get('module')
        if class_id != None and chapter_id != None and module_id != None:
            module = ClassMaterialsModule.objects.get(id=module_id)
            files = Files(ClassMaterialsModule=module, file_name=title, url=url)
            files.save()
            try:
                cl = Class.objects.get(id=class_id)
            except Class.DoesNotExist:
                messages.error(request, "Class Does not exist")
                print('Class Does not exist')
                return redirect('class:index')
            print('saving')
            messages.success(request, "New material added successfully!")
            return redirect('class:class_info', cl)

        class_name = request.POST.get('class')
        data['class_name'] = class_name
        print("ajax class_name ", data)

        result_set = []
        all_chapters = []
        answer = str(class_name[1:-1])
        try:
            selected_class = Class.objects.get(name=answer)
            all_chapters = selected_class.classmaterialschapter_set.all()
            print("selected class name ", selected_class)
        except:
            try:
                selected_chapter = ClassMaterialsChapter.objects.get(id=class_name)
                all_chapters = selected_chapter.classmaterialsmodule_set.all()
            except:
                return JsonResponse('', safe=False)

        for chapter in all_chapters:
            result_set.append({'name': chapter.name, 'id': chapter.id})
        print(result_set)

        return JsonResponse(result_set, safe=False)
        # return HttpResponse(json.dumps(result_set), mimetype='application/json',     content_type='application/json')

    else:
        return redirect('class:add_new_mat')

def new_chapter(request):
    if request.method == 'POST':
        class_name = request.POST.get('class_id_for_new_chapter')
        new_chapter = request.POST.get('new_chapter')
        new_chapter_desc = request.POST.get('new_chapter_desc')
        try:
            class_name = Class.objects.get(id=class_name)
        except Class.DoesNotExist:
            messages.error(request, "Class Does not exist")
            print('Class Does not exist')
            return redirect('class:index')
        print(class_name.classmaterialschapter_set.filter(name=new_chapter))
        if class_name.classmaterialschapter_set.filter(name=new_chapter):
            messages.warning(request, "Chapter name: \'" + str(new_chapter) + "\' already exists in this class")
            return redirect('class:add_new_mat')
        else:
            if class_name.teacher.user == request.user:
                new_chap = ClassMaterialsChapter(name=new_chapter, className=class_name, description=new_chapter_desc)
                new_chap.save()
                class_name = class_name.name
                messages.success(request, str(new_chapter) + " Added successfully to " + str(class_name))
                return redirect('class:add_new_mat')
            else:
                messages(request, 'You do not have access to this class')
                return redirect('class:add_new_mat')
    else:
        return redirect('class:add_new_mat')

def new_module(request):
    if request.method == 'POST':
        chapter_id = request.POST.get('chapter_id_for_new_module')
        new_module = request.POST.get('new_module')
        new_module_desc = request.POST.get('new_module_desc')
        chapter = ClassMaterialsChapter.objects.get(id=chapter_id)
        print(chapter.classmaterialsmodule_set.filter(name=new_module))
        if chapter.classmaterialsmodule_set.filter(name=new_module):
            messages.warning(request, "Module name: \'" + str(new_module) + "\' already exists in this chapter")
            return redirect('class:add_new_mat')
        else:
            teacher = chapter.className.teacher
            if teacher.user == request.user:
                new_mod = ClassMaterialsModule(name=new_module, chapter=chapter, description=new_module_desc)
                new_mod.save()
                messages.success(request, str(new_module) + " Added successfully to " + str(chapter.name))
                return redirect('class:add_new_mat')
            else:
                messages(request, 'You do not have access to this chapter')
                return redirect('class:add_new_mat')
    else:
        return redirect('class:add_new_mat')

def class_assignment(request, class_name, class_assignment_id):
    # if request.method == 'POST':

    assignment = Assignment.objects.get(id=class_assignment_id)
    try:
        submitted = SubmittedAssignments.objects.get(assignment=assignment, submitted_by=request.user)
        stat = submitted.status
        grade = submitted.grade
    except:
        stat = 'awaiting submission'
        grade = ''
    ass_class = assignment.class_name
    try:
        student = Student.objects.get(user=request.user)
        cl = student.className.all()
        for c in cl:
            if class_name in c.name:
                student_class = True
    except:
        student_class = False
        pass
    if ass_class.teacher.user == request.user or student_class:
        if request.method == 'POST':
            file_posted = request.FILES.get('file')
            print(request.FILES.get('file'))
            file = SubmittedAssignments(submitted_by=request.user, assignment=assignment, files=file_posted,
                                        status='submitted')
            file.save()
            return redirect('class:class_info', class_name)
    else:
        return redirect('class:index')
    ctx = {'assignment': assignment, 'status': stat, 'grade': grade, 'class_name': class_name}
    return render(request, 'class/assignment.html', ctx)

def new_assignment(request, class_name):
    try:
        Class.objects.get(name=class_name)
    except:
        return redirect('class:index')
    if request.method == 'POST':
        class_name = Class.objects.get(name=class_name)
        title = request.POST.get('title')
        description = request.POST.get('description')
        noted = request.POST.get('noted')
        points = request.POST.get('points')
        if points == '':
            points = None
        due_date = request.POST.get('due_date')
        assignment = Assignment(title=title, description=description, noted=noted, points=points, class_name=class_name,
                                due_date=due_date)
        assignment.save()
        return redirect('class:class_info', class_name.name)
    return render(request, 'class/new_assignment.html')

def view_all_submitted_assignment(request, class_name):
    try:
        cl = Class.objects.get(name=class_name)
    except Class.DoesNotExist:
        messages.error(request, "Class Does not exist")
        print('Class does not exist')
        return redirect('class:index')

    if request.user.groups.filter(name='Teachers').exists() and cl.teacher.user == request.user:
        all_ass = Assignment.objects.filter(class_name=cl)
        ctx = {'all_ass': all_ass, 'class_name': cl}
        return render(request, 'class/view_all_submitted_assignment.html', ctx)
    else:
        messages.error(request, "You do not have access")
        print('You do not have access')
        return redirect('class:class_info', class_name)

def view_submitted_assignment(request, class_name, assignment):
    try:
        class_name = Class.objects.get(name=class_name)
    except:
        return redirect('class:index')
    if request.user.groups.filter(name='Teachers').exists():
        if request.method == 'POST':
            ass = Assignment.objects.get(id=assignment)
            points = request.POST.get('points')
            submitted_id = request.POST.get('submitted_id')
            if int(points) > ass.points:
                messages.error(request, "Something went wrong")
                print('you cant do that')
            else:
                print(points)
                SubmittedAssignments.objects.select_for_update().filter(id=submitted_id).update(
                    grade=points, status='graded'
                )
                submitted_by = SubmittedAssignments.objects.get(id=submitted_id).submitted_by
                try:
                    student = Student.objects.get(user=submitted_by)
                except Student.DoesNotExist:
                    messages.error(request, "student does not existt")
                    print('student does not exist')
                    return redirect('class:view_submitted_assignment', class_name, assignment)
                number_of_total_grades = ass.points
                gr = int(points)
                try:
                    grade = Grade.objects.get(className=class_name, studentName=student)
                    number_of_total_grades = grade.number_of_total_grades + ass.points
                    gr = grade.grade + int(points)
                except Grade.DoesNotExist:
                    grade = Grade(className=class_name, studentName=student)
                    grade.save()
                Grade.objects.select_for_update().filter(className=class_name, studentName=student).update(
                    number_of_total_grades=number_of_total_grades, grade=gr)

        assignment = Assignment.objects.filter(id=assignment)

        ctx = {'assignment': assignment}
        return render(request, 'class/submitted_assignments.html', ctx)
    else:
        return redirect('class:class_info', class_name)

def new_quiz(request, class_name):
    data = {}
    if request.user.groups.filter(name='Teachers').exists():
        # * to remove
        # teacher = Teacher.objects.get(user=request.user)
        # all_classes = Class.objects.filter(teacher=teacher)

        # if request.method == 'POST':
        #     for key, value in request.POST.items():
        #         print('%s' % (value) )

        if request.is_ajax() and request.POST.get('question') is not None:
            print('Question: ', request.POST.get('question'))
            print('quiz_id: ', request.POST.get('quiz_id'))
            quiz_id = request.POST.get('quiz_id')
            quiz = Quiz.objects.get(id=quiz_id)
            quest = Question(text=request.POST.get('question'), quiz=quiz)
            quest.save()
            list = []
            for key, value in request.POST.items():
                if 'choice' in str(key):
                    list.append(value)
            i = 0
            while i < len(list):
                ans = Answers(text=list[i:i + 2][0], correct=list[i:i + 2][1], question=quest)
                ans.save()
                i = i + 2
            print(list)

        if request.is_ajax() and request.POST.get('question') is None:
            title = request.POST.get('title')
            duration = request.POST.get('duration')
            to_pass = request.POST.get('to_pass')
            max_date = request.POST.get('max_date')
            max_time = request.POST.get('max_time')
            data['title'] = title
            data['duration'] = duration
            data['to_pass'] = to_pass
            data['max_date'] = max_date
            data['max_time'] = max_time
            date_time_str = max_date + ' ' + max_time + ':00'
            print(date_time_str)
            date_time = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
            try:
                class_object = Class.objects.get(name=class_name)
            except Class.DoesNotExist:
                messages.error(request, "Class does not exist")
                print('Class Does not exist')
                return redirect('class:index')
            quiz = Quiz(title=title, class_name=class_object, time=duration, points=to_pass, max_time_to_take=date_time)
            quiz.save()
            quiz = Quiz.objects.get(id=quiz.id)
            data['quiz_id'] = quiz.id
            print(data)
            return JsonResponse(data, safe=False)

        ctx = {'class_name': class_name}
        return render(request, 'class/new_quiz.html', ctx)
    else:
        return redirect('class:class_info', class_name)

def quiz(request, class_name, quiz_id):
    try:
        quiz = Quiz.objects.get(id=quiz_id)
    except Quiz.DoesNotExist:
        messages.error(request, "Quiz does not exist")
        print('quiz Does not exist')
        return redirect('class:class_info', class_name)

    quiz_class = quiz.class_name
    student_class = False
    try:
        student = Student.objects.get(user=request.user)
        cl = student.className.all()
        for c in cl:
            if class_name in c.name:
                student_class = True
                break
    except:
        student_class = False
        pass
    if quiz_class.teacher.user == request.user or student_class:
        # print(len(quiz.get_questions()))
        if request.method == 'POST':
            answers_list = []
            correct_answers_list = []
            questions_answered_list = []
            total_ans_count = 0
            for quest in quiz.get_questions():
                total_ans_count += len(quest.answers_set.filter(correct=True))
                ans = Answers.objects.filter(question=quest, correct=True)
                for x in range(len(ans)):
                    correct_answers_list.append([ans[x].question.text, ans[x].text])
            # print(total_ans_count)
            correct_count = 0
            for key, value in request.POST.items():
                if key != 'csrfmiddlewaretoken':
                    reg = key.split('_')
                    quest_id = reg[0].strip('question')
                    choice_id = reg[1].strip('choice')
                    answers_list.append([reg[0].strip('question'), reg[1].strip('choice')])
                    answer = Answers.objects.get(id=choice_id)
                    question = Question.objects.get(id=quest_id)
                    # correct_answers_list.append([ans.question, ans.answer])
                    # print(quest_id, questions_answered_list)
                    # if answer.correct and quest_id not in questions_answered_list:
                    #     correct_answers_list.append([question.text, answer.text])
                    #     correct_count += 1
                    #     continue
                    # elif answer.correct and quest_id in questions_answered_list:
                    #     correct_count -= 1
                    #     # correct_answers_list.remove([question.text, answer.text])
                    # else:
                    #     correct_count -= 1
                    # questions_answered_list.append(quest_id)

            # print('answers_list: ', answers_list)
            # print('----------------------------------')

            your_answers = []
            for q in answers_list:
                quest = Question.objects.get(id=q[0])
                answ = Answers.objects.get(id=q[1])
                your_answers.append([quest.text, answ.text])

            # print(your_answers)
            # print('----------------------------------')
            # print(correct_answers_list)
            # print('----------------------------------')

            score_count = 0
            for ans in your_answers:
                if ans in correct_answers_list:
                    score_count += 1
                else:
                    score_count -= 1
            print(score_count)

            if score_count <= 0:
                print('Your score is: ', 0)
                score = 0
            else:
                score = int((quiz.points * score_count) / total_ans_count)
                print('Your score is: ', score, '%')

            rere = Results.objects.filter(user=request.user, quiz=quiz, score=0)
            print(rere)
            Results.objects.select_for_update().filter(user=request.user, quiz=quiz, score=0).update(score=score)

            number_of_total_grades = int(quiz.points)
            gr = int(score)
            clss = Class.objects.get(name=class_name)
            try:
                try:
                    grade = Grade.objects.get(className=clss, studentName=student)
                    number_of_total_grades = grade.number_of_total_grades + int(quiz.points)
                    gr = grade.grade + int(score)
                except Grade.DoesNotExist:
                    grade = Grade(className=clss, studentName=student)
                    grade.save()
                Grade.objects.select_for_update().filter(className=clss, studentName=student).update(
                    number_of_total_grades=number_of_total_grades, grade=gr)
            except:
                pass

            # print(correct_answers_list)
            # print(checks)
            # for c in checks:
            #     if c in correct_answers_list:
            #         print(c, ' correct')
            #     else:
            #         print(c, ' False'

            ctx_quiz = {'correct_answers_list': correct_answers_list, 'answers': your_answers, 'score': score}
            return render(request, 'class/quiz_score.html', ctx_quiz)
        ctx = {'quiz': quiz}
        if len(quiz.results_set.filter(user=request.user)) != 0:
            messages.error(request, "You already took the test")
            print('You already took the test')
            return redirect('class:class_info', class_name)
        res = Results(quiz=quiz, user=request.user, score=0)
        res.save()
        return render(request, 'class/quiz.html', ctx)

    else:
        messages.error(request, "You do not have access")
        print('you do not have access')
        return redirect('class:class_info', class_name)

def submitted_quizes(request, class_name):
    if request.user.groups.filter(name='Teachers').exists():
        try:
            cl = Class.objects.get(name=class_name)
        except Class.DoesNotExist:
            messages.error(request, "Class does not exist")
            print('Class Does not exist')
            return redirect('class:index')
        class_quizes = Quiz.objects.filter(class_name=cl)

        if cl.teacher.user == request.user:
            pass
        else:
            messages.error(request, "You do not have access")
            print("you do not have access")
            return redirect('class:class_info', class_name)

        for quiz in class_quizes:
            print(quiz, quiz.results_set.all())

        ctx = {'class_quizes': class_quizes, 'class_name': class_name}
        return render(request, 'class/submitted_quizes.html', ctx)
    else:
        messages.error(request, "You do not have access")
        print("you do not have access")
        return redirect('class:class_info', class_name)

def view_submitted_quiz(request, class_name, quiz_id):
    try:
        quiz = Quiz.objects.get(id=quiz_id)
    except Quiz.DoesNotExist:
        messages.error(request, "Quiz does not exist")
        print('quiz Does not exist')
        return redirect('class:class_info', class_name)

    try:
        cl = Class.objects.get(name=class_name)
    except Class.DoesNotExist:
        messages.error(request, "Quiz does not exist")
        print('Class Does not exist')
        return redirect('class:index')

    if request.user.groups.filter(name='Teachers').exists():
        if cl.teacher.user == request.user:
            pass
        else:
            messages.error(request, "You do not have access")
            print("you do not have access")
            return redirect('class:class_info', class_name)

        submitted = Results.objects.filter(quiz=quiz)
        ctx = {'class_name': class_name, 'quiz': quiz, 'submitted': submitted}
        return render(request, 'class/view_submitted_quiz.html', ctx)
    else:
        messages.error(request, "You do not have access")
        print("you do not have access")
        return redirect('class:class_info', class_name)

def class_grades(request, class_name):
    is_teacher = False
    try:
        cl = Class.objects.get(name=class_name)
    except Class.DoesNotExist:
        messages.error(request, "Class does not exist")
        print('Class Does not exist')
        return redirect('class:index')

    if (request.user.groups.filter(name='Teachers').exists()):
        is_teacher = True
        all_students = Student.objects.filter(className=cl)
        grades = {}
        for st in all_students:
            try:
                grade = Grade.objects.get(studentName=st)
                print(f"student: {st}-grade:{grade.grade}")
                grades[st] = grade.grade
            except:
                print(f"student: {st}-no grades yet for this student")
                grades[st] = ""
        print(grades)
        ctx = {'is_teacher': is_teacher, 'grades': grades, 'class_name': class_name}
        return render(request, 'class/class_grades.html', ctx)

    try:
        student = Student.objects.get(user=request.user)
    except:
        messages.error(request, "You cannot perform this action for the following reason(s): You are not a student")
        print("You are not a student")
        return redirect('class:class_info', class_name)

    try:
        grades = Grade.objects.get(studentName=student, className=cl)
    except Grade.DoesNotExist:
        messages.error(request, "Student has no grades yet")
        print('Student has no grades yet')
        grades = ''
    class_quizes = Quiz.objects.filter(class_name=cl)
    res = []
    for quiz in class_quizes:
        try:
            res.append(Results.objects.get(user=request.user, quiz=quiz))
        except:
            pass

    assignemnts = Assignment.objects.filter(class_name=cl, noted=True)
    submitted_ass = []
    for ass in assignemnts:
        try:
            submitted_ass.append(ass.submittedassignments_set.get(submitted_by=request.user, status='graded'))
        except:
            pass
    print(submitted_ass)

    ctx = {'grades': grades, 'res': res, 'submitted_ass': submitted_ass}
    return render(request, 'class/class_grades.html', ctx)

def student_grade(request, class_name, student_name):
    try:
        teacher = Teacher.objects.get(user=request.user)
    except Teacher.DoesNotExist:
        messages.error(request, "Student has no grades yet")
        print('You are not a teacher')
        return redirect('class:class_info', class_name)

    try:
        cl = Class.objects.get(name=class_name)
    except Class.DoesNotExist:
        messages.error(request, "Class Does not exist")
        print('Class Does not exist')
        return redirect('class:index')

    try:
        user = User.objects.get(username=student_name)
    except User.DoesNotExist:
        messages.error(request, "User with that username does not exist")
        print("User with that username does not exist")
        return redirect('class:class_info', class_name)

    try:
        student = Student.objects.get(user=user)
    except Student.DoesNotExist:
        messages.error(request, "Student does not exist")
        print("Student Does not exist")
        return redirect('class:class_info', class_name)

    is_part = False
    if cl in teacher.class_set.all():
        for st in Student.objects.filter(className=cl):
            if st == student:
                is_part = True
                break
    else:
        messages.error(request, "You are not a teacher for that class")
        print('You are not a teacher of that class')
        return redirect('class:class_info', class_name)

    if is_part:
        class_quizes = Quiz.objects.filter(class_name=cl)
        res = []
        try:
            grades = Grade.objects.get(studentName=student, className=cl)
        except Grade.DoesNotExist:
            messages.warning(request, "Student has no grades yet")
            print('Student has no grades yet')
            grades = ''
        for quiz in class_quizes:
            try:
                res.append(Results.objects.get(user=user, quiz=quiz))
            except:
                pass

        assignemnts = Assignment.objects.filter(class_name=cl, noted=True)
        submitted_ass = []
        for ass in assignemnts:
            try:
                submitted_ass.append(ass.submittedassignments_set.get(submitted_by=user, status='graded'))
            except:
                pass
    else:
        messages.error(request, "This student is not a part of your class")
        print('This student is not a part of your class')
        return redirect('class:class_grades', class_name)

    ctx = {'grades': grades, 'res': res, 'submitted_ass': submitted_ass}
    return render(request, 'class/student_grade.html', ctx)

def enroll(request):
    classes_list = Class.objects.all()
    classes_to_enroll = []
    try:
        student = Student.objects.get(user=request.user)
        classes = student.className.all()
    except:
        messages.error(request, "This action cannot be completed for the following reason(s): You are not a student")
        print("you are not a student")
        return redirect('class:index')
    for cl in classes_list:
        if cl in classes:
            continue
        else:
            classes_to_enroll.append(cl)

    ctx = {"classes_list": classes_to_enroll}
    return render(request, 'class/enroll.html', ctx)

def enroll_class(request, class_name):
    if request.method == "POST":
        class_id = request.POST.get('class_id')
        try:
            class_name = Class.objects.get(id=class_id)
        except Class.DoesNotExist:
            messages.error(request, "Class Does not exist")
            print('Class Does not exist')
            return redirect('class:index')

        try:
            student = Student.objects.get(user=request.user)
        except Student.DoesNotExist:
            messages.error(request, "Student Does not exist")
            print("Student does not exist")
            return redirect("class:enroll")

        try:
            EnrollementsAwaiting.objects.get(student=student, class_name=class_name)
            messages.error(request,
                           "You have already submitted a request to enroll in this class please wait you administration response!")
            print(
                "You have already submitted a request to enroll in this class please wait you administration response!")
            return redirect("class:enroll")
        except:
            pass

        EnrollementsAwaiting(student=student, class_name=class_name).save()
        messages.success(request,
                         f"Your request to enroll for class \"{class_name.name}\" was submitted please wait patiently for your application to be reviewed!")
        print(
            f"Your request to enroll for class \"{class_name.name}\" was submitted please wait patiently for your application to be reviewed!")
        return redirect('class:enroll')

    try:
        student = Student.objects.get(user=request.user)
        classes = student.className.all()
    except:
        messages.error(request, "The following error has occured: You are not a student")
        print("you are not a student")
        return redirect('class:index')
    try:
        class_obj = Class.objects.get(name=class_name)
    except Class.DoesNotExist:
        messages.error(request, "Class does not exist")
        print('Class Does not exist')
        return redirect('class:index')

    if class_obj in classes:
        messages.error(request, "You are already enrolled in that class")
        print("You are already enrolled in that class")
        return redirect('class:class_info', class_obj.name)

    times = Times.objects.filter(class_name=class_obj)
    print(times)

    ctx = {"class_obj": class_obj, "times": times}
    return render(request, 'class/enroll_class.html', ctx)

def check_enrollement_submissions(request):
    if request.method == "POST":
        student_id = request.POST.get('student_id')
        class_id = request.POST.get('class_id')
        approve = request.POST.get('approve')
        try:
            class_name = Class.objects.get(id=class_id)
        except Class.DoesNotExist:
            messages.error(request, "Class does not exist")
            print('Class Does not exist')
            return redirect('class:index')

        try:
            student = Student.objects.get(id=student_id)
        except Student.DoesNotExist:
            messages.error(request, "Student does not exist")
            print("Student Does Not Exist")
            return redirect('class:index')

        if approve == 'yes':
            student.className.add(class_name)
            messages.success(request, f"student {student.user.username} successfully enrolled in {class_name}")
            print(f'student {student.user.username} successfully enrolled in {class_name}')
        else:
            messages.error(request, "Declined")
            print('declined')

        EnrollementsAwaiting.objects.filter(student=student, class_name=class_name).delete()

    if not request.user.is_superuser:
        print('You cannot do that')
        return redirect('class:index')

    submissions = EnrollementsAwaiting.objects.all()

    ctx = {"submissions": submissions}
    return render(request, 'class/check_enrollement_submissions.html', ctx)

# def new_for_class(request):
#     groups = request.user.groups.all()
#     ctx = {}
#     if (request.user.groups.filter(name='Teachers').exists()):
#         teacher = Teacher.objects.get(user=request.user)
#         teacher_classes = Class.objects.filter(teacher=teacher)

#         ctx = {'teacher_classes': teacher_classes}
#     if request.method == 'POST':
#         classe = request.POST.get('class')
#         print(classe)
#         try:
#             Class.objects.get(id=classe)
#         except:
#             messages.error(request, 'Please Select a class first')
#             return render(request, 'class/test.html', ctx)

#         return redirect('class:new_for_chapter', classe)

#     return render(request, 'class/test.html', ctx)

# def new_for_chapter(request, classe):
#     print(classe)
#     ctx = {}
#     try:
#         classe = Class.objects.get(id=classe)
#         ctx = {'class':classe}
#     except:
#         messages.error(request, 'Please Select a class first')
#         return redirect('class:new_for_class')
#     if request.method == 'POST':
#         chapter = request.POST.get('chapter')
#         try:
#             ClassMaterialsChapter.objects.get(id=chapter)
#         except:
#             messages.error(request, 'Please Select a chapter first')
#             return render(request, 'class/test.html', ctx)

#         return redirect('class:new_for_module', chapter)

#     return render(request, 'class/test.html', ctx)

# def new_for_module(request, chapter):
#     try:
#         chapter = ClassMaterialsChapter.objects.get(id=chapter)
#         ctx = {'chapter': chapter}
#     except:
#         messages.error(request, 'Please Select a class first')
#         return redirect('class:new_for_class')

#     if request.method == 'POST':
#         module = request.POST.get('module')
#         try:
#             module = ClassMaterialsModule.objects.get(id=module)
#         except:
#             messages.error(request, 'Please Select a module first')
#         return redirect('class:adding_materials', module.chapter.className, module.chapter, module)

#     return render(request, 'class/test.html', ctx)

# def adding_materials(request, class_name, chapter_name, module_name):
#     ctx = {'all_info':(class_name, chapter_name, module_name)}
#     try:
#         classe = Class.objects.get(name=class_name)
#         if classe.teacher.user.username != request.user.username:
#             messages.error(request, 'You do not have access to that page')
#             return redirect('class:index')

#         c = classe.classmaterialschapter_set.all()
#         for clas in c:
#             if chapter_name == clas.name:
#                 c = clas
#                 break
#         mod = c.classmaterialsmodule_set.all()
#         for m in mod:
#             if module_name == m.name:
#                 modu = m
#                 break
#     except:
#         messages.error(request, 'Something went wrong')

#     if request.method == 'POST':
#         mat_name = request.POST.get('name')
#         mat_url = request.POST.get('url')

#         f = Files(file_name=mat_name, url=mat_url, ClassMaterialsModule=modu)
#         f.save()
#         return redirect('class:index')

#     return render(request, 'class/test.html', ctx)


import pandas as pd
from flask import Flask, request, render_template
import requests
import json
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def index():
 	return render_template('home.html')
@app.route('/process',methods=['GET', 'POST'])
def nextFn():
    # Dataset from - https://archive.ics.uci.edu/ml/datasets/SMS+Spam+Collection
    df = pd.read_table('smsspamcollection/SMSSpamCollection',
                   sep='\t',
                   header=None,
                   names=['label', 'sms_message'])
    df['label'] = df.label.map({'ham':0, 'spam':1})
    count_vector = CountVectorizer()

    X_train, X_test, y_train, y_test = train_test_split(df['sms_message'],
                                                    df['label'],
                                                    random_state=1)

    count_vector = CountVectorizer()

    # Fit the training data and then return the matrix
    training_data = count_vector.fit_transform(X_train)
    naive_bayes = MultinomialNB()
    naive_bayes.fit(training_data, y_train)
    user_text= request.form.get('raw')
    t_data= count_vector.transform([str(user_text)])
    p = naive_bayes.predict(t_data)
    if p[0]==0:
        result="Not Spam"
        return render_template('response.html', result=result)
    else:
        result="Spam"
        return render_template('response.html', result=result)

if __name__=='__main__':
    app.run()


import xlsxwriter
from constants import XLS_FILE

class Excel:
    def __init__(self):
        self.workbook = xlsxwriter.Workbook(XLS_FILE)
        self.worksheet = self.workbook.add_worksheet()
        self.actual_row = 0
        self.col_names = [
            "referencia",
            "nombre",
            "precio",
            "imagen",
            "descripcion",
        ]

        self.__write_headers()

    def __write_headers(self):
        row = 0
        col = 0

        for header in self.col_names:
            self.worksheet.write(row, col, header)
            col += 1

        self.actual_row = 1

    def __write_row(self, element: dict):
        print(element)
        col = 0
        for value in list(element.values()):
            self.worksheet.write(self.actual_row, col, value)
            col += 1

        self.actual_row += 1

    def write_data(self, item_list: list):
        print("\nEscribiendo datos en excel...")
        for element in item_list:
            self.__write_row(element)


import os

import numpy as np
import pandas as pd
from scipy.special import expit

from Utils import Utils

class DataLoader:
    def preprocess_for_graphs(self, csv_path):
        df = pd.read_csv(os.path.join(os.path.dirname(__file__), csv_path), header=None)
        return self.__convert_to_numpy(df)

    @staticmethod
    def load_train_test_jobs(train_path, test_path, iter_id):
        train_arr = np.load(train_path)
        test_arr = np.load(test_path)
        np_train_X = train_arr['x'][:, :, iter_id]
        np_train_T = Utils.convert_to_col_vector(train_arr['t'][:, iter_id])
        np_train_e = Utils.convert_to_col_vector(train_arr['e'][:, iter_id])
        np_train_yf = Utils.convert_to_col_vector(train_arr['yf'][:, iter_id])

        train_X = np.concatenate((np_train_X, np_train_e, np_train_yf), axis=1)
        train_X, val_X, train_T, val_T = \
            Utils.test_train_split(train_X, np_train_T, split_size=0.90)

        np_test_X = test_arr['x'][:, :, iter_id]
        np_test_T = Utils.convert_to_col_vector(test_arr['t'][:, iter_id])
        np_test_e = Utils.convert_to_col_vector(test_arr['e'][:, iter_id])
        np_test_yf = Utils.convert_to_col_vector(test_arr['yf'][:, iter_id])

        test_X = np.concatenate((np_test_X, np_test_e, np_test_yf), axis=1)

        print("Numpy Train Statistics:")
        print(train_X.shape)
        print(train_T.shape)

        print("Numpy Val Statistics:")
        print(val_X.shape)
        print(val_T.shape)

        print(" Numpy Test Statistics:")
        print(test_X.shape)
        print(np_test_T.shape)

        # X -> x1.. x17, e, yf -> (19, 1)
        return train_X, test_X, val_X, train_T, np_test_T, val_T

    def preprocess_data_from_csv_augmented(self, csv_path, split_size):
        # print(".. Data Loading synthetic..")
        # data load
        df = pd.read_csv(os.path.join(os.path.dirname(__file__), csv_path), header=None)
        np_covariates_X, np_treatment_Y = self.__convert_to_numpy_augmented(df)
        # print("ps_np_covariates_X: {0}".format(np_covariates_X.shape))
        # print("ps_np_treatment_Y: {0}".format(np_treatment_Y.shape))

        np_covariates_X_train, np_covariates_X_test, np_covariates_Y_train, np_covariates_Y_test = \
            Utils.test_train_split(np_covariates_X, np_treatment_Y, split_size)
        # print("np_covariates_X_train: {0}".format(np_covariates_X_train.shape))
        # print("np_covariates_Y_train: {0}".format(np_covariates_Y_train.shape))
        # print("---" * 20)
        # print("np_covariates_X_test: {0}".format(np_covariates_X_test.shape))
        # print("np_covariates_Y_test: {0}".format(np_covariates_Y_test.shape))
        return np_covariates_X_train, np_covariates_X_test, np_covariates_Y_train, np_covariates_Y_test

    @staticmethod
    def convert_to_tensor(ps_np_covariates_X, ps_np_treatment_T):
        return Utils.convert_to_tensor(ps_np_covariates_X, ps_np_treatment_T)

    def prepare_tensor_for_DCN(self, ps_np_covariates_X, ps_np_treatment_T, ps_list,
                               is_synthetic):
        # ps_np_covariates_X -> X1 .. X17, y_f
        # print("ps_np_covariates_X: {0}".format(ps_np_covariates_X.shape))
        # print("ps_np_treatment_Y: {0}".format(ps_np_treatment_Y.shape))
        X = Utils.concat_np_arr(ps_np_covariates_X, ps_np_treatment_T, axis=1)

        # col of X -> x1 .. x17, Y_f, T, Ps
        X = Utils.concat_np_arr(X, np.array([ps_list]).T, axis=1)
        print("Big X: {0}".format(X.shape))

        df_X = pd.DataFrame(X)
        treated_df_X, treated_ps_score, treated_df_Y_f, treated_df_e = \
            self.__preprocess_data_for_DCN(df_X, treatment_index=1,
                                           is_synthetic=is_synthetic)

        control_df_X, control_ps_score, control_df_Y_f, control_df_e = \
            self.__preprocess_data_for_DCN(df_X, treatment_index=0,
                                           is_synthetic=is_synthetic)

        np_treated_df_X, np_treated_ps_score, np_treated_df_Y_f, np_treated_df_e = \
            self.__convert_to_numpy_DCN(treated_df_X, treated_ps_score, treated_df_Y_f, treated_df_e)

        np_control_df_X, np_control_ps_score, np_control_df_Y_f, np_control_df_e = \
            self.__convert_to_numpy_DCN(control_df_X, control_ps_score, control_df_Y_f, control_df_e)

        # np_treated_df_Y_f = Utils.convert_to_col_vector()

        print(" Treated Statistics ==>")
        print(np_treated_df_X.shape)
        print(" Control Statistics ==>")
        print(np_control_df_X.shape)

        return {
            "treated_data": (np_treated_df_X, np_treated_ps_score,
                             np_treated_df_Y_f, np_treated_df_e),
            "control_data": (np_control_df_X, np_control_ps_score,
                             np_control_df_Y_f, np_control_df_e)
        }

    @staticmethod
    def __convert_to_numpy(df):
        covariates_X = df.iloc[:, 5:]
        treatment_Y = df.iloc[:, 0:1]
        outcomes_Y = df.iloc[:, 1:3]

        np_covariates_X = Utils.convert_df_to_np_arr(covariates_X)
        np_outcomes_Y = Utils.convert_df_to_np_arr(outcomes_Y)
        np_X = Utils.concat_np_arr(np_covariates_X, np_outcomes_Y, axis=1)

        np_treatment_Y = Utils.convert_df_to_np_arr(treatment_Y)

        return np_X, np_treatment_Y

    @staticmethod
    def __convert_to_numpy_augmented(df):
        covariates_X = df.iloc[:, 5:]
        treatment_Y = df.iloc[:, 0:1]
        outcomes_Y = df.iloc[:, 1:3]

        np_covariates_X = Utils.convert_df_to_np_arr(covariates_X)
        np_std = np.std(np_covariates_X, axis=0)
        np_outcomes_Y = Utils.convert_df_to_np_arr(outcomes_Y)

        noise = np.empty([747, 25])
        id = -1
        for std in np_std:
            id += 1
            noise[:, id] = np.random.normal(0, 1.96 * std)

        random_correlated = np_covariates_X + noise

        random_X = np.random.permutation(np.random.random((747, 25)) * 10)
        np_covariates_X = np.concatenate((np_covariates_X, random_X), axis=1)
        np_covariates_X = np.concatenate((np_covariates_X, random_correlated), axis=1)
        np_X = Utils.concat_np_arr(np_covariates_X, np_outcomes_Y, axis=1)

        np_treatment_Y = Utils.convert_df_to_np_arr(treatment_Y)

        return np_X, np_treatment_Y

    @staticmethod
    def __preprocess_data_for_DCN(df_X, treatment_index, is_synthetic):
        df = df_X[df_X.iloc[:, -2] == treatment_index]
        # col of X -> x1 .. x17, Y_f, T, Ps
        if is_synthetic:
            # for synthetic dataset #covariates: 75
            df_X = df.iloc[:, 0:75]
        else:
            # for original dataset #covariates: 17
            df_X = df.iloc[:, 0:17]

        ps_score = df.iloc[:, -1]
        df_Y_f = df.iloc[:, -3]
        df_e = df.iloc[:, -4]

        return df_X, ps_score, df_Y_f, df_e

    @staticmethod
    def __convert_to_numpy_DCN(df_X, ps_score, df_Y_f, df_e):
        np_df_X = Utils.convert_df_to_np_arr(df_X)
        np_ps_score = Utils.convert_df_to_np_arr(ps_score)
        np_df_Y_f = Utils.convert_df_to_np_arr(df_Y_f)
        np_df_e = Utils.convert_df_to_np_arr(df_e)
        # print("np_df_X: {0}".format(np_df_X.shape))
        # print("np_ps_score: {0}".format(np_ps_score.shape))
        # print("np_df_Y_f: {0}".format(np_df_Y_f.shape))

        return np_df_X, np_ps_score, np_df_Y_f, np_df_e

    @staticmethod
    def preprocess_dataset_for_training(csv_path):
        data_X = np.loadtxt(csv_path, delimiter=",", skiprows=1)

        # Define features
        x = data_X[:, :30]
        no, dim = x.shape

        # Define potential outcomes
        potential_y = data_X[:, 30:]
        # Die within 1 year = 1, otherwise = 0
        potential_y = np.array(potential_y < 9999, dtype=float)

        # Assign treatment
        coef = np.random.uniform(-0.01, 0.01, size=[dim, 1])
        prob_temp = expit(np.matmul(x, coef) + np.random.normal(0, 0.01, size=[no, 1]))
        prob_t = prob_temp / (2 * np.mean(prob_temp))
        prob_t[prob_t > 1] = 1

        t = np.random.binomial(1, prob_t, [no, 1])
        t = t.reshape([no, ])
        t = Utils.convert_to_col_vector(t)
        y = np.zeros([no, 1])
        y = np.transpose(t) * potential_y[:, 1] + np.transpose(1 - t) * potential_y[:, 0]
        y = np.reshape(np.transpose(y), [no, ])
        y_f = Utils.convert_to_col_vector(y)
        np_X = np.concatenate((x, y_f, potential_y), axis=1)

        print(x.shape)
        print(potential_y.shape)
        print(y_f.shape)
        print(t.shape)

        print(np_X.shape)
        return np_X, t


# coding:utf-8

from flask import Flask,make_response,jsonify
from flask_restful import Api, Resource
from flask_restful import reqparse
from flask_restful import fields, marshal
from flask_httpauth import HTTPBasicAuth
from algorithm import KeywordExtraction

app = Flask(__name__,static_url_path="")
app.config.update(RESTFUL_JSON=dict(ensure_ascii=False)) #保证输出为中文而不是Unicode码，默认为True，所有的非ascii字符在输出时都会被转义为\uxxxx的序列
api = Api(app)

# auth = HTTPBasicAuth()

# @auth.get_password
# def get_password(username):
#     if username == 'wzy':
#         return '123456'
#     return None

# @auth.error_handler
# def unauthorized():
#     return make_response(jsonify({'error': 'Unauthorized access'}), 403) #这里需要使用jsonify

class KnoAPI(Resource):
    # decorators = [auth.login_required]
    def __init__(self):
        self.reqparse = reqparse.RequestParser()
        self.reqparse.add_argument('text', type = str, required = True, help = 'No text provided', location = 'json')
        self.reqparse.add_argument('mode', type = int, default = 0, location = 'json')
        self.reqparse.add_argument('num', type = int, default = 5, location = 'json')
        self.reqparse.add_argument('weight', type = int, default = 0, location = 'json')
        super(KnoAPI, self).__init__()

    def post(self):
        args = self.reqparse.parse_args()
        inputDict = {}
        inputDict['text'] = args['text']
        if(args['mode'] == 0 or args['mode'] == 1 ):
            inputDict['mode'] = args['mode']
        else:
            inputDict['mode'] = 0
        if(args['weight'] == 0 or args['weight'] == 1 ):
            inputDict['weight'] = args['weight']
        else:
            inputDict['weight'] = 0
        inputDict['keywordNum'] = args['num']
        inputDict['url'] = None
        keywordResult = KeywordExtraction(inputDict)

        result = {
            'mode':args['mode'],
            'num':args['num'],
            'keyword':keywordResult
        }

        return {'result':result}

api.add_resource(KnoAPI, '/kno/api/v1.0/getKeyword', endpoint = 'keyword')

if __name__ == '__main__':
    app.run(debug=True)


import cv2
import numpy as np
import config

# Get color ranges and noise removal kernels from config
ball_color_range = config.get("colors", config.get("vision", "ball_color"))
ball_noise_kernel = config.get("vision", "ball_noise_kernel")
# basket_color_range = config.get("colors", config.get("vision", "basket_color"))
edge_color_range = config.get("colors", config.get("vision", "bounds"))

def apply_ball_color_filter(hsv, basket=False, bounds = False, korv=None, piir=None):
    #print(ball_color_range)
    #print(hsv)
    height = 0
    w, h, x2, y2 = 0,0,0,0

    basket_color_range = config.get("colors", config.get("vision", "basket_color"))

    """
        args:
            hsv image
        returns:
            1) masked image of the ball
            2) masked image of the basket
            3) [x, y] for the coordinates of the ball
            4) [x, y] for the coordinates of the basket
    """
    # hsv = cv2.blur(hsv, (2,2))
    #print(ball_color_range)
    if basket:
        masked_img = cv2.inRange(hsv, basket_color_range["min"], basket_color_range["max"])
        #cv2.imshow("hsv", hsv)
        #print(basket_color_range["min"], basket_color_range["max"])
    elif bounds:
        masked_img = cv2.inRange(hsv, edge_color_range["min"], edge_color_range["max"])
        pass
    else:
        masked_img = cv2.inRange(hsv, ball_color_range["min"], ball_color_range["max"])
        #print(ball_color_range["min"], ball_color_range["max"])
    kernel = np.ones((5, 5), np.uint8)
    masked_img2 = cv2.morphologyEx(masked_img, cv2.MORPH_OPEN, kernel)
    erosion = cv2.erode(masked_img2, kernel, iterations=1)
    dilation = cv2.dilate(erosion, kernel, iterations=1)
    cont, hie = cv2.findContours(dilation, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    # Get blob contour coordinates
    try:
        cnt = max(cont, key=cv2.contourArea)
        x2, y2, w, h = cv2.boundingRect(cnt)
        # Draw bounding rect to pic
        height = y2+h
        cv2.rectangle(hsv, (x2, y2), (x2 + w, y2 + h), (0, 255, 0), 2)
    except:
        #print("unable to draw contours")
        pass

    contour_img = cv2.drawContours(masked_img2, cont, -1, (255, 0, 255))
    # print("cont", cont)
    try:
        max_cont = max(cont, key=cv2.contourArea)
        (x, y), r = cv2.minEnclosingCircle(max_cont)
        x = int(x)
        y = int(y)
        r = int(r)
        # print("vision color filter: ", (x, y), r)
        print(korv)
        if korv is not None and korv > 190:
            print("liiga lähedal")
            x = None;
            y = None;
            r = None
            w = None
            h = None
            x2 = None
            y2 = None
        count = 0
        if piir is not None:
            try:
                for i in range(len(piir)-y):
                    alla = piir[y+i][x]
                    if alla > 0:
                        count += 1
                    elif alla == 0:
                        count = 0
                    if count == 7:
                        #print("pall on väljaspool")
                        x = None;
                        y = None;
                        r = None
                        w = None
                        h = None
                        x2 = None
                        y2 = None
                        count = 0
                        return x, y, r, contour_img
            except:
                pass

        if r < 3 and not basket:
            raise NotImplementedError
        if r < 10 and basket:
            raise NotImplementedError
    except Exception as e:
        # print("Nothing found, returning 0, 0, 0")
        x = None;
        y = None;
        r = None
        w = None
        h = None
        x2 = None
        y2 = None

    # Only return the blob of the largest objects of the same color
    #print(x,y)
    return x, y, r, contour_img, w, h, height, x2, y2


from __future__ import absolute_import

from datetime import datetime, timedelta
import time
import uuid

from sentry.testutils import APITestCase
from django.core.urlresolvers import reverse
from sentry.testutils import SnubaTestCase

class OrganizationDiscoverTest(APITestCase, SnubaTestCase):
    def setUp(self):
        super(OrganizationDiscoverTest, self).setUp()

        now = datetime.now()

        self.login_as(user=self.user)

        self.org = self.create_organization(owner=self.user, name='foo')

        self.project = self.create_project(
            name='bar',
            organization=self.org,
        )

        events = [{
            'event_id': uuid.uuid4().hex,
            'primary_hash': uuid.uuid4().hex,
            'project_id': self.project.id,
            'message': 'message!',
            'platform': 'python',
            'datetime': now.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
            'data': {
                'received': time.mktime(now.timetuple()),
                'exception': {
                    'values': [
                        {
                            'type': 'ValidationError',
                            'value': 'Bad request',
                            'mechanism': {
                                'type': '1',
                                'value': '1',
                            },
                            'stacktrace': {
                                'frames': [
                                    {
                                        'function': '?',
                                        'filename': 'http://localhost:1337/error.js',
                                        'lineno': 29,
                                        'colno': 3,
                                        'in_app': True
                                    },
                                ]
                            },
                        }
                    ]
                },
            },

        }]

        self.snuba_insert(events)

    def test(self):
        url = reverse('sentry-api-0-organization-discover', args=[self.org.slug])
        response = self.client.post(url, {
            'projects': [self.project.id],
            'fields': ['message', 'platform'],
            'start': (datetime.now() - timedelta(seconds=10)).strftime('%Y-%m-%dT%H:%M:%S'),
            'end': datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
            'orderby': '-timestamp',
        })

        assert response.status_code == 200, response.content
        assert len(response.data['data']) == 1
        assert response.data['data'][0]['message'] == 'message!'
        assert response.data['data'][0]['platform'] == 'python'

    def test_array_join(self):
        url = reverse('sentry-api-0-organization-discover', args=[self.org.slug])
        response = self.client.post(url, {
            'projects': [self.project.id],
            'fields': ['message', 'exception_stacks.type'],
            'start': (datetime.now() - timedelta(seconds=10)).strftime('%Y-%m-%dT%H:%M:%S'),
            'end': datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
            'orderby': '-timestamp',
        })
        assert response.status_code == 200, response.content
        assert len(response.data['data']) == 1
        assert response.data['data'][0]['exception_stacks.type'] == 'ValidationError'


#https://networkx.github.io/
import networkx as nx
import matplotlib.pyplot as plt
import matplotlib.colors as mc

def gen_color(G):
    colors = []
    for n in G:
        d=nx.get_node_attributes(G,'description')[n]
        if d == "Station":
            colors.append(mc.to_rgba('red'))
        elif d == "Slow down Tag":
            colors.append(mc.to_rgba('green'))
        else:
            colors.append(mc.to_rgba('gray'))
    return colors

G=nx.DiGraph()

# adding just one node:
G.add_nodes_from([1,4,5,6], description="Station")
G.add_node(2, description="Slow down Tag")
G.add_node(3, description="Bifurcation", merge_to_node=2)

G.add_edge(1, 2, length=10 )
G.add_edge(2, 3, length=5 )
G.add_edge(3, 4, length=10 )
G.add_edge(3, 5, length=10 )
G.add_edge(4, 6, length=17 )
G.add_edge(5, 6, length=10 )

pos = {1:(0,0),2:(10,0),3:(20,0),4:(30,10),5:(30,-10), 6:(40,-10)}

#col=mc.to_rgba('red')
#nx.draw(G, with_labels=True, pos =pos)

#print(nx.shortest_path(G, source=1, target=6,weight="length"))

im = plt.imread("AGV.png")

nx.draw_networkx(G, with_labels=True, pos =pos,node_color=gen_color(G))
#implot = plt.imshow(im, extent=[1,9,-5,5])
#plt.plot([0,25],[0,15])
plt.xlim((-5,45))
plt.ylim((-15,15))
plt.savefig("Map.png") # save as png
plt.show() # display

Prev Tests/Graph tests/00 Generate map.py


import numpy as np
import pandas as pd
import sys, os, pickle, keras, warnings

np.random.seed(43)
warnings.filterwarnings('ignore')

from scipy import stats
from time import time

import tensorflow as tf
from keras.models import Model
from keras.layers import Input, Dense,Dropout
from keras import optimizers, regularizers, backend as K

from sklearn.preprocessing import StandardScaler
from matplotlib import pyplot as plt
import seaborn as sn
# %matplotlib inline

from preprocessing import get_data
import Utils

# train ,test ,indx = get_data("multiclass")
train ,test = get_data("Binary")

train_label = train.label
train = train.drop(["label"],axis=1)

Scaler = StandardScaler()
train = Scaler.fit_transform(train.values)[np.where(train_label == 1)]

xtest , ytest = Scaler.transform(test.drop(["label"],axis=1)), test.label.values

def fit_model(params,X,latent=10,BS=250,ep = 100):
    input_dim = train.shape[1]
    latent_space_size = latent
    K.clear_session()
    input_ = Input(shape = (input_dim, ))

    layer_1 = Dense(100, activation=params[0])(input_)
    layer_2 = Dense(50, activation=params[0],kernel_regularizer=regularizers.l2(0.01))(layer_1)
    layer_3 = Dense(25, activation=params[0])(layer_2)

    encoding = Dense(latent_space_size,activation=None)(layer_3)

    layer_6 = Dense(25, activation=params[0])(encoding)
    layer_7 = Dense(50, activation=params[0],kernel_regularizer=regularizers.l2(0.01))(layer_6)
    layer_8 = Dense(100, activation=params[0])(layer_7)

    decoded = Dense(input_dim,activation=None)(layer_8)

    autoencoder = Model(inputs=input_ , outputs=decoded)
    opt = optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
    autoencoder.compile(metrics=['accuracy'],loss='mean_squared_error',optimizer=opt)

    print("Training Started ....")
    autoencoder.fit(X, X,epochs=ep,validation_split=0.2,batch_size=BS,shuffle=True,verbose=0)
    print("Training Finished ....")

    return autoencoder

if os.path.exists('models/nslkdd_reg_ep100_bs250_final.pickle'):
    with open('models/model_tanh_Adam_reg_ep95_bs250.pickle', 'rb') as fid:
        model = pickle.load(fid)

else:
    model = fit_model(["tanh","Adam"],X=train,latent=12,BS=250,ep=95)
    with open('models/nslkdd_reg_ep100_bs250_final.pickle', 'wb') as f:
                pickle.dump(model, f)

losses = Utils.get_losses(model, train)

thresholds = Utils.confidence_intervals(losses,0.95)
#choose the upper interval as threshold
threshold = thresholds[1]
pred = Utils.predictAnomaly(model,xtest,threshold)
Utils.performance(ytest,pred)


import logging
import datetime
import socket
import time
import sqlite3
from sqlite3 import Error
from typing import List, Tuple, Optional, Any

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class Database:
    def __init__(self, db_name: str):
        self.db_name = db_name
        self.logger = logger  # Initialize logger
    def create_connection(self) -> Optional[sqlite3.Connection]:
        try:
            conn = sqlite3.connect(self.db_name)
            return conn
        except Error as e:
            print(e)
        return None

    def create_table(self) -> None:
        conn = self.create_connection()
        try:
            with conn:
                conn.execute('''CREATE TABLE IF NOT EXISTS data
                             (time text, N2O_ppm real, CO2_ppm real, CH4_ppm real, NH3_ppb real)''')
                conn.execute('''CREATE TABLE IF NOT EXISTS plc_history
                             (time text, ip_address text, event text)''')
                conn.execute('''CREATE TABLE IF NOT EXISTS status_history
                             (time text, ip_address text, status int)''')
        except Error as e:
            logger.error(f"Failed to create table: {e}")

    def insert_status_change(self, data: List) -> None:
        conn = self.create_connection()
        if conn is None:
            logger.error("Database connection failed for status change.")
            return
        try:
            with conn:
                conn.execute("INSERT INTO status_history (time, ip_address, status) VALUES (?, ?, ?)", data)
        except Error as e:
            logger.error(f"Failed to insert status change: {e}")
    def insert_plc_history(self, data: List) -> None:
        conn = self.create_connection()
        if conn is None:
            logger.error("Database connection failed for history write.")
            return
        try:
            with conn:
                conn.execute("INSERT INTO plc_history (time, ip_address, event) VALUES (?, ?, ?)", data)
        except Error as e:
            logger.error(f"Failed to insert history: {e}")

    def insert_data(self, data: List) -> None:
        conn = self.create_connection()
        if conn is None:
            logger.error("Database connection failed for data write.")
            return
        try:
            with conn:
                conn.execute("INSERT INTO data (time, N2O_ppm, CO2_ppm, CH4_ppm, NH3_ppb) VALUES (?, ?, ?, ?, ?)", data)
        except Error as e:
            logger.error(f"Failed to insert status change: {e}")

    def query_data(self, last_plotted_time: Optional[str] = None, lim: int = 1000) -> List[Tuple]:
        conn = self.create_connection()
        with conn:
            cur = conn.cursor()
            if last_plotted_time is None:
                cur.execute(f"SELECT * FROM data ORDER BY time DESC LIMIT {lim}")
            else:
                cur.execute("SELECT * FROM data WHERE time > ? ORDER BY time ASC", (last_plotted_time,))
            return cur.fetchall()

def main():
    HOST = '10.0.20.3'
    PORT = 51020
    db = Database('my_database.sqlite')
    db.create_table()

    while True:
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            s.connect((HOST, PORT))
            s.sendall(b"_Meas_GetConc\r")
            data = s.recv(1024).decode('utf-8')

        values = data.strip().split(';')
        values_to_plot = [float(values[2]), float(values[7]), float(values[9]), float(values[12])]
        # Insert new data into the database
        db.insert_data([str(datetime.datetime.now())] + values_to_plot)
        logger.info(f"Inserted data: {values_to_plot}")
        # Sleep for 5 seconds
        time.sleep(5)

if __name__ == "__main__":
    main()


import cv2
import numpy as np
import matplotlib.pyplot as plt

def erosion(img, se):
    img = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)[1]

    rows, columns = img.shape
    output_img = np.zeros((rows, columns), dtype=np.uint8)

    for i in range(0, rows-se.shape[0]):
        for j in range(0, columns-se.shape[1]):
            cut_img = img[i:i + se.shape[0], j:j + se.shape[1]]
            nh_img = (cut_img * se)
            output_img[i, j] = np.min(nh_img)

    return output_img

def dilation(img, se):
    img = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)[1]

    rows, columns = img.shape
    output_img = np.zeros((rows, columns), dtype=np.uint8)

    for i in range(0, rows-se.shape[0]):
        for j in range(0, columns-se.shape[1]):
            cut_img = img[i:i + se.shape[0], j:j + se.shape[1]]
            nh_img =  (cut_img * se)
            output_img[i, j] = np.max(nh_img)

    return output_img

def Task_5a():
    # Load image
    img = cv2.imread('im.png', cv2.IMREAD_GRAYSCALE)

    # Structuring element
    se = np.ones((3,3), dtype=np.uint8)

    # Erosion and Dilation
    eroded_img = erosion(img, se)
    dilated_img = dilation(img, se)

    # Display results
    plt.figure(figsize=(10, 8))

    plt.subplot(2, 2, 1)
    plt.imshow(cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)[1], cmap='gray')
    plt.title('Before Erosion')

    plt.subplot(2, 2, 2)
    plt.imshow(eroded_img, cmap='gray')
    plt.title('After Erosion')

    plt.subplot(2, 2, 3)
    plt.imshow(dilated_img, cmap='gray')
    plt.title('After Dilation')

def Task_5b():
    # Load image
    img = cv2.imread('fingerprint.png', cv2.IMREAD_GRAYSCALE)

    # Structuring element
    se = np.ones((5,5), dtype=np.uint8)

    # cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)[1]

    # Erosion and Dilation
    erosion_img = erosion(img, se)
    opening = dilation(erosion_img, se)

    dilation_img = dilation(opening, se)
    closing = erosion(dilation_img, se)

    # Display results
    plt.figure(figsize=(10, 8))

    plt.subplot(2, 2, 1)
    plt.imshow(cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)[1], cmap='gray')
    plt.title('Before Erosion')

    plt.subplot(2, 2, 2)
    plt.imshow(opening, cmap='gray')
    plt.title('After Opening')

    plt.subplot(2, 2, 3)
    plt.imshow(closing, cmap='gray')
    plt.title('After Closing')

def Task_5c():
    # Load image
    img = cv2.imread('boundary.png', cv2.IMREAD_GRAYSCALE)

    # Structuring element
    se = np.ones((10,10), dtype=np.uint8)

    # Erosion and Dilation
    erosion_img = erosion(img, se)
    boundary = img-erosion_img

    # Display results
    plt.figure(figsize=(10, 8))

    plt.subplot(2, 1, 1)
    plt.imshow(cv2.threshold(img, 127, 255, cv2.THRESH_BINARY)[1], cmap='gray')
    plt.title('Original image')

    plt.subplot(2, 1, 2)
    plt.imshow(boundary, cmap='gray')
    plt.title('The boundary')

def main():
    # Task_5a()
    # Task_5b()
    Task_5c()
    plt.tight_layout()
    plt.show()

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
import scipy.interpolate
from tqdm import tqdm

def gaussian_peak(x, amplitude, loc, scale):
    return amplitude * np.exp((-(x - loc)**2) / (2 * scale**2))

def asymmetrical_gaussian_peak(x, amplitude, loc, s0, s1, s2=0.0, eps=1e-7):
    # assymetrical_gaussian_peak(..., s1=0) is the same as gaussian_peak(...)
    # add additional asymmetries by adding to the denominator inside np.exp:
    #   s0 + s1*(x-loc) + s2*(x-loc)**2 ..., sN*(x-loc)**N

    peak = np.exp(-1/2*((x - loc)/(eps + s0 + s1*(x - loc) + s2*(x - loc)**2))**2)
    peak *= amplitude

    if s1 > 0:
        out = loc - 4 * s0
        idx = np.argmin(np.abs(x - out))
        peak[:idx] = 0.0
    else:
        out = loc + 4 * s0
        idx = np.argmin(np.abs(x - out))
        peak[idx:] = 0.0

    return peak

def apply_white_noise(chromatogram, apices, signal_to_noise_ratio):
    # obtain a stddev that approximately results in noise levels close to SNR
    stddev = apices.mean() / signal_to_noise_ratio / 4
    noise = np.random.normal(0, stddev, len(chromatogram))
    return chromatogram + noise

def apply_pink_noise(chromatogram, apices, signal_to_noise_ratio, num_sources=8):
    nrows = len(chromatogram)
    ncols = num_sources
    noise = np.full((nrows, ncols), np.nan)
    noise[0, :] = np.random.random(ncols)
    noise[:, 0] = np.random.random(nrows)

    cols = np.random.geometric(0.5, nrows)
    cols[cols >= ncols] = 0
    rows = np.random.randint(nrows, size=nrows)
    noise[rows, cols] = np.random.random(nrows)

    noise = pd.DataFrame(noise)
    noise.fillna(method='ffill', axis=0, inplace=True)
    noise = noise.sum(axis=1).to_numpy()
    noise = (noise - noise.mean())

    # make noise approximately match SNR
    noise = (apices.mean() / signal_to_noise_ratio) * noise / 2
    return chromatogram + noise

def apply_baseline_drift(chromatogram, resolution, multiplier_range):

    def sigmoid(x, a, b, multiplier):
        return 1 / (1 + np.exp( - (x * a + b) )) * multiplier

    x = np.linspace(-1, 1, resolution)
    baseline_drift = np.zeros(resolution, dtype='float32')
    n = 10
    for _ in range(n):
        multiplier = np.random.uniform(*multiplier_range)
        a = np.random.uniform(-20, 20)
        b = np.random.uniform(-20, 20)
        baseline_drift += sigmoid(x, a, b, multiplier) / n
    return chromatogram + baseline_drift

def random_truncated_normal(loc, scale, minimum, maximum, size):
    values = np.zeros(size)
    logical_test = values.copy().astype(bool)
    while not np.all(logical_test):
        logical_test = np.logical_and((minimum < values), (values < maximum))
        values = np.where(logical_test, values, scale * np.random.randn(size) + loc)
    return values

def random_logarithmic_uniform(minimum, maximum, size=None):
    return 10 ** np.random.uniform(np.log10(minimum), np.log10(maximum), size=size)

def random_logarithmic_randint(minimum, maximum, size=None):
    return np.round(random_logarithmic_uniform(minimum, maximum, size)).astype(int)

class Simulator:

    def __init__(
        self,
        remove_collision_fn=None,
        resolution=8192,
        num_peaks_range=(10, 100),
        snr_range=(3.0, 300.0),
        amplitude_range=(10, 100),
        loc_range=(0.05, 0.95),
        scale_range=(0.001, 0.002),
        asymmetry_range=(0.00, 0.30),
        baseline_drift_magnitude=(-5, 5), # Not used
        white_noise_prob=1.0,
    ):
        self.remove_collision_fn = remove_collision_fn
        self.resolution = resolution
        self.num_peaks_range = num_peaks_range
        self.snr_range = snr_range
        self.amplitude_range = amplitude_range
        self.loc_range = loc_range
        self.scale_range = scale_range
        self.asymmetry_range = asymmetry_range
        self.baseline_drift_magnitude = baseline_drift_magnitude
        self.white_noise_prob = white_noise_prob

    def sample_batch(self, indices, verbose=0):
        if verbose:
            indices = tqdm(indices)
        for i in indices:
            yield self._generate_example(i)

    def _get_random_scales(self, locs, n_peaks):
        scale_loc = np.random.uniform(*self.scale_range)
        scales = random_truncated_normal(
            scale_loc,                      # mean
            scale_loc/10,                   # std
            scale_loc - scale_loc/10 * 3,   # min (mean - 3std)
            scale_loc + scale_loc/10 * 3,   # max (mean + 3std)
            n_peaks                         # size
        )
        a = np.random.uniform(1, 3)
        b = 0.0 # np.random.uniform(0, 4)
        scales *= 1 + a*locs +  b*locs**2
        return scales, a, b

    def _get_random_amplitudes(self, locs, a, b, n_peaks):
        amplitudes = np.random.uniform(*self.amplitude_range, size=n_peaks)
        amplitudes /= (1 + a*locs + b*locs)
        return amplitudes

    def _get_random_locs(self, n_peaks):
        locs = np.random.uniform(*self.loc_range, size=n_peaks)
        return np.sort(locs)

    def _get_random_asymmetries(self, locs, n_peaks):
        asymmetry_loc = np.random.uniform(*self.asymmetry_range)
        s1 = random_truncated_normal(
            asymmetry_loc,                          # mean
            asymmetry_loc/10,                       # std
            asymmetry_loc - asymmetry_loc/10 * 3,   # min (mean - 3std)
            asymmetry_loc + asymmetry_loc/10 * 3,   # max (mean + 3std)
            n_peaks                                 # size
        )
        s1 *= (1 - locs)
        s2 = -np.random.uniform(np.abs(s1/2), np.abs(s1*2))
        return s1, s2

    def _generate_example(self, random_seed):

        np.random.seed(random_seed)

        n_peaks = random_logarithmic_randint(*self.num_peaks_range)

        # 1. Generate (noise-free) chromatogram
        # a. Obtain peak parameters
        locs = self._get_random_locs(n_peaks)
        scales, a, b = self._get_random_scales(locs, n_peaks)
        ampls = self._get_random_amplitudes(locs, a, b, n_peaks)
        s1s, s2s = self._get_random_asymmetries(locs, n_peaks)

        # b. Remove collisions:
        #    If two peaks fall inside the same window, remove one of the peaks
        #    Otherwise we will train the model to predict one peak (for the
        #    given heavily-overlapping peak) even though there are two peaks
        if self.remove_collision_fn is not None:
            locs, scales, s1s, s2s, ampls = self.remove_collision_fn(
                locs, scales, s1s, s2s, ampls
            )

        # 3. Generate (noise-free) chromatogram from peaks (and obtain areas)
        x = np.linspace(0, 1, self.resolution)
        areas = np.zeros([0], dtype='float32')
        chromatogram = np.zeros_like(x)
        for i, (ampl, loc, scale, s1, s2) in enumerate(zip(ampls, locs, scales, s1s, s2s)):
            peak = asymmetrical_gaussian_peak(x, ampl, loc, scale, s1, s2)
            areas = np.concatenate([areas, [np.trapz(peak, dx=x[1]-x[0])]])
            chromatogram += peak

        # 2. Apply noise to (noise-free) chromatogram
        # a. Obtain signal-to-noise ratio (levels of noise)
        snr = random_logarithmic_uniform(*self.snr_range)
        # b. Apply white noise or pink noise to chromatogram
        if np.random.random() <= self.white_noise_prob:
            chromatogram = apply_white_noise(chromatogram, ampls, snr)
        else:
            chromatogram = apply_pink_noise(chromatogram, ampls, snr)

        # distort baseline a bit
        chromatogram += np.random.randn() * 2

        # c. Apply baseline drift to chromatogram
        #chromatogram = apply_baseline_drift(
        #   chromatogram, self.resolution, self.baseline_drift_magnitude)

        return {
            'chromatogram': chromatogram,
            'loc': locs,
            'scale': scales,
            'amplitude': ampls,
            's1': s1s,
            's2': s2s,
            'area': areas,
        }


import os
import time

import pymysql
import requests

import config

sess = config.sess

def grab(url, filename):
    print(url, filename)
    path = os.path.join(config.res, filename)
    if os.path.exists(path):
        print(path, 'already had')
        return
    with open(path, 'wb') as f:
        resp = sess.get(url)
        f.write(resp.content)
    time.sleep(0.2)

def scrawl():
    conn = pymysql.connect(**config.database)
    cur = conn.cursor(pymysql.cursors.DictCursor)
    rows = cur.execute("select * from files")
    for i in range(rows):
        it = cur.fetchone()
        grab(it['url'], it['path'])
    cur.close()
    conn.close()

scrawl()

爬取审批系统/全部/4-下载二进制.py


#!/usr/bin/env python3

from absl import app
from absl import flags
from absl import logging
from base64 import b64encode
from numpy import arange
from params import loadParams
from string import Template

import os
import sys

"""
Ranges are iterated over to generate the appropriate flags files. All unique
combinations will be generated. See `flagfile`, `flagfile.template`, and
`params.py` for flags that are available. Edit this block when starting a new
swarm run.
"""
RANGES = {
    "maxTime": range(100, 200, 50),
    "tau": arange(.1, 1, .3),
    "popCount": range(10, 50, 10),
}

sys.setrecursionlimit(1000)
FLAGS = flags.FLAGS

flags.DEFINE_string("flags_template", "flagfile.template", "Template file for generating flags.")
flags.DEFINE_string("flags_directory", "flags", "Template file for generating flags.")

def load_template():
    with open(FLAGS.flags_template, "r") as flags_template:
        return Template(flags_template.read())

def build_table_matrix(ranges, accumulated = {}):
    if not ranges:
        yield accumulated
        return

    for key in ranges:
        rang = ranges[key]
        for value in rang:
            next_accumulated = {**accumulated}
            next_accumulated[key] = value
            next_ranges = {**ranges}
            del next_ranges[key]
            yield from build_table_matrix(next_ranges, accumulated=next_accumulated)
        return

def write_template(template, build_id, default_params, matrix_params):
    params = {
        **default_params,
        **matrix_params,
    } # Merge param dictionaries

    params["build_id"] = build_id

    try:
        flags_file_contents = template.substitute(params)
        path = os.path.join(FLAGS.flags_directory, "flags.%s" % build_id)
        logging.info("Writing flags file %s" % path)
        with open(path, "w") as flags_file:
            flags_file.write(flags_file_contents)
    except Exception as err:
        logging.warn("Failed to write template for %s" % build_id)
        logging.info(err)

def encode_params(matrix_params):
    encoded_params = ''
    for key in matrix_params:
        encoded_params = "%s__%s_%s" % (encoded_params, key, matrix_params[key])
    return encoded_params[2:] # Strip leading __

def main(argv):
    if not os.path.exists(FLAGS.flags_directory):
        os.makedirs(FLAGS.flags_directory)

    template = load_template()

    default_params = loadParams()

    for matrix_params in build_table_matrix(RANGES):
        build_id = encode_params(matrix_params)
        write_template(template, build_id, default_params, matrix_params)

if __name__ == '__main__':
    app.run(main)


# Databricks notebook source
# MAGIC %run "../includes/configuration"

# COMMAND ----------

# MAGIC %run "../includes/common_functions"

# COMMAND ----------

dbutils.widgets.text("p_data_source", "")

# COMMAND ----------

#Hive DDL style schema
constructors_schema = "constructorId INTEGER,constructorRef STRING,name STRING,nationality STRING,url STRING"
df_constructors = spark.read\
.schema(constructors_schema)\
.json(f"{raw_folder_path}/constructors.json")

# COMMAND ----------

from pyspark.sql.functions import col
#df_constructors_drop= df_constructors.drop("url")
#df_constructors_drop= df_constructors.drop(df_constructors["url"])
df_constructors_drop= df_constructors.drop(col("url"))

# COMMAND ----------

from pyspark.sql.functions import current_timestamp
df_constructors_final= df_constructors_drop.withColumnRenamed("constructorId","constructor_id")\
.withColumnRenamed("constructorRef","constructor_ref")\
.withColumnRenamed("name","constructor_name")\
.withColumnRenamed("nationality","constructor_nationality")\
.withColumn("ingestion_date",current_timestamp())

# COMMAND ----------

display(df_constructors_final)

# COMMAND ----------

df_constructors_final.write.mode("overwrite").parquet(f"{processed_folder_path}/constructors")

# COMMAND ----------

# MAGIC %fs
# MAGIC ls /mnt/formula1dlv4/processed/constructors

# COMMAND ----------

dbutils.notebook.exit("Success")