"""
The File Upload application is designed to allow the management of files
associated with submissions. This can be used to upload new files, manage
URLs of existing files, and delete files.
"""

from collections import namedtuple
import json
import logging

from django.db import IntegrityError
from django.utils.functional import cached_property

from openassessment.assessment.models.base import SharedFileUpload
from openassessment.fileupload.exceptions import FileUploadError

from . import backends

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

KEY_SEPARATOR = '/'

def get_upload_url(key, content_type):
    """
    Returns a url (absolute or relative, depending on the endpoint) which can be used to upload a file to.
    """
    return backends.get_backend().get_upload_url(key, content_type)

def get_download_url(key):
    """
    Returns the url at which the file that corresponds to the key can be downloaded.
    """
    url = backends.get_backend().get_download_url(key)
    if not url:
        logger.warning('FileUploadError: Could not retrieve URL for key %s', key)
    return url

def remove_file(key):
    """
    Remove file from the storage
    """
    return backends.get_backend().remove_file(key)

def get_student_file_key(student_item_dict, index=0):
    """
    Args:
        student_item_dict: A dictionary containing keys ('student_id', 'course_id', 'item_id').
        index (int, optional): The index of a file.
    """
    key_template = KEY_SEPARATOR.join(('{student_id}', '{course_id}', '{item_id}'))
    index = int(index)
    if index > 0:
        key_template += KEY_SEPARATOR + '{index}'
    return key_template.format(index=index, **student_item_dict)

def can_delete_file(current_user_id, teams_enabled, key, team_id=None, shared_file=None):
    """
    A user is allowed to delete any file they own if this is not a team-enabled response.
    If the response is team-enabled, a user, who is a member of a team,
    is allowed to delete a file they own as long as they are still
    a member of the team with which the file has been shared.

    params:
      current_user_id (string): The anonymous id of the current user in an ORA block.
      teams_enabled (boolean): Indicates if teams are enabled for an ORA block.
      key (string): The key of the file to check if we can delete.
      team_id (string): The id of the team of the user who may be able to delete the file.
      shared_file (SharedFileUpload): Optional. A SharedFileUpload object corresponding to the given
      key.  It's useful to pass this in if you've already fetched all of the SharedFileUpload records
      for a given item/team.

    raises:
        SharedFileUpload.DoesNotExist If teams are enabled, a team_id is provided,
        and no SharedFileUpload corresponding to the file key exists.
    returns:
        Boolean indicating if the file with the given key can be deleted by the current user.
    """
    if not teams_enabled:
        return True

    if not shared_file:
        try:
            shared_file = SharedFileUpload.by_key(key)
        except SharedFileUpload.DoesNotExist:
            logger.info('While checking ORA file-deletion ability, could not find file with key: %s', key)
            return True

    if shared_file.owner_id != current_user_id:
        return False

    if shared_file.team_id != team_id:
        return False

    # If we've made it this far, the current user has a team, and it's the same
    # team that the file is shared with, so let them (as the file's owner) delete it.
    return True

def delete_shared_files_for_team(course_id, item_id, team_id):
    """
    Delete shared files for a team for this block
    """
    uploads = SharedFileUpload.by_team_course_item(team_id, course_id, item_id)

    for upload in uploads:
        remove_file(upload.file_key)
        upload.delete()

def _safe_load_json_list(field, log_error=False):
    """
    Tries to load JSON-ified string,
    returns an empty list if we try to load some non-JSON-encoded string.
    """
    try:
        return json.loads(field)
    except ValueError:
        if log_error:
            logger.exception(
                "URLWorkaround: Safe Load failed for data field:%s with type:%s", field, type(field)
            )
        return []

class FileUpload:
    """
    A layer of abstraction over the various components of file
    data stored as ORA XBlock user-scoped fields.
    """
    def __init__(self, name=None, description=None, size=None, index=0, descriptionless=False, **student_item_dict):
        """
        Args:
            name (str): The name of a file.
            description (str): The student-provided description of a file.
            size (int): The size, in bytes, of a file.
            index (int): The position of a file relative to all other uploaded files for a given user.
            descriptionless (bool): True if this file exists but has no description, name, or size.
                                    False (default) otherwise.
            student_item_dict (dict): Contains the student_id, course_id, and item_id, i.e. the "student item"
                                      triple associated with this file upload.
        """
        self.name = name
        self.description = description
        self.size = size
        self.index = index
        self.student_id = student_item_dict.get('student_id')
        self.course_id = student_item_dict.get('course_id')
        self.item_id = student_item_dict.get('item_id')
        self.descriptionless = descriptionless

    @property
    def exists(self):
        return (self.description is not None) or self.descriptionless

    @property
    def download_url(self):
        """
        Returns the url at which the file that corresponds to the key
        can be downloaded if exists.
        """
        if self.exists:
            try:
                return get_download_url(self.key)
            except FileUploadError as exc:
                logger.exception(
                    'FileUploadError: URL retrieval failed for key %s with error %s',
                    self.key,
                    exc,
                    exc_info=True,
                )
                return ''
        return None

    @property
    def key(self):
        """
        Simple utility method to generate a common file upload key based on
        the student item.

        Returns:
            A string representation of the key.
        """
        student_item_dict = {
            'student_id': self.student_id,
            'course_id': self.course_id,
            'item_id': self.item_id,
        }
        return get_student_file_key(student_item_dict, index=self.index)

    def _to_dict(self):
        """
        Returns:
            A dictionary representation of the FileUpload.
        """
        attrs = ('description', 'name', 'size', 'course_id', 'student_id', 'item_id', 'descriptionless')
        return {
            key: getattr(self, key, None) for key in attrs
        }

    def __eq__(self, other):
        """
        Returns:
            True if self's dict representation equals other's dict representation,
            False otherwise.
        """
        return self._to_dict() == other._to_dict()  # pylint: disable=protected-access

    def __hash__(self):
        """
        Returns a hash of the FileUpload's dict representation
        """
        return hash(self._to_dict())

FileDescriptor = namedtuple('FileDescriptor', ['download_url', 'description', 'name', 'size', 'show_delete_button'])
TeamFileDescriptor = namedtuple('TeamFileDescriptor', ['download_url', 'description', 'name', 'size', 'uploaded_by'])

class FileUploadManager:
    """
    Manages the CRUD operations of file uploads
    that take place in the context of an OpenAssessmentBlock.

    e.g. inside an XBlock:
    self.upload_manager = FileUploadManager(self)
    for file_upload in self.upload_manager.get_uploads():
        log.info(file_upload.download_url)

    new_uploads = [
        {'name': 'file-1.jpg', 'description': 'File 1', 'size': 1024},
        {'name': 'file-2.jpg', 'description': 'File 2', 'size': 2048},
    ]
    self.upload_manager.append_uploads(*new_uploads)

    self.upload_manager.delete_upload(index=0)
    """
    def __init__(self, openassessment_xblock):
        self.block = openassessment_xblock
        self.shared_uploads_for_team_by_key_cache = {}

    @cached_property
    def student_item_dict(self):
        """ Returns a dict containing 'student_id', 'course_id', and 'item_id'. """
        return self.block.get_student_item_dict()

    def get_uploads(self, team_id=None, include_deleted=False):
        """
        Returns:
            A list of FileUpload objects associated with an instance of an Open Assessment Block.
            This will include FileUpload objects corresponding to existing files.
            It does **not** include entries for files that have been deleted.
        """
        descriptions, names, sizes = self._get_metadata_from_block()
        user_uploads = self._file_uploads_from_list_fields(descriptions, names, sizes, include_deleted=include_deleted)

        if self.block.is_team_assignment():
            return self._uploads_shared_with_team_by_current_user(user_uploads, team_id)

        return user_uploads

    def get_team_uploads(self, team_id=None):
        if self.block.is_team_assignment():
            return self._uploads_owned_by_teammates(team_id)
        return []

    def _uploads_shared_with_team_by_current_user(self, user_uploads, team_id):
        """
        Helper function that filters a given list of ``user_uploads``
        down to those ``FileUploads`` that are owned by the current user
        **and** the given team
        """
        jointly_owned_uploads = []

        for upload in user_uploads:
            shared_upload = self.shared_uploads_for_student_by_key.get(upload.key)
            if shared_upload and (shared_upload.team_id == team_id):
                jointly_owned_uploads.append(upload)
            elif not upload.exists:
                # we should return entries for deleted files, here,
                # to uphold the invariant around file indices.
                jointly_owned_uploads.append(upload)

        return jointly_owned_uploads

    def _uploads_owned_by_teammates(self, team_id):
        """
        Returns a list of FileUpload objects owned by other members of the given team.
        Does not include FileUploads of the current user.
        """
        shared_uploads_from_other_users = sorted(
            [
                shared_upload
                for shared_upload in self.shared_uploads_for_team_by_key(team_id).values()
                if shared_upload.owner_id != self.student_item_dict['student_id']
            ],
            key=lambda upload: upload.file_key,
        )

        return [
            FileUpload(
                name=shared_upload.name,
                description=shared_upload.description,
                size=shared_upload.size,
                student_id=shared_upload.owner_id,
                course_id=shared_upload.course_id,
                item_id=shared_upload.item_id,
                index=shared_upload.index,
            ) for shared_upload in shared_uploads_from_other_users
        ]

    def file_descriptors(self, team_id=None, include_deleted=False):
        """
        Used in the response template context to provide file information
        (file URL, description, name, show_delete_button) for each uploaded
        file in this block.

        If self.block is team-enabled, this will return only entries for files
        that have been shared with the specified team
        """

        descriptors = []

        for upload in self.get_uploads(team_id=team_id, include_deleted=include_deleted):
            show_delete_button = bool(upload.exists)

            if upload.exists and self.block.is_team_assignment():
                shared_upload = self.shared_uploads_for_team_by_key(team_id)[upload.key]
                show_delete_button = can_delete_file(
                    self.student_item_dict['student_id'],
                    self.block.is_team_assignment(),
                    upload.key,
                    team_id=team_id,
                    shared_file=shared_upload,
                )

            descriptors.append(FileDescriptor(
                download_url=upload.download_url,
                description=upload.description,
                name=upload.name,
                size=upload.size,
                show_delete_button=show_delete_button,
            )._asdict())

        return descriptors

    def team_file_descriptors(self, team_id=None):
        """
        Returns the list of TeamFileDescriptors owned by other team members
        shown to a user when self.block is a team assignment.
        """
        return [
            TeamFileDescriptor(
                download_url=upload.download_url,
                description=upload.description,
                name=upload.name,
                size=upload.size,
                uploaded_by=self.block.get_username(upload.student_id)
            )._asdict()
            for upload in self.get_team_uploads(team_id=team_id)
        ]

    @cached_property
    def shared_uploads_for_student_by_key(self):
        """
        Returns **and caches** all of the SharedFileUpload records
        for this student/course/item.
        """
        shared_uploads = SharedFileUpload.by_student_course_item(**self.student_item_dict)
        return {shared_upload.file_key: shared_upload for shared_upload in shared_uploads}

    def shared_uploads_for_team_by_key(self, team_id):
        """
        Returns **and caches** all of the SharedFileUpload records
        for this student/course/item and team.

        Realistically, only one team_id will ever be requested, but this is a simple enough pattern
        """
        if team_id not in self.shared_uploads_for_team_by_key_cache:
            shared_uploads = SharedFileUpload.by_team_course_item(
                team_id=team_id,
                course_id=self.student_item_dict['course_id'],
                item_id=self.student_item_dict['item_id'],
            )
            shared_uploads_for_team_by_key = {
                shared_upload.file_key: shared_upload for shared_upload in shared_uploads
            }
            self.shared_uploads_for_team_by_key_cache[team_id] = shared_uploads_for_team_by_key
        return self.shared_uploads_for_team_by_key_cache[team_id]

    def invalidate_cached_shared_file_dicts(self):
        """
        Invalidates SharedFileUpload records that we have cached.
        """
        if hasattr(self, 'shared_uploads_for_student_by_key'):
            del self.shared_uploads_for_student_by_key

        self.shared_uploads_for_team_by_key_cache = {}

    def append_uploads(self, *new_uploads):
        """
        Given lists of new file metadata, write the new metadata to our stored file metadata fields

        Args:
            descriptions_to_add: a list of file descriptions
            names_to_add: a list of file names
            sizes_to_add: a list of file sizes as integers

        Returns: newly updated file metadata fields
        """
        required_keys = ('description', 'name', 'size')
        try:
            (
                descriptions_to_add,
                names_to_add,
                sizes_to_add,
            ) = self._dicts_to_key_lists(new_uploads, required_keys)
        except FileUploadError as exc:
            logging.exception(
                "FileUploadError: Metadata save for %s failed with error %s",
                exc,
                new_uploads
            )
            raise

        existing_file_descriptions, existing_file_names, existing_file_sizes = self._get_metadata_from_block()

        new_descriptions = existing_file_descriptions + descriptions_to_add
        self._set_file_descriptions(new_descriptions)

        new_names = existing_file_names + names_to_add
        self._set_file_names(new_names)

        new_sizes = existing_file_sizes + sizes_to_add
        self._set_file_sizes(new_sizes)

        new_file_uploads = self._file_uploads_from_list_fields(new_descriptions, new_names, new_sizes)

        if self.block.is_team_assignment() and self.block.has_team():
            existing_file_upload_key_set = {
                fileupload.key for fileupload in
                self._file_uploads_from_list_fields(
                    existing_file_descriptions,
                    existing_file_names,
                    existing_file_sizes
                )
            }

            for new_file_upload in new_file_uploads:
                if new_file_upload.key not in existing_file_upload_key_set:
                    self.create_shared_upload(new_file_upload)

        self.invalidate_cached_shared_file_dicts()
        return new_file_uploads

    def create_shared_upload(self, fileupload):
        try:
            SharedFileUpload.objects.create(
                team_id=self.block.team.team_id,
                owner_id=fileupload.student_id,
                course_id=fileupload.course_id,
                item_id=fileupload.item_id,
                file_key=fileupload.key,
                description=fileupload.description,
                size=fileupload.size,
                name=fileupload.name,
            )
        except IntegrityError as e:
            logger.error("Unable to create shared upload. %s", str(e))
            raise e

    def get_file_key(self, index):
        return get_student_file_key(self.student_item_dict, index)

    def delete_upload(self, index):
        """
        Given a file index to remove, null out its metadata in our stored file metadata fields.
        This will also delete any ``SharedFileUpload`` records associated with the file's key
        (if the file has been shared with a team).

        Args:
            index (integer): file index to remove
        """
        file_key = self.get_file_key(index)
        remove_file(file_key)

        stored_file_descriptions, stored_file_names, stored_file_sizes = self._get_metadata_from_block()

        stored_file_descriptions[index] = None
        self._set_file_descriptions(stored_file_descriptions)

        stored_file_names[index] = None
        self._set_file_names(stored_file_names)

        stored_file_sizes[index] = 0
        self._set_file_sizes(stored_file_sizes)

        if self.block.is_team_assignment():
            try:
                SharedFileUpload.by_key(file_key).delete()
            except SharedFileUpload.DoesNotExist:
                logger.warning('Could not find SharedFileUpload to delete: %s', file_key)

        self.invalidate_cached_shared_file_dicts()

    def _get_metadata_from_block(self):
        descriptions = self._get_file_descriptions()
        names = self._get_file_names(descriptions)
        sizes = self._get_file_sizes(descriptions)
        return descriptions, names, sizes

    def _file_uploads_from_list_fields(self, descriptions, names, sizes, include_deleted=False):
        """
        Given file upload data as list fields, return a list of FileUploads constructed from those fields
        """
        file_fields_by_key = {
            'name': names,
            'description': descriptions,
            'size': sizes,
        }

        if not descriptions:
            return self._descriptionless_uploads()

        file_uploads = []
        for index in range(len(descriptions)):
            file_upload_kwargs = {
                key: file_field[index] for key, file_field in file_fields_by_key.items()
            }

            file_upload_kwargs.update(self.student_item_dict)
            file_upload_kwargs['index'] = index

            file_upload = FileUpload(**file_upload_kwargs)
            if include_deleted or file_upload.exists:
                file_uploads.append(file_upload)

        return file_uploads

    def _descriptionless_uploads(self):
        """
        This is the old behavior, required for a corner case and should be eventually removed.
        https://github.com/openedx/edx-ora2/pull/1275 closed a loophole that allowed files
        to be uploaded without descriptions. In that case, an ORA block's saved_file_descriptions would be
        an empty list, but a key corresponding to their student item information would exist (and thus,
        so would a valid download URL).
        If there are users in that state who have files uploaded
        with no descriptions but have not yet submitted, they will fall here.
        """
        file_uploads = []

        for index in range(self.block.MAX_FILES_COUNT):
            file_key = get_student_file_key(self.student_item_dict, index)

            download_url = ''
            try:
                download_url = get_download_url(file_key)
            except FileUploadError:
                pass

            if download_url:
                file_uploads.append(FileUpload(
                    name='', description='', size=0, index=index, descriptionless=True, **self.student_item_dict
                ))
            else:
                break

        return file_uploads

    def _get_file_descriptions(self):
        """ Returns a list of file descriptions associated with this manager's OA block. """
        return _safe_load_json_list(self.block.saved_files_descriptions)

    def _set_file_descriptions(self, file_description_list):
        """ Updates the file descriptions associated with this manager's OA block. """
        self.block.saved_files_descriptions = json.dumps(file_description_list)

    def _get_file_names(self, descriptions=None):
        """ Returns a list of file names associated with this manager's OA block. """
        descriptions = descriptions or self._get_file_descriptions()
        file_names = _safe_load_json_list(self.block.saved_files_names)
        if len(file_names) != len(descriptions):
            file_names = [None for _ in range(len(descriptions))]
            self._set_file_names(file_names)
        return file_names

    def _set_file_names(self, file_name_list):
        """ Updates the list of file names associated with this manager's OA block. """
        self.block.saved_files_names = json.dumps(file_name_list)

    def _get_file_sizes(self, descriptions=None):
        """ Returns a list of file sizes associated with this manager's OA block. """
        descriptions = descriptions or self._get_file_descriptions()
        file_sizes = _safe_load_json_list(self.block.saved_files_sizes)
        if len(file_sizes) != len(descriptions):
            file_sizes = [None for _ in range(len(descriptions))]
            self._set_file_sizes(file_sizes)
        return file_sizes

    def _set_file_sizes(self, file_size_list):
        self.block.saved_files_sizes = json.dumps(file_size_list)

    def _dicts_to_key_lists(self, dicts, required_keys):
        """
        Transposes a list of dictionaries with certain required keys
        to a tuple of lists, each containing the values of the required keys.
        """
        result = {
            key: [] for key in required_keys
        }

        for _dict in dicts:
            for key in required_keys:
                if key not in _dict:
                    raise FileUploadError(f'Missing required key {key} in {_dict}')
                result[key].append(_dict[key])

        return tuple(result[key] for key in required_keys)


""" Boyuan Chen:
This file quickly trims a data file to your desired data range. You can use it either on raw data file, or processed
data file.
"""

import pandas as pd
import argparse

parser = argparse.ArgumentParser(description='Semantic Scan args parser')

parser.add_argument('--input_file', type=str, default="../data/ED_2016/ED_2016_full.txt")
parser.add_argument('--output_file', type=str, default="../data/ED_2016/ED_2016_comma.txt")
parser.add_argument('--start_time', type=str, default="12/01/2015 00:00")
parser.add_argument('--end_time', type=str, default="12/27/2016 23:59")

FLAGS = parser.parse_args()

def compress_by_date(input_file, output_file, start_date, end_date):
    chunk_idx = 0
    chunksize = 10000
    reader = pd.read_csv(input_file, sep='\t', iterator=True, chunksize=chunksize, encoding='latin1', low_memory=False)
    for cc_data in reader:
        chunk_idx += 1
        print(f"Chunk {chunk_idx} starts")
        cc_data['date'] = cc_data['date'].astype(str)
        cc_data['time'] = cc_data['time'].astype(str)
        cc_data['time'] = cc_data['time'].str.replace('24:00', '23:59')
        cc_data = cc_data.join(
            cc_data['date'].str.split('/', expand=True).rename(index=int, columns={0: 'month', 1: 'day', 2: 'year'}))
        cc_data = cc_data.join(
            cc_data['time'].str.split(':', expand=True).rename(index=int, columns={0: 'hour', 1: 'minute'}))
        # Drop the rows with invalid date and time
        cc_data.dropna(subset=['day', 'month', 'year', 'hour', 'minute'], how='any', inplace=True)

        # Reconstruct valid full date
        cc_data['year'] = '20' + cc_data['year'].astype(str)
        cc_data['month'] = cc_data['month'].astype(str).str.zfill(2)
        cc_data['day'] = cc_data['day'].astype(str).str.zfill(2)
        cc_data['hour'] = cc_data['hour'].astype(str).str.zfill(2)
        cc_data['full_date'] = pd.to_datetime(
            cc_data['year'] + '/' + cc_data['month'] + '/' + cc_data['day'] + ' ' + cc_data['hour'] + ':' + cc_data[
                'minute'], format='%Y/%m/%d %H:%M')

        cc_data = cc_data[(cc_data['full_date'] >= start_date) & (cc_data['full_date'] <= end_date)]
        cc_data = cc_data.drop(columns=['month','day','year','hour','minute','full_date'])
        cc_data.to_csv(output_file, sep=",", index=False, header=(chunk_idx==1), mode='a', encoding='utf-8')

def main(args):
    pd.set_option('expand_frame_repr', False)
    compress_by_date(args.input_file, args.output_file, args.start_time, args.end_time)

if __name__=="__main__":
    main(FLAGS)

MUSES Open Source Software Code/preprocessing/data_compressor.py


from .models import Lista
from rest_framework import serializers
from .models import Item

class ItemSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Item
        fields = ['name', 'list', 'comprado', 'url']

class ListaSerializer(serializers.HyperlinkedModelSerializer):
    item_set = ItemSerializer(many=True)

    class Meta:
        model = Lista
        fields = ['owner', 'name', 'item_set' , 'url']


# 로그 남기기
import logging
from datetime import datetime

# logging.basicConfig(level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s")
#
# logging.debug("아 이거 누가 짠거야~~")
# logging.info("자동화 수행 준비")
# logging.warning("이 스크립트는 오래 되었습니다.")
# logging.error("에러 발생")
# logging.critical("치명적인 문제")

# 터미널과 파일에 함께 로그 남기기
logFormatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger()

# 로그 레벨 설정
logger.setLevel(logging.DEBUG)

# 스트림
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(logFormatter)
logger.addHandler(streamHandler)    # 로거에 스트림핸들러를 추가

# 파일
filename = datetime.now().strftime("mylogfile_%Y%m%d%H%M%S.log")    # mylog_20201010141011.log
fileHandler = logging.FileHandler(filename, encoding="utf-8")
fileHandler.setFormatter(logFormatter)
logger.addHandler(fileHandler)      # 로거에 파일 핸들러를 추가

logger.debug("로그를 남겨보는 테스트2")   # 로거는 스트림, 파일 핸들러로 출력


from statistics import mode
from unittest import result
from flask import Flask, render_template, request
import pickle
import numpy as np
import sys

app = Flask(__name__)

model = pickle.load(open('model.tflite', 'rb'))

@app.route("/")
def home():
    return render_template("index.html")

@app.route("/predict", methods=['POST', 'GET'])
def predict():
    float_features = [float(x) for x in request.form.values()]

    #print('Hello world!' + float_features, file=sys.stderr)
    #input = [[0, 0, 0, 0, 700, 0, 3809677.791, 1796366, 0, 104]]
    final = [np.array(float_features)]
    output = model.predict(final)

    print(output[0], file=sys.stderr)

    if (output[0] == 0):
        return render_template("index.html", result="Not a attack")
    else:
        return render_template("index.html", result="Attack")

if (__name__ == "__main__"):
    app.run()


from datetime import datetime
from uuid import uuid4

from sqlalchemy import Column, DateTime, String
from sqlalchemy.sql import expression as sql

from db import Base

class User(Base):
    __tablename__ = "users"
    id = Column(String, primary_key=True)
    full_name = Column(String)
    created_at = Column(DateTime, index=True, default=datetime.utcnow)

    def __repr__(self):
        return (
            f"<{self.__class__.__name__}("
            f"id={self.id}, "
            f"full_name={self.full_name}, "
            f")>"
        )

    @classmethod
    async def create(cls, db, **kwargs) -> "User":
        query = (
            sql.insert(cls)
            .values(id=str(uuid4()), **kwargs)
            .returning(cls.id, cls.full_name)
        )
        users = await db.execute(query)
        await db.commit()
        return users.first()

    @classmethod
    async def update(cls, db, id, **kwargs) -> "User":
        query = (
            sql.update(cls)
            .where(cls.id == id)
            .values(**kwargs)
            .execution_options(synchronize_session="fetch")
            .returning(cls.id, cls.full_name)
        )
        users = await db.execute(query)
        await db.commit()
        return users.first()

    @classmethod
    async def get(cls, db, id) -> "User":
        query = sql.select(cls).where(cls.id == id)
        users = await db.execute(query)
        (user,) = users.first()
        return user

    @classmethod
    async def get_all(cls, db) -> list["User"]:
        query = sql.select(cls)
        users = await db.execute(query)
        users = users.scalars().all()
        return users

    @classmethod
    async def delete(cls, db, id) -> bool:
        query = (
            sql.delete(cls)
            .where(cls.id == id)
            .returning(
                cls.id,
                cls.full_name,
            )
        )
        await db.execute(query)
        await db.commit()
        return True


from flask import Blueprint
from project import db
from project.Categories.models import Categories

"""
in this file we are defining the Categories blue print setting & wrote all the end points that related to Categories
"""
categories = Blueprint('categories', __name__,  static_folder='static')

#display Categories:
@categories.route('/Categories/', methods = ['GET'])
@categories.route('/Categories/<id>')
def all_Categories(id = -1):
    Categories_res =[]
    if int(id) == -1:
        for category in Categories.query.all():
            Categories_res.append({"id":category.id, "name":category.name, "Description":category.Description, "link":category.link, "img":category.img})
        return Categories_res
    if int(id) > -1:
        category = Categories.query.get(int(id))
        Categories_res.append({"id":category.id, "name":category.name, "Description":category.Description, "link":category.link, "img":category.img})
        return Categories_res


from flask import request

from xivo_dao.alchemy.callfiltermember import Callfiltermember as CallFilterMember
from xivo_dao.helpers import errors
from xivo_dao.helpers.exception import NotFoundError

from wazo_confd.auth import required_acl
from wazo_confd.helpers.restful import ConfdResource

from .schema import CallFilterRecipientUsersSchema, CallFilterSurrogateUsersSchema

class CallFilterRecipientUserList(ConfdResource):
    schema = CallFilterRecipientUsersSchema
    has_tenant_uuid = True

    def __init__(self, service, call_filter_dao, user_dao):
        self.service = service
        self.call_filter_dao = call_filter_dao
        self.user_dao = user_dao

    @required_acl('confd.callfilters.{call_filter_id}.recipients.users.update')
    def put(self, call_filter_id):
        tenant_uuids = self._build_tenant_list({'recurse': True})
        call_filter = self.call_filter_dao.get(
            call_filter_id, tenant_uuids=tenant_uuids
        )
        form = self.schema().load(request.get_json())
        try:
            recipients = []
            for user_form in form['users']:
                user = self.user_dao.get_by(
                    uuid=user_form['user']['uuid'], tenant_uuids=tenant_uuids
                )
                recipient = self.service.find_recipient_by_user(call_filter, user)
                if not recipient:
                    recipient = CallFilterMember()
                    recipient.user = user
                recipient.timeout = user_form['timeout']
                recipients.append(recipient)

        except NotFoundError as e:
            raise errors.param_not_found('users', 'User', **e.metadata)

        self.service.associate_recipients(call_filter, recipients)
        return '', 204

class CallFilterSurrogateUserList(ConfdResource):
    schema = CallFilterSurrogateUsersSchema
    has_tenant_uuid = True

    def __init__(self, service, call_filter_dao, user_dao):
        self.service = service
        self.call_filter_dao = call_filter_dao
        self.user_dao = user_dao

    @required_acl('confd.callfilters.{call_filter_id}.surrogates.users.update')
    def put(self, call_filter_id):
        tenant_uuids = self._build_tenant_list({'recurse': True})
        call_filter = self.call_filter_dao.get(
            call_filter_id, tenant_uuids=tenant_uuids
        )
        form = self.schema().load(request.get_json())
        try:
            surrogates = []
            for user_form in form['users']:
                user = self.user_dao.get_by(
                    uuid=user_form['user']['uuid'], tenant_uuids=tenant_uuids
                )
                surrogate = self.service.find_surrogate_by_user(call_filter, user)
                if not surrogate:
                    surrogate = CallFilterMember()
                    surrogate.user = user
                surrogates.append(surrogate)

        except NotFoundError as e:
            raise errors.param_not_found('users', 'User', **e.metadata)

        self.service.associate_surrogates(call_filter, surrogates)
        return '', 204


import requests
import json
import pyttsx3
import speech_recognition as sr
import re
import threading
import time

API_KEY = 'tA0_xgVpoaiH'
PROJECT_TOKEN = 'tKijAox01o6c'
RUN_TOKEN = 'tJ9Vk3gTO8kM'

class Data:
    def __init__(self,api_key,project_token):
        self.api_key = api_key
        self.project_token = project_token
        self.params = {
            "api_key": self.api_key

        }
        self.data = self.get_data()

    def get_data(self):
        response = requests.get(f'https://parsehub.com/api/v2/projects/{PROJECT_TOKEN}/last_ready_run/data',
                                params={'api_key': API_KEY})

        data = json.loads(response.text)
        return data

    def all_teams(self):#list of all the teams that are playing
        teamsname = self.data['team1']
        versus = []

        for content in teamsname:
            versus.append(content['team1name'])
            versus.append(content['playing'])

        return versus

    def update_data(self):
        response = requests.post(f'https://parsehub.com/api/v2/projects/{self.project_token}/run',params = self.params)

        def poll():
            time.sleep(0.1)
            old_data = self.data
            while True:
                new_data = self.get_data()
                if new_data != old_data:
                    self.data = new_data
                    print("Data Updated")
                    break
                time.sleep(5)

        t = threading.Thread(target=poll)
        t.start()

#Voice
def speak(text):
    engine = pyttsx3.init()
    engine.say(text)
    engine.runAndWait()

def get_audio():
    r = sr.Recognizer()
    with sr.Microphone() as source:
        audio = r.listen(source)
        said = ''

        try:
            said = r.recognize_google(audio)
        except Exception as e:
            print("Exception:", str(e))
    return said.lower()

def main():
    data = Data(API_KEY, PROJECT_TOKEN)
    print("Program Started")
    END_PHRASE ='stop'
    result = None

    TOTAL_PATTERNS = {re.compile("[\w\s]+ teams [\w\s]+playing"):data.all_teams,re.compile("[\w\s]+playing"): data.all_teams
    }

    UPDATE_COMMAND = "update"
    while True:
        print("Speak")
        text = get_audio()
        print(text)

        for pattern,func in TOTAL_PATTERNS.items():
            if pattern.match(text):
                result = func()
                break
        if text ==  UPDATE_COMMAND:
            result = "Data is being updated"
            data.update_data()

        if result:
            print(result)
            speak(result)

        if text.find(END_PHRASE):
            break

main()


# 文件读写
import shutil
txtpath = '/Users/fenggao/git/third/python_learn/Demo1/iotest.txt'
try:
    f = open(txtpath, 'r') #
    print(f.read())
finally:
    f.close()

# 等同于上面的语句
with open(txtpath, 'r') as f:
    print(f.read())

# 内存中读写数据
from io import StringIO, BytesIO

f = StringIO()
f.write('Hello ')
f.write('World!2')
print(f.getvalue())

f2 = StringIO('Hello! \nGoodBye! \n')
while True:
    s = f2.readline()
    if s == '':
        break
    print(s)

b = BytesIO()
b.write('中文'.encode('utf-8'))
print(b.getvalue())

b2 = BytesIO(b.getvalue())
print(b2.read())

# 调用操作系统的接口命令
import os
print(os.uname())
print(os.environ)
print(os.environ.get('PATH'))
# 查看当前目录的绝对路径:
print(os.path.abspath('.'))
# 要通过os.path.join()函数，这样可以正确处理不同操作系统的路径分隔符,同理os.path.split()拆分路径
# os.mkdir(os.path.join('/Users/reamongao/git/python/Demo1', 'testDir'))
# os.rmdir('/Users/reamongao/git/python/Demo1/testDir')
print("splitext---")
print(os.path.splitext(txtpath))

# os.rename('iotest.txt', 'iotest.py')
# os.remove('iotest.py')

shutil.copyfile('iotest.txt', 'iotest.py')

# 列出当前目录文件夹
print([x for x in os.listdir('.') if os.path.isdir(x)])
# 列出所以py文件
print([x for x in os.listdir('.') if os.path.isfile(x) and os.path.splitext(x)[1]=='.py'])

# 序列化和反序列化
import pickle

d = dict(name = 'reamon', age = 20, sex = 'male')
f = open('dump', 'wb')
pickle.dump(d, f)
f.close()

f2 = open('dump', 'rb')
d2 = pickle.load(f2)

print(d2)

print('----------')
import json
json_str = json.dumps(d)
print(json_str)

print('json.loads')
print(json.loads(json_str))

# 一般的类转json
class Student(object):
    def __init__(self, name, age, score):
        self.name = name
        self.age = age
        self.score = score

# 这个方法使得对象转为dict,就可以json转了
def student2dict(std):
    return {
        'name': std.name,
        'age': std.age,
        'score': std.score
    }
s = Student('Bob', 20, 88)
print('json.dumps Student')
print(json.dumps(s, default=student2dict))

# json转为实例
json_str = '{"age": 20, "score": 88, "name": "Bob"}'
def dict2student(d):
    return Student(d['name'], d['age'], d['score'])
print(json.loads(json_str, object_hook=dict2student))


# import time
from pathlib import Path
from typing import Tuple, List, Dict

import numpy as np
import torch
from transformers import AutoTokenizer

class Embedder(object):
    """Class for wrapping up the tokenizer of the BERT models.
    """

    def __init__(self, model_path: str = 'xlm-roberta-base'):
        """Constructor of the class.

        Args:
            model_path (str, optional): BERT model name or path to the BERT model folder. Defaults to 'xlm-roberta-base'.
        """
        self.model_path = model_path
        self.tokenizer = AutoTokenizer.from_pretrained(model_path)
        if any([r'\\' in model_path, '/' in model_path, '\\' in model_path]):
            self.model_name = '-'.join(Path(model_path).stem.split('-')[:-4])
        else:
            self.model_name = model_path

    def __call__(self, first_sentence: str, second_sentence: str = None, DEBUG: bool = False) -> Dict[str, torch.Tensor]:
        """Pipeline for the tokenizer.

        Args:
            first_sentence (str): First input sentence.
            second_sentence (str): Second input sentence.
            DEBUG (bool, optional): Flag for DEBUG. Defaults to False.

        Returns:
            Tuple[torch.FloatTensor, torch.FloatTensor]: Context mask to indicate the paddings of the sequence (1D torch.FloatTensor) and token ids (1D torch.FloatTensor).
            Dict[str, torch.Tensor]: {
                'input_ids' (torch.LongTensor): Token indices for the sentences. 2D tensor (batch_size, max_sequence_length of concatenated sentence pairs).
                'attention_mask' (torch.LongTensor): Indices used to mask padded tokens for the sentences. 2D tensor (batch_size, max_sequence_length of concatenated sentence pairs).
                'offset_mappping' (torch.LongTensor): Character indices for each token in the sentences. 3D tensor (batch_size, max_sequence_length of concatenated sentence pairs, 2)
            }
        """
        if second_sentence is not None:
            token_dict = self.tokenizer(first_sentence, second_sentence, return_tensors='pt')

        else:
            token_dict = self.tokenizer(first_sentence, return_tensors='pt')

        # pad_ind = self.tokenizer.convert_tokens_to_ids(
        #     self.tokenizer.pad_token)
        # pad_mask = torch.where(token_dict['input_ids'] == pad_ind, torch.ones_like(
        #     token_dict['input_ids']), torch.zeros_like(token_dict['input_ids']))

        if not DEBUG:
            return token_dict
        else:
            print('\n')
            print(f'model_name: {self.model_name}')
            print(self.tokenizer.__str__)
            print(f"Token (str): {self.tokenizer.convert_ids_to_tokens(token_dict['input_ids'][0])}")
            print(f"Token (int): {token_dict['input_ids']}")
            print(f"Context mask: {token_dict['attention_mask']}")

    """Authored by Yury Kashnitsky <kashnitsky @ Kaggle>"""

    @staticmethod
    def get_indices_of_the_words_by_start_char_id(offset_mapping: List[Tuple[int, int]],
                                                  start_char_id1: int,
                                                  start_char_id2: int):
        """
        :param offset_mapping: a list of tuples, each of them indicating start and end ids
                                (in the original string) of word
                                pieces after tokenization with XLMRobertaTokenizerFast
        :param start_char_id1: start id in the first sentence
        :param start_char_id2: start id in the second sentence

        """

        zero_idx = [i for i, (s, e) in enumerate(offset_mapping) if (s, e) == (0, 0)]

        offset_mapping_first_sent = offset_mapping[:zero_idx[2]]
        offset_mapping_second_sent = offset_mapping[zero_idx[2]:]

        id1, id2 = 0, 0
        for i, (s, e) in enumerate(offset_mapping_first_sent):
            if (s, e) == (0, 0):
                continue
            # Check if the offset mappings start with the char index or include it (this is for cases where the tokenization creates deviation, i.e. training.en-en.7353 sentence2).
            if (s == start_char_id1) or (start_char_id1 > s and start_char_id1 <= e):
                # Find the last token (ordered) that has the start index as specified.
                # This is to avoid some cases where the returned offset mappings are troublesome.
                # For example, when encoding training.en-en.914, two entities of the offset mappings are [7, 8] and [7, 16], where [7, 8] is trivial.
                id1 = i
                # break

        for i, (s, e) in enumerate(offset_mapping_second_sent):
            if (s, e) == (0, 0):
                continue
            if (s == start_char_id2) or (start_char_id2 > s and start_char_id2 <= e):
                id2 = i
                # break

        # The saved alignments are separate for the sentence pairs,
        # so for this application we only need id2 counting from the start without offset.
        # id2 += len(offset_mapping_first_sent)

        return id1, id2

class BatchEmbedder(Embedder):
    """Class for wrapping up the batch tokenizer of the BERT models.
    """

    def __call__(self, first_sentences: list, second_sentences: list = None, DEBUG: bool = False) -> Dict[str, torch.Tensor]:
        """Pipeline for the batch tokenizer.

        Args:
            first_sentences (list): List of first input sentences.
            second_sentences (list): List of second input sentences.
            DEBUG (bool, optional): Flag for DEBUG. Defaults to False.

        Returns:
            Dict[str, torch.Tensor]: {
                'input_ids' (torch.LongTensor): Token indices for the sentence pairs. 2D tensor (batch_size, max_sequence_length of concatenated sentence pairs).
                'attention_mask' (torch.LongTensor): Indices used to mask padded tokens for the sentence pairs. 2D tensor (batch_size, max_sequence_length of concatenated sentence pairs).
                'offset_mapping' (torch.LongTensor): Character indices for each token in the sentence pairs. 3D tensor (batch_size, max_sequence_length of concatenated sentence pairs, 2)
            }
        """

        # tmp_int_time = time.time()

        if second_sentences is not None:
            token_dict = self.tokenizer(first_sentences, second_sentences, return_tensors='pt',
                                        return_offsets_mapping=True, padding=True)
        else:
            token_dict = self.tokenizer(
                first_sentences, return_tensors='pt', return_offsets_mapping=True, padding=True)

        if not DEBUG:
            return token_dict
        else:
            print('\n')
            print(f'model_name: {self.model_name}')
            print(self.tokenizer.__str__)
            print("Token (str): {}".format(
                list(self.tokenizer.convert_ids_to_tokens(token_dict['input_ids'][i]) for i in np.arange(token_dict['input_ids'].shape[0]))))
            print("Token (int): {}".format(list(token_dict['input_ids'][i] for i in np.arange(
                token_dict['input_ids'].shape[0]))))
            print(f"Context mask: {token_dict['attention_mask']}")


from CRM.Processors.Service.BWUsage import show_bw_usage
from CRM.Processors.Service.Dedicated import view_dedicated_service, add_new_dedicate_service, delete_dedicate_service, \
    get_dedicated_service_detail
from CRM.Processors.Service.Disconnect import kill_user
from CRM.Processors.Service.FrmServiceSummery import show_user_service_summery, report_service_problem
from CRM.Processors.Service.FrmTempRecharge import temp_recharge
from CRM.Processors.Service.IPStaticManagement import view_ip_static, add_new_static_ip, delete_static_ip, \
    request_ip_static, delete_user_static_ip, toggle_free_ip
from CRM.Processors.Service.ServiceGroupManagement import assign_service_to_group, \
    assign_package_to_group
from CRM.Processors.Service.ServiceManagement import view_float_service, view_service_formula, add_new_formula, \
    get_formula_detail, remove_formula, add_float_service, get_float_service_detail, remove_basic_service, \
    view_custom_option, add_float_custom_options, get_custom_option_detail, remove_custom_option, get_selected_options, \
    assign_custom_option, view_custom_option_group, add_custom_option_group, remove_custom_option_group, \
    get_custom_option_group, buy_float_service, calculate_float_price, get_service_options, get_hidden_groups, \
    validate_selection, add_float_template, show_all_service, update_normal_service, \
    import_ibs_services, delete_normal_service, get_normal_service_details, get_related_services, set_related_services, \
    assign_service_map, get_service_map_list, create_float_invoice, assign_default_group, view_float_service_discount, \
    get_float_service_discount, delete_float_service_discount, add_float_service_discount, view_service_templates, \
    delete_service_template, view_float_template_options, service_float_get_client_template, \
    service_float_assign_template, user_template_toggle_test, user_template_toggle_public_test, \
    service_kill_current_user, view_float_package_discount, add_float_package_discount, get_float_package_discount, \
    delete_float_package_discount
from CRM.Processors.Service.views import show_all_service_properties, service_group_management, assign_service_to_user, \
    service_switch_old
from CRM.Processors.Service.UpdateUserService import update_user_service

from django.conf.urls import patterns, url
__author__ = 'Administrator'
urlpatterns = patterns('',
                       url(r'^show/all/$', show_all_service, name='show all services'),
                       url(r'^dc_current/$', service_kill_current_user, name='service_kill_current'),
                       url(r'^import/', import_ibs_services, name='service_import_ibs'),
                       url(r'^create/$', update_normal_service, name='create service'),
                       url(r'^j/$', get_normal_service_details, name='service_normal_get'),
                       url(r'^delete/$', delete_normal_service, name='delete service'),
                       url(r'^summery/$', show_user_service_summery, name='user services summery'),
                       url(r'^reports/$', report_service_problem, name='report_service_problem'),
                       url(r'^assign/$', assign_service_to_user, name='assign service to user'),
                       url(r'^switch/(?P<user_id>\d+)/$', service_switch_old,name='service_switch_old'),
                       url(r'^temp/$', temp_recharge, name='temp recharge'),
                       url(r'^update/$', update_user_service, name='update_user_service'),
                       url(r'^ip/show/$', request_ip_static, name='view_ip_static_request'),
                       url(r'^ip/rmu/$', delete_user_static_ip, name='delete_user_static_ip'),
                       url(r'^dc/$', kill_user, name='kill_online_user'),
                       url(r'^graph/$', show_bw_usage, name='show_bw_usage'),
                       url(r'^groups/$', service_group_management, name='service_group_management'),
                       url(r'^groups/pack/$', assign_package_to_group, name='assign_package_to_group'),
                       url(r'^groups/service/$', assign_service_to_group, name='assign_service_to_group'),
                       url(r'^properties/$', show_all_service_properties, name='show_all_service_properties'),
                       url(r'^ip/pool/$', view_ip_static, name='view_ip_statics'),
                       url(r'^ip/pool/add/$', add_new_static_ip, name='add_ip_statics'),
                       url(r'^ip/pool/delete/$', delete_static_ip, name='delete_ip_static'),
                       url(r'^ip/mkf/$', toggle_free_ip, name='toggle_free_ip'),
                       url(r'^float/$', view_float_service, name='service_view_float'),
                       url(r'^float/discount/$', view_float_service_discount, name='service_float_discount_view'),
                       url(r'^float/discount/j/$', get_float_service_discount, name='service_float_discount_single'),
                       url(r'^float/discount/rm/$', delete_float_service_discount, name='service_float_discount_delete'),
                       url(r'^float/discount/add/$', add_float_service_discount, name='service_float_discount_add'),
                       url(r'^float/default/$', assign_default_group, name='service_float_assign_default'),
                       url(r'^float/related/j/$', get_related_services, name='service_float_related_get'),
                       url(r'^float/related/add/$', set_related_services, name='service_float_related_set'),
                       url(r'^float/add/$', add_float_service, name='service_float_add'),
                       url(r'^float/j/$', get_float_service_detail, name='service_float_detail'),
                       url(r'^float/rm/$', remove_basic_service, name='service_float_remove'),
                       url(r'^float/formula/$', view_service_formula, name='service_float_view_formula'),
                       url(r'^float/formula/add/$', add_new_formula, name='service_float_add_formula'),
                       url(r'^float/formula/j/$', get_formula_detail, name='service_float_formula_view_json'),
                       url(r'^float/formula/rm/$', remove_formula, name='service_float_formula_rm'),
                       url(r'^float/option/$', view_custom_option, name='service_float_option_view'),
                       url(r'^float/option/add/$', add_float_custom_options, name='service_float_option_add'),
                       url(r'^float/option/j/$', get_custom_option_detail, name='service_float_option_get'),
                       url(r'^float/option/rm/$', remove_custom_option, name='service_float_option_remove'),
                       url(r'^float/option/service/add/$', assign_service_map, name='service_float_option_add_map'),
                       url(r'^float/option/service/j/$', get_service_map_list, name='service_float_option_view_map'),
                       url(r'^float/options/$', get_selected_options, name='service_float_options_get'),
                       url(r'^float/option/group/$', view_custom_option_group, name='service_float_option_group_view'),
                       url(r'^float/option/group/add/$', add_custom_option_group,
                           name='service_float_option_group_add'),
                       url(r'^float/option/group/rm/$', remove_custom_option_group,
                           name='service_float_option_group_remove'),
                       url(r'^float/option/group/j/$', get_custom_option_group,
                           name='service_float_option_group_view_single'),
                       url(r'^float/assign/$', assign_custom_option, name='service_float_assign_option'),
                       url(r'^float/buy/$', buy_float_service, name='service_float_buy'),
                       url(r'^float/buy/invoice/', create_float_invoice, name='service_float_invoice'),
                       url(r'^float/buy/cal/$', calculate_float_price, name='service_float_buy_calculate'),
                       url(r'^float/buy/options/$', get_service_options, name='service_float_buy_options'),
                       url(r'^float/buy/related/$', get_hidden_groups, name='service_float_buy_related_group'),
                       url(r'^float/buy/validate/$', validate_selection, name='service_float_validate'),
                       url(r'^float/template/add/$', add_float_template, name='service_float_template_add'),
                       url(r'^float/template/$', view_service_templates, name='service_float_template_view'),
                       url(r'^float/template/toggle/$', user_template_toggle_test,
                           name='service_float_template_toggle_test'),
                       url(r'^float/template/toggle_public/$', user_template_toggle_public_test,
                           name='service_float_template_toggle_public_test'),
                       url(r'^float/template/rm/$', delete_service_template,
                           name='service_float_template_rm'),
                       url(r'^float/template/assign/$', service_float_assign_template,
                           name='service_float_template_assign'),
                       url(r'^float/template/user/$', service_float_get_client_template,
                           name='service_float_template_user_get'),
                       url(r'^float/template/options/$', view_float_template_options,
                           name='service_float_template_options_view'),
                       url(r'^dedicate/$', view_dedicated_service, name='service_dedicate_view'),
                       url(r'^dedicate/add/$', add_new_dedicate_service, name='service_dedicate_add'),
                       url(r'^dedicate/rm/$', delete_dedicate_service, name='service_dedicate_delete'),
                       url(r'^dedicate/j/$', get_dedicated_service_detail, name='service_dedicate_detail'),
                       url(r'^float/discount/package/$', view_float_package_discount,
                           name='service_float_discount_package_view'),
                       url(r'^float/discount/package/add/$', add_float_package_discount,
                           name='service_float_discount_package_add'),
                       url('^float/discount/package/j/$', get_float_package_discount,
                           name='service_float_discount_package_get'),
                       url('^float/discount/package/rm/$', delete_float_package_discount,
                           name='service_float_discount_package_delete')
                       )


import sys
import itertools
import random
import simplechains.tex.tex_fuzzer as tex_fuzzer
from stateless.status import *
from stateless.exceptions import *
from tokens import pick
import stateless.config as CONFIG

ALLOWED_BYTES = [i for i in CONFIG.ALL_BYTES if i not in CONFIG.DISALLOWED_BYTES]

MYSET_OF_BYTES = []
PREF_OF_BYTES = []

for k in ALLOWED_BYTES:
	MYSET_OF_BYTES.append(bytes([k]))
for k in CONFIG.PREF_BYTES:
	PREF_OF_BYTES.append(bytes([k]))

SEEN_AT = []

def init_set_of_bytes(s_bytes):
    global SET_OF_BYTES
    SET_OF_BYTES = s_bytes

def logit(v):
    if CONFIG.LOG:
        print(v, file=sys.stderr)

def new_byte(choices):
    open_brak = False
    x = random.randrange(7)
    if x == 0:
        token, open_brak = pick()
        v = b''
        v = v + token
    elif x>4:
        v = random.choice(PREF_OF_BYTES)
    else:
        v = random.choice(choices)
    return v, open_brak

def backtrack(prev_bytes, all_choices, limit=0):
    global SEEN_AT
    if not prev_bytes:
        raise BacktrackLimitException('Cant backtrack beyond zero index')
    if limit == -1:
        raise BacktrackLimitException('Cant backtrack beyond last valid inputs')
    # backtrack one byte
    seen = SEEN_AT[len(prev_bytes)-1]
    SEEN_AT = SEEN_AT[:-1]
    last_byte = prev_bytes[-1]
    logit('backtracking %d %s' % (len(prev_bytes), last_byte))
    #assert (last_byte,) in seen
    prev_bytes = prev_bytes[:-1]
    choices = [i for i in all_choices if i not in seen]
    if not choices:
        return backtrack(prev_bytes, all_choices, limit - 1)
    return seen, prev_bytes, choices

def till_n_length_choices(my_choices, rs):
    return my_choices # disable fudging
    all_choices = []
    for r in range(1, rs+1):
        v = [bytes(b''.join(i)) for i in itertools.product(my_choices, repeat=r)]
        #random.shuffle(v)
        all_choices.extend(v)
    return all_choices

def generate(validator, prev_bytes=None, limit=0):
    global SEEN_AT
    all_choices = MYSET_OF_BYTES
    prev_bytes=None
    if prev_bytes is None: prev_bytes = b''
    min_input_len = random.choice(CONFIG.MIN_INPUT_LEN)
	# The trace list works like a stack.
	# The goal is to keep track of open curley braces and open dollar signs in the tex input
    trace = []
    prev_trace = []
    seen = set()
    iter_limit = CONFIG.ITERATION_LIMIT
    while iter_limit:
        if len(prev_bytes) > CONFIG.MAX_INPUT_LEN:
            raise InputLimitException('Exceeded %d bytes' % CONFIG.MAX_INPUT_LEN)
        iter_limit -= 1
        choices = [i for i in all_choices if i not in seen]
        if not choices:
            seen, prev_bytes, choices = backtrack(prev_bytes, all_choices, limit=-1) # disable

        byte, open_brak = new_byte(choices)
        cur_bytes = prev_bytes + byte

        trace = prev_trace.copy()
        if byte == b'{' or open_brak == True: # Dealing with { byte
            trace.append('}')
        elif byte == b'}' and '}' in trace:
            if trace[-1] == '}':
                trace.pop(-1)
            else:
                continue
        elif byte == b'$' and 'a$' not in trace: # Dealing with $ byte
            trace.append('a$')
        elif byte == b'$' and 'a$' in trace:
            if trace[-1] == 'a$':
                trace.pop(-1)
            else:
                continue

        cur_input = str(cur_bytes)[2:-1]
        rv, n, x = tex_fuzzer.validate_tex(cur_input, min_input_len, trace)
        if rv == Status.Complete:
            prev_trace = trace.copy()
            SEEN_AT.append(seen)
            return cur_bytes
        elif rv == Status.Incomplete:
            prev_trace = trace.copy()
            seen.add(byte)  # don't explore this byte again
            prev_bytes = cur_bytes
            SEEN_AT.append(seen)
            seen = set()
            # reset this if it was modified by incorrect
            all_choices = MYSET_OF_BYTES
        elif rv == Status.Incorrect:
            if n is None or n == -1:
                seen.add(byte)
                continue
            else:
                if n > 0:
                    #raise Exception('Backtrack disabled..')
                    logit("%s %s" % (len(choices), len(seen)))
                    if n < len(SEEN_AT):
                        seen = SEEN_AT[n]
                        SEEN_AT = SEEN_AT[:n]

                    seen.add(byte)
                    rs = len(cur_bytes) - n
                    all_choices = till_n_length_choices(MYSET_OF_BYTES, min(rs, 2))
                    prev_bytes = prev_bytes[:n]
                else:
                    pass
                    # likely a core dump
        else:
            print(str(rv))
            print(str(Status.Incorrect))
            raise Exception(rv)
    raise IterationLimitException('Exhausted %d loops' % CONFIG.ITERATION_LIMIT)


#!/usr/bin/env python
#
# This program is a Youtube Client that removes some videos from the Unprocessed playlist

import os
import sys
import optparse
import json
from io import open
import csv

import googleapiclient.errors

import auth
from auth import lib

# from thinkland import classes
from thinkland.classes import log
from thinkland.classes import appendProcessedVideo
from thinkland.playlist import PlaylistDB
from thinkland.meeting import MeetingDB

unprocessed_playlist = 'PLzr1p9rMdhyAPAN1c6O-AeoJPUk4LkQj2'

UNPROCESSED_LIMIT = 50

class AuthenticationError(Exception): pass

class RequestError(Exception): pass

def get_youtube_handler(options):
    """Return the API Youtube object."""
    home = os.path.expanduser("~")
    default_credentials = os.path.join(home, ".youtube-upload-credentials.json")
    client_secrets = options.client_secrets or os.path.join(home, ".client_secrets.json")
    credentials = options.credentials_file or default_credentials
    lib.debug("Using client secrets: {0}".format(client_secrets))
    lib.debug("Using credentials file: {0}".format(credentials))
    get_code_callback = (auth.browser.get_code
                         if options.auth_browser else auth.console.get_code)
    return auth.get_resource(client_secrets, credentials,
                             get_code_callback=get_code_callback)

def get_unprocessed_videos(youtube, playlistDB, meetingDB, process_limit, minutesAllow=15):
    next_page_token = ''
    ret = list()
    eligible = 0
    page = 1
    while eligible < process_limit:
        print("start Unprocessed Page %d" % page)
        page += 1
        request = youtube.playlistItems().list(
            part='snippet,contentDetails',
            maxResults=UNPROCESSED_LIMIT,
            pageToken=next_page_token,
            playlistId=unprocessed_playlist
        )
        pl_items_list = request.execute()

        # print(json.dumps(pl_items_list, indent=4))

        for item in pl_items_list['items']:
            video = {
                    'id': item['contentDetails']['videoId'],
                    'title': item['snippet']['title'],
                    'desc': item['snippet']['description'],
                    'itemId': item['id']
                }

            if errorVideo(video):
                ret.append(video)
                eligible += 1
                if eligible >= process_limit:
                    break

        if "nextPageToken" in pl_items_list:
            next_page_token = pl_items_list['nextPageToken']
        else:
            break
    return ret

def errorVideo(video):
    sections = video['title'].split(' ')
    if (len(sections) < 3 or len(sections[1]) < 11 or len(sections[2]) < 6 or
            not sections[1][3:].isdigit() or not sections[2].isdigit()):
        return True
    return False

def dry_run(youtube, playlistDB, meetingDB, processLimit, minutesAllow=15):
    unprocessed_videos = get_unprocessed_videos(youtube, playlistDB, meetingDB, processLimit)

    count_matched = 0
    count_valid = 0
    for video in unprocessed_videos:
        if errorVideo(video):
            # TODO: print some error messages
            print('Video title error: %s' % video['title'])
            original_title = video["desc"].split("\n")[-1]
            print("oritinal title: %s" % original_title)
            meeting = meetingDB.match(original_title, minutesAllow)
            if not meeting:
                print("not found meeting")
                continue

            playlist = playlistDB.getPlaylistId(meeting.classId, meeting.teacherName)
            if not playlist:
                print("not found playlist")
                continue

            next_page_token = ""
            found = False
            while not found:
                request = youtube.playlistItems().list(
                    part='snippet,contentDetails',
                    maxResults=UNPROCESSED_LIMIT,
                    pageToken=next_page_token,
                    playlistId=playlist
                )
                pl_items_list = request.execute()

                for item in pl_items_list['items']:
                    if item['contentDetails']['videoId'] != video['id']:
                        continue

                    # request = youtube.playlistItems().delete(id=video['itemId'])
                    # request.execute()
                    # log('Removed video %s from unprocessed playlist' % video['id'])
                    print("The video is already in the target playlist.")
                    found = True
                    break

                if "nextPageToken" in pl_items_list:
                    next_page_token = pl_items_list['nextPageToken']
                else:
                    break

def run_main(parser, options, args, output=sys.stdout, minutesAllow=15):
    """Run the main scripts from the parsed options/args."""
    youtube = get_youtube_handler(options)
    if not youtube:
        raise AuthenticationError("Cannot get youtube resource")

    meeting_csv_file = options.meeting_csv or 'data/meetings.csv'
    playlistDB = PlaylistDB(meeting_csv_file)
    meetingDB = MeetingDB(meeting_csv_file)
    processedCSV = options.processed_csv or 'data/processed.csv'

    process_limit = options.process_limit or 50

    if not options.dry_run_off:
        dry_run(youtube, playlistDB, meetingDB, process_limit)
        return

    unprocessed_videos = get_unprocessed_videos(youtube, playlistDB, meetingDB, process_limit)
    youtube_points = 1
    v_processed = 0
    skip_processed = 0

    for video in unprocessed_videos:
        if errorVideo(video):
            # TODO: print some error messages
            print('Video title error: %s' % video['title'])
            original_title = video["desc"].split("\n")[-1]
            print("oritinal title: %s" % original_title)
            meeting = meetingDB.match(original_title, minutesAllow)
            if not meeting:
                print("not found meeting")
                continue

            playlist = playlistDB.getPlaylistId(meeting.classId, meeting.teacherName)
            if not playlist:
                print("not found playlist")
                continue

            next_page_token = ""
            found = False
            while not found:
                request = youtube.playlistItems().list(
                    part='snippet,contentDetails',
                    maxResults=UNPROCESSED_LIMIT,
                    pageToken=next_page_token,
                    playlistId=playlist
                )
                pl_items_list = request.execute()

                for item in pl_items_list['items']:
                    if item['contentDetails']['videoId'] != video['id']:
                        continue

                    print("The video is already in the target playlist.")
                    request = youtube.playlistItems().delete(id=video['itemId'])
                    request.execute()
                    log('Removed video %s from unprocessed playlist' % video['id'])
                    found = True
                    break

                if "nextPageToken" in pl_items_list:
                    next_page_token = pl_items_list['nextPageToken']
                else:
                    break

def main(arguments):
    usage = """TODO(jackwaashere): Add usage"""
    parser = optparse.OptionParser(usage)

    # Authentication
    parser.add_option('', '--client-secrets', dest='client_secrets',
                      type="string", help='Client secrets JSON file')
    parser.add_option('', '--credentials-file', dest='credentials_file',
                      type="string", help='Credentials JSON file')
    parser.add_option('', '--auth-browser', dest='auth_browser', action='store_true',
                      help='Open a GUI browser to authenticate if required')

    # Business specific flags
    parser.add_option('', '--meeting_csv', dest='meeting_csv',
                      type="string", help='path to the csv file of meetingDB')
    parser.add_option('', '--processed_csv', dest='processed_csv',
                      type='string', help='path to the csv file for processed vidoes')
    parser.add_option('', '--dry_run_off', dest='dry_run_off', action='store_true',
                      help='Turns off dry run mode')
    parser.add_option('', '--process_limit', dest='process_limit',
                      type='int', help='Limit the maximum number of videos to process')

    options, args = parser.parse_args(arguments)

    try:
        run_main(parser, options, args)
    except googleapiclient.errors.HttpError as error:
        response = bytes.decode(error.content).strip()
        raise RequestError(u"Server response: {0}".format(response))

if __name__ == '__main__':
    main(sys.argv[1:])


from datasets import load_dataset
from transformers import BartForConditionalGeneration, AutoTokenizer
import torch
import pandas as pd

# for possible versions, see: https://huggingface.co/datasets/cnn_dailymail
# use this dataset instead of the original: https://huggingface.co/datasets/ccdv/cnn_dailymail
# see this bug for the reason why: https://github.com/huggingface/datasets/issues/996
cnn_versions = ['3.0.0', '1.0.0', '2.0.0']
cnn = load_dataset("ccdv/cnn_dailymail", cnn_versions[0])  # for general text summarization

model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
tokenizer = AutoTokenizer.from_pretrained('facebook/bart-large-cnn', add_prefix_space=True)

premise = cnn['train'][0]['article']
hypothesis = cnn['train'][0]['highlights']

# run through model pre-trained on MNLI
device = "cuda"
x = tokenizer.encode(
    premise,
    hypothesis,
    return_tensors='pt',
    truncation_strategy='longest_first'
)
model.to(device)
with torch.no_grad():
    generated = model.generate(x.to(device))

def postprocess(token_ids: list) -> str:
    """
    Converts tokens to text.

    :param token_ids: tokens from a tokenizer
    """
    return ''.join([
        tokenizer.decode(x, skip_special_tokens=True, clean_up_tokenization_spaces=True)
        for x in token_ids
    ])

bart_abs_sum = postprocess(generated)

import nltk
from sentence_transformers import SentenceTransformer, util
import numpy as np
from LexRank import degree_centrality_scores

model = SentenceTransformer('all-MiniLM-L6-v2')

# Our input document we want to summarize
# As example, we take the first section from Wikipedia
document = premise

# Split the document into sentences
sentences = nltk.sent_tokenize(document)
print("Num sentences:", len(sentences))

# Compute the sentence embeddings
embeddings = model.encode(sentences, convert_to_tensor=True)

# Compute the pair-wise cosine similarities
cos_scores = util.cos_sim(embeddings, embeddings).cpu().numpy()

# Compute the centrality for each sentence
centrality_scores = degree_centrality_scores(cos_scores, threshold=None)

# We argsort so that the first element is the sentence with the highest score
most_central_sentence_indices = np.argsort(-centrality_scores)

# Summarize top 5 sentences
lexrank_sum = ' '.join([sentences[idx].strip() for idx in most_central_sentence_indices[0:5]])

import pytextrank
import spacy
from math import sqrt
from operator import itemgetter

nlp = spacy.load("en_core_web_sm")

# add PyTextRank to the spaCy pipeline
nlp.add_pipe("textrank")
doc = nlp(cnn['train'][0]['article'])

def get_textrank_summary(doc: spacy.tokens.doc.Doc, summary_len: int = 3):
    """Returns summary from a Spacy parsed document."""
    sent_text = {sent_id: sent.text for sent_id, sent in enumerate(doc.sents)}
    sent_bounds = [[s.start, s.end, set([])] for s in doc.sents]

    # iterate through the top ranked phrases and add them to the phrase vector for each sentence
    limit_phrases = 4

    phrase_id = 0
    unit_vector = []

    for p in doc._.phrases:
        unit_vector.append(p.rank)

        for chunk in p.chunks:
            for sent_start, sent_end, sent_vector in sent_bounds:
                if chunk.start >= sent_start and chunk.end <= sent_end:
                    sent_vector.add(phrase_id)
                    break

        phrase_id += 1

        if phrase_id == limit_phrases:
            break

    # normalize the unit vector
    sum_ranks = sum(unit_vector)
    unit_vector = [rank / sum_ranks for rank in unit_vector]

    # iterate through each sentence, calculating its euclidean distance from the unit vector
    sent_rank = {}
    sent_id = 0

    for sent_start, sent_end, sent_vector in sent_bounds:
        sum_sq = 0.0
        for phrase_id in range(len(unit_vector)):

            if phrase_id not in sent_vector:
                sum_sq += unit_vector[phrase_id] ** 2.0

        sent_rank[sent_id] = sqrt(sum_sq)
        sent_id += 1

    # extract the sentences with the lowest distance, up to the limit requested
    results = [
        (sent_id, rank, sent_text[sent_id])
        for sent_id, rank in sorted(sent_rank.items(), key=itemgetter(1))
    ]
    summary = ' '.join([s[2] for s in results[:summary_len]])
    return summary, results

textrank_sum, textrank_sentranks = get_textrank_summary(doc=doc)

from transformers import pipeline

summarizer = pipeline("summarization", model="t5-base", tokenizer="t5-base", framework="pt")
t5_abs_sum = summarizer(document, min_length=5, max_length=100)[0]['summary_text']

from rouge import Rouge

def format_rouge_scores(rouge_scores):
    """Flattens Rouge scores"""
    rs_new = {}
    for rouge_type_dict in rouge_scores:
        for rouge_type, metric in rouge_type_dict.items():
            for name, val in metric.items():
                rs_new[rouge_type + "_" + name] = val
    return rs_new

# evaluate CNN summaries
rouge = Rouge()
scores = {
    'bart': format_rouge_scores(rouge.get_scores(hyps=bart_abs_sum, refs=hypothesis)),
    't5': format_rouge_scores(rouge.get_scores(hyps=t5_abs_sum, refs=hypothesis)),
    'lexrank': format_rouge_scores(rouge.get_scores(hyps=lexrank_sum, refs=hypothesis)),
    'textrank': format_rouge_scores(rouge.get_scores(hyps=textrank_sum, refs=hypothesis)),
}

from sentence_transformers.cross_encoder import CrossEncoder

model = CrossEncoder('facebook/bart-large')
bart_cos_sim_scores = model.predict([
    [bart_abs_sum, hypothesis],
    [t5_abs_sum, hypothesis],
    [lexrank_sum, hypothesis],
    [textrank_sum, hypothesis],
])
for idx, m in enumerate(list(scores)):
    scores[m]['bart_cos_sim'] = bart_cos_sim_scores[idx]

scores_df = pd.DataFrame(scores)
scores_df.to_csv('summarizer_scores.csv')

breakpoint()


import numpy as np
import torch.utils.data as data
import torchvision.transforms as transforms
from PIL import Image
from torch.utils.data import Dataset
from torch.utils.data.sampler import BatchSampler

from datasets.sampling import UniformSampler
from utils import params

def get_filenames_and_labels(data_folder, test_or_train='test'):
    images_paths = []
    images_indices = []
    images_labels = []
    train_labels = []
    test_labels = []
    train_images = []
    test_images = []

    if test_or_train == 'train':
        for i in range(1275):
            for j in range(4):
                path = data_folder + ('/ukbench%05d.jpg' % (i * 4 + j))
                images_paths.append(path)
                images_indices.append(i)
                train_images.append(path)
                train_labels.append(i)
        images_labels = train_labels

    if test_or_train == 'test':
        for i in range(1275, 2550):
            for j in range(4):
                path = data_folder + ('/ukbench%05d.jpg' % (i * 4 + j))
                images_paths.append(path)
                images_indices.append(i + 1275)
                test_images.append(path)
                test_labels.append(i)
        images_labels = test_labels

    images_labels = np.array(images_labels)
    train_labels = np.array(train_labels)
    test_labels = np.array(test_labels)
    print('images_labels.shape ', images_labels.shape)

    images_indices = np.array(images_indices)
    train_images = np.array(train_images)
    test_images = np.array(test_images)

    return images_indices, images_labels, images_paths, train_images, train_labels, test_images, test_labels

class UKB(Dataset):
    def __init__(self, data_folder, transform=None, test_or_train='test'):
        self.data_folder = data_folder
        self.transform = transform
        if test_or_train == 'train':
            self.train = True
        else:
            self.train = False
        self.images_indices, \
        self.images_labels, \
        self.images_paths, \
        self.train_images, \
        self.train_labels, \
        self.test_images, \
        self.test_labels = get_filenames_and_labels(data_folder,
                                                    test_or_train=test_or_train)

        print('self.images_labels ', self.images_labels)

    def __len__(self):
        if self.train:
            return len(self.train_images)
        else:
            return len(self.test_images)

    def __getitem__(self, index):
        transform_for_correction = transforms.Compose([
            transforms.ToPILImage(),
        ])
        if self.train:
            image = self.transform(Image.open(self.train_images[index]))
            label = self.train_labels[index]
        else:
            image = self.transform(Image.open(self.test_images[index]))
            label = self.test_labels[index]

        if image.shape[0] == 1:
            # print('Grayscale image is found! ', self.images_paths[index])
            image = transform_for_correction(image)
            image = transforms.ImageOps.colorize(image, (0, 0, 0), (255, 255, 255))
            image = self.transform(image)
            # print('new image.shape ', image.shape)

        if image.shape[1] < params.initial_image_size or image.shape[2] < params.initial_image_size:
            print('image is too small', image.shape)

        return image, label

def create_transformations_for_test_and_train():
    transform_train = transforms.Compose([
        transforms.Scale(params.initial_image_scale_size),
        transforms.RandomCrop(params.initial_image_size, padding=0),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),

        # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])
    transform_test = transforms.Compose([
        transforms.Scale(params.initial_image_scale_size),
        transforms.CenterCrop(params.initial_image_size),
        transforms.ToTensor(),
        # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])
    return transform_test, transform_train

def create_new_train_and_test_datasets(transform_train, transform_test, data_folder):
    # create new dataset for representational learning
    # where in train we have first 100 classes and in test the remaining 100
    new_train_dataset = UKB(data_folder=data_folder,
                                 transform=transform_train,
                                 test_or_train='train'
                                 )
    new_test_dataset = UKB(data_folder=data_folder,
                                transform=transform_test,
                                test_or_train='test'
                                )
    print('len(new_train_dataset.train_images) ', len(new_train_dataset.train_images))
    print('len(new_train_dataset.test_images) ', len(new_train_dataset.test_images))

    print('len(new_test_dataset.train_images) ', len(new_test_dataset.train_images))
    print('len(new_test_dataset.test_images) ', len(new_test_dataset.test_images))

    return new_test_dataset, new_train_dataset

def download_UKB_for_representation(data_folder):
    transform_train, transform_test = create_transformations_for_test_and_train()
    new_test_dataset, new_train_dataset = create_new_train_and_test_datasets(transform_train, transform_test,
                                                                             data_folder)

    train_loader = data.DataLoader(new_train_dataset,
                                   batch_sampler=BatchSampler(
                                       sampler=UniformSampler(new_train_dataset,
                                                              batch_size=params.batch_size_for_representation,
                                                              number_of_samples_with_the_same_label_in_the_batch=
                                                              params.number_of_samples_with_the_same_label_in_the_batch),
                                       batch_size=params.batch_size_for_representation,
                                       drop_last=True),
                                   num_workers=2)
    print('train_loader.batch_size = ', train_loader.batch_size,
          ' train_loader.batch_sampler.batch_size =', train_loader.batch_sampler.batch_size,

          ' train_loader.dataset ', train_loader.dataset)
    print('new_test_dataset.images_paths', new_test_dataset.images_paths)
    print('new_test_dataset.images_labels', new_test_dataset.images_labels)
    print('ful batch size = ', len(new_test_dataset.test_labels))
    test_loader = data.DataLoader(new_test_dataset,

                                  # unfortunately we don't have enough memory to evaluate easily on FULL test
                                  batch_size=params.batch_size_for_representation,

                                  drop_last=True,  # we need to drop last batch because it can had length less than k
                                  # and we won't be able to calculate recall at k
                                  shuffle=True,  # shuffle is extremely importatnt here because we take 10 neighbors
                                  # out of 16 images in the batch
                                  num_workers=2)

    print('new_train_dataset ', new_train_dataset.__len__())
    print('new_test_dataset ', new_test_dataset.__len__())
    print('new_train_dataset.images_paths', new_train_dataset.images_paths)
    print('new_train_dataset.images_labels', new_train_dataset.images_labels)
    print('ful batch size = ', len(new_train_dataset.test_labels))

    return train_loader, test_loader

def download_UKB_for_evaluation_or_spoc(data_folder):
    transform_train, transform_test = create_transformations_for_test_and_train()
    new_test_dataset, new_train_dataset = create_new_train_and_test_datasets(transform_train, transform_test,
                                                                             data_folder)

    # loaders with NO shuffling!!!!
    train_loader = data.DataLoader(new_train_dataset,
                                   batch_size=params.batch_size_for_representation,
                                   num_workers=2)
    print('loaders with NO shuffling train_loader.batch_size = ', train_loader.batch_size,
          ' loaders with NO shuffling train_loader.batch_sampler.batch_size =', train_loader.batch_sampler.batch_size,

          ' loaders with NO shuffling train_loader.dataset ', train_loader.dataset)
    test_loader = data.DataLoader(new_test_dataset,
                                  batch_size=params.batch_size_for_representation,
                                  num_workers=2)

    return train_loader, test_loader


import scrapy

class BlogsSpider(scrapy.Spider):
  name = 'naverblogSpider'

  start_urls=['https://d2.naver.com/home?page=0']

  def parse(self, response):
    print(response.css('h2 a'))

    # for posts in response.css('a'):
    #   print(posts.css('a::text').get())


import jsonpickle
from tinydb import TinyDB

from tournament import Tournament

def tournaments_list():
    print(
        "Liste des tournois sauvegardés:\n"
        "-------------------------------\n")
    db = TinyDB("data/db.json")
    tournaments_table = db.table("tournaments")
    count = 0
    for _ in tournaments_table:
        count += 1
        tournament = Tournament(jsonpickle.decode(_["Tournaments"]).__dict__)
        print(f"{count} - {tournament.name}")


from .util import get_data_file_path
import logging
from collections import OrderedDict, namedtuple
from operator import itemgetter
import re
import string
import traceback
from vt.object import Object

__all__ = ("SampleInfo",
           "LabeledSample",
           "AvLabels",)

SampleInfo = namedtuple('SampleInfo',
                        ['md5', 'sha1', 'sha256', 'labels'])

LabeledSample = namedtuple('LabeledSample',
                           ['md5', 'sha1', 'sha256', 'labels', 'family', 'tokens', 'groundtruth', 'is_pup'])

def log(message):
    print(message)

class AvLabels(object):

    def __init__(self, gen_file=None, alias_file=None, av_file=None):
        """Class to operate on AV labels, such as extracting the most likely family name.

        :param gen_file:
        :param alias_file:
        :param av_file:
        """
        # Read generic token set from file
        try:
            if gen_file is None:
                gen_file = get_data_file_path("default.generics")
            self.gen_set = self.read_generics(gen_file)
        except:
            logging.warning(traceback.format_exc())
            self.gen_set = set()

        # Read aliases map from file
        try:
            if alias_file is None:
                alias_file = get_data_file_path("default.aliases")
            self.aliases_map = self.read_aliases(alias_file)
        except:
            logging.warning(traceback.format_exc())
            self.aliases_map = {}

        # Read AV engine set from file
        self.avs = self.read_avs(av_file) if av_file else None

    @staticmethod
    def read_aliases(alfile):
        """Read aliases map from given file"""
        if alfile is None:
            return {}
        almap = {}
        with open(alfile, 'r') as fd:
            for line in fd:
                alias, token = line.strip().split()[0:2]
                almap[alias] = token
        return almap

    @staticmethod
    def read_generics(generics_file):
        """Read generic token set from given file"""
        gen_set = set()
        with open(generics_file) as gen_fd:
            for line in gen_fd:
                if line.startswith('#') or line == '\n':
                    continue
                gen_set.add(line.strip())
        return gen_set

    @staticmethod
    def read_avs(avs_file):
        """Read AV engine set from given file"""
        with open(avs_file) as fd:
            avs = set(map(str.strip, fd.readlines()))
        return avs

    @staticmethod
    def get_sample_info(data):
        """Parse and extract sample information from JSON data
           Returns a SampleInfo named tuple: md5, sha1, sha256, label_pairs

           This method has been improved to handle multiple JSON data formats
           Recognized formats include:
           - VT File Report
           - VT Notification
           - AVClass simplified JSON
        """
        def clean(value):
            return "".join([x for x in value if x in string.printable]).strip()

        if isinstance(data, Object):
            if data.type == 'file':
                label_pairs = list(
                    map(lambda r: (r['engine_name'], clean(r['result'])),
                        filter(lambda x: x['result'] is not None,
                               data.last_analysis_results.values()))
                )
                return SampleInfo(data.md5, data.sha1, data.sha256, label_pairs)
        elif isinstance(data, dict):
            try:
                if "response_code" in data:
                    # VT file report

                    if data["response_code"] == 0:
                        return None
                    label_pairs = [(av, clean(result["result"])) for av, result in data["scans"].items() if
                                   result["result"] is not None]
                elif "ruleset_name" in data:
                    # VT notification

                    label_pairs = [(av, clean(result)) for av, result in data["scans"].items() if result is not None]
                else:
                    label_pairs = data["av_labels"]
            except KeyError:
                return None

            return SampleInfo(str(data['md5']), str(data['sha1']), str(data['sha256']), label_pairs)
        return None

    @staticmethod
    def is_pup(av_label_pairs):
        """This function classifies the sample as PUP or not
           using the AV labels as explained in the paper:
           "Certified PUP: Abuse in Authenticode Code Signing"
           (ACM CCS 2015)
           It uses the AV labels of 11 specific AVs.
           The function checks for 13 keywords used to indicate PUP.
           Return:
              True/False/None
        """
        # If no AV labels, nothing to do, return
        if not av_label_pairs:
            return None
        # Initialize
        pup = False
        threshold = 0.5
        # AVs to use
        av_set = {'Malwarebytes', 'K7AntiVirus', 'Avast', 'AhnLab-V3', 'Kaspersky', 'K7GW', 'Ikarus', 'Fortinet',
                  'Antiy-AVL', 'Agnitum', 'ESET-NOD32'}
        # Tags that indicate PUP
        tags = {'PUA', 'Adware', 'PUP', 'Unwanted', 'Riskware', 'grayware', 'Unwnt', 'Adknowledge', 'toolbar', 'casino',
                'casonline', 'AdLoad', 'not-a-virus'}

        # Set with (AV name, Flagged/not flagged as PUP), for AVs in av_set
        bool_set = set([(pair[0], t.lower() in pair[1].lower()) for t in tags
                        for pair in av_label_pairs
                        if pair[0] in av_set])

        # Number of AVs that had a label for the sample
        av_detected = len([p[0] for p in av_label_pairs
                           if p[0] in av_set])

        # Number of AVs that flagged the sample as PUP
        av_pup = list(map(lambda x: x[1], bool_set)).count(True)  # python 2/3, inefficient on Py2

        # Flag as PUP according to a threshold
        if (float(av_pup) >= float(av_detected) * threshold) and av_pup != 0:
            pup = True
        return pup

    @staticmethod
    def __remove_suffixes(av_name, label):
        """Remove AV specific suffixes from given label

        :param av_name:
        :param label:
        :return: updated label
        """

        # Truncate after last '.'
        if av_name in {'Norman', 'Avast', 'Avira', 'Kaspersky', 'ESET-NOD32', 'Fortinet', 'Jiangmin', 'Comodo', 'GData',
                       'Avast', 'Sophos', 'TrendMicro-HouseCall', 'TrendMicro', 'NANO-Antivirus', 'Microsoft'}:
            label = label.rsplit('.', 1)[0]

        # Truncate after last '.'
        # if suffix only contains digits or uppercase (no lowercase) chars
        if av_name == 'AVG':
            tokens = label.rsplit('.', 1)
            if len(tokens) > 1 and re.match("^[A-Z0-9]+$", tokens[1]):
                label = tokens[0]

        # Truncate after last '!'
        if av_name in {'Agnitum', 'McAffee', 'McAffee-GW-Edition'}:
            label = label.rsplit('!', 1)[0]

        # Truncate after last '('
        if av_name in {'K7AntiVirus', 'K7GW'}:
            label = label.rsplit('(', 1)[0]

        # Truncate after last '@'
        # GData would belong here, but already trimmed earlier
        if av_name in {'Ad-Aware', 'BitDefender', 'Emsisoft', 'F-Secure', 'Microworld-eScan'}:
            label = label.rsplit('(', 1)[0]

        return label

    def update_aliases(self, alias_file):
        try:
            alias_map = AvLabels.read_aliases(alias_file)
            self.aliases_map.update(alias_map)
        except:
            logging.warning(traceback.format_exc())

    def __normalize(self, label, hashes):
        """Tokenize label, filter tokens, and replace aliases"""

        # If empty label, nothing to do
        if not label:
            return []

        # Initialize list of tokens to return
        ret = []

        # Split label into tokens and process each token
        for token in re.split("[^0-9a-zA-Z]", label):
            # Convert to lowercase
            token = token.lower()

            # Remove digits at the end
            end_len = len(re.findall("\d*$", token)[0])
            if end_len:
                token = token[:-end_len]

            # Ignore short token
            if len(token) < 4:
                continue

            # Remove generic tokens
            if token in self.gen_set:
                continue

            # Ignore token if prefix of a hash of the sample
            # Most AVs use MD5 prefixes in labels,
            # but we check SHA1 and SHA256 as well
            hash_token = False
            for hash_str in hashes:
                if hash_str[0:len(token)] == token:
                    hash_token = True
                    break
            if hash_token:
                continue

            # Replace alias
            token = self.aliases_map[token] if token in self.aliases_map \
                else token

            # Add token
            ret.append(token)
        return ret

    def get_family_ranking(self, sample_info):
        """Returns sorted dictionary of most likely family names for sample

        :param sample_info:
        :return:
        """
        # Extract info from named tuple
        av_label_pairs = sample_info[3]
        hashes = [sample_info[0], sample_info[1], sample_info[2]]

        # Whitelist the AVs to filter the ones with meaningful labels
        av_whitelist = self.avs

        # Initialize auxiliary data structures
        labels_seen = set()
        token_map = {}

        # Process each AV label
        for (av_name, label) in av_label_pairs:
            # If empty label, nothing to do
            if not label:
                continue

            ################
            # AV selection #
            ################
            if av_whitelist and av_name not in av_whitelist:
                continue

            #####################
            # Duplicate removal #
            #####################

            # If label ends in ' (B)', remove it
            if label.endswith(' (B)'):
                label = label[:-4]

            # If we have seen the label before, skip
            if label in labels_seen:
                continue
            # If not, we add it to the set of labels seen
            else:
                labels_seen.add(label)

            ##################
            # Suffix removal #
            ##################
            label = self.__remove_suffixes(av_name, label)

            ########################################################
            # Tokenization, token filtering, and alias replacement #
            ########################################################
            tokens = self.__normalize(label, hashes)

            # Increase token count in map
            for t in tokens:
                c = token_map[t] if t in token_map else 0
                token_map[t] = c + 1

        ##################################################################
        # Token ranking: sorts tokens by decreasing count and then token #
        ##################################################################
        sorted_tokens = sorted(token_map.items(),
                               key=lambda x: x[1] or 0,
                               reverse=True)

        # Delete the tokens appearing only in one AV, add rest to output
        sorted_dict = OrderedDict()
        for t, c in sorted_tokens:
            if c > 1:
                sorted_dict[t] = c
            else:
                break

        return sorted_dict


#!/usr/bin/env python

from lxml.etree import XMLParser, fromstring, XMLSchema

schema_doc = open('schema.xsd').read()
inst_doc = open('inst.xml').read()

parser = XMLParser(resolve_entities=False)
elt = fromstring(inst_doc, parser)
schema = XMLSchema(fromstring(schema_doc))
schema.validate(elt)


from django.shortcuts import render
from django.http import HttpResponseRedirect
from django import forms

# Create your views here.
def bmi(request):

    if request.method == 'POST':
        height =  request.POST.get('height','')
        h_unit = request.POST.get('unit_height', '')
        weight = request.POST.get('weight','')
        w_unit = request.POST.get('unit_weight','')

        bmi_val = float(weight)/(float(height)/100)**2
    return render(request,'bmi.html', locals())


from typing import List
from collections import defaultdict
from heapq import heappush, heappop

class Solution:
    def topKFrequent(self, nums: List[int], k: int) -> List[int]:
        freq = defaultdict(int)
        heap = []
        for n in nums:
            freq[n] += 1
        for n in freq:
            heappush(heap, (-freq[n], n))
        res = []
        for _ in range(k):
            _, val = heappop(heap)
            res.append(val)
        return res

if __name__ == '__main__':
    nums = [1, 1, 1, 2, 2, 3]
    s = Solution()
    print(s.topKFrequent(nums, 2))