import logging
import queue
import re

from .mdlist import MangaDexList
from .errors import HTTPException, InvalidURL, MangaDexException, NotLoggedIn
from .network import Net, base_url
from .manga import ContentRating, Manga
from .fetcher import get_list
from .user import User
from .language import get_language
from .config import ConfigTypeError, _validate_bool as validate_boolean
from .utils import validate_url, get_local_attr

log = logging.getLogger(__name__)

class BaseIterator:
    def __init__(self):
        self.queue = queue.Queue()
        self.offset = 0

    def __iter__(self):
        return self

    def __next__(self):
        if self.queue.empty():
            # Maximum number of results from MangaDex API
            if self.offset >= 10000:
                raise StopIteration()
            else:
                self.fill_data()

        try:
            return self.next()
        except queue.Empty:
            raise StopIteration()

    def fill_data(self):
        raise NotImplementedError

    def next(self):
        return self.queue.get_nowait()

class SearchFilterError(MangaDexException):
    def __init__(self, key, msg):
        text = f"Search filter error '{key}' = {msg}"

        super().__init__(text)

class IteratorManga(BaseIterator):
    def __init__(
        self,
        title,
        authors=None,
        artists=None,
        year=None,
        included_tags=None,
        included_tags_mode=None,
        excluded_tags=None,
        excluded_tags_mode=None,
        status=None,
        original_language=None,
        excluded_original_language=None,
        available_translated_language=None,
        publication_demographic=None,
        content_rating=None,
        created_at_since=None,
        updated_at_since=None,
        has_available_chapters=None,
        group=None,
        order=None
    ):
        super().__init__()

        _default_content_ratings = [
            'safe',
            'suggestive',
            'erotica',
            'pornographic'
        ]

        self.limit = 100
        self.title = title

        # Validation
        value_and_or = ['AND', 'OR']

        if year:
            m = re.match(r'[0-9]{4}')
            if not m:
                raise SearchFilterError(
                    "year",
                    f"value must be integer and length must be 4"
                )

        _locals = locals()
        def validate_tags_mode(key):
            value = _locals[key]
            if value and value.upper() not in value_and_or:
                raise SearchFilterError(
                    key,
                    f"value must be 'OR' or 'AND', not '{value}'"
                )

        validate_tags_mode("included_tags_mode")
        validate_tags_mode("excluded_tags_mode")

        def validate_uuid(key):
            new_values = []
            values = _locals[key]
            if values is None:
                return

            if isinstance(values, str):
                values = [values]

            for value in values:
                # Get the id
                try:
                    _id = validate_url(value)
                except InvalidURL:
                    raise SearchFilterError(
                        key,
                        f"'{value}' is not valid UUID"
                    )
                else:
                    new_values.append(_id)

            return new_values

        group = validate_uuid("group")

        tags = self._get_tags()
        def validate_tags(key):
            new_values = []
            values = _locals[key]
            if values is None:
                return

            if isinstance(values, str):
                values = [values]

            # Lowercase to prevent error
            values = [i.lower() for i in values]

            for value in values:
                # Try to match the keyword tags
                try:
                    _id = tags[value]
                except KeyError:
                    pass
                else:
                    new_values.append(_id)
                    continue

                # Try to get uuid
                try:
                    _id = validate_url(value)
                except InvalidURL:
                    raise SearchFilterError(
                        key,
                        f"'{value}' is not valid keyword or uuid tag"
                    )

                new_values.append(_id)

            return new_values

        included_tags = validate_tags("included_tags")
        excluded_tags = validate_tags("excluded_tags")

        def validate_values_from_list(key, array):
            values = _locals[key]
            if values is None:
                return

            if isinstance(values, str):
                values = [values]

            for value in values:
                if value.lower() not in array:
                    raise SearchFilterError(
                        key,
                        f"Value must be one of {array}, not {value}"
                    )

        _status_values = [
            'ongoing',
            'completed',
            'hiatus',
            'cancelled'
        ]
        validate_values_from_list("status", _status_values)

        def validate_language(key):
            new_values = []
            values = _locals[key]
            if values is None:
                return

            if isinstance(values, str):
                values = [values]

            for value in values:
                try:
                    lang = get_language(value)
                except ValueError as e:
                    raise SearchFilterError(key, e)
                else:
                    new_values.append(lang.value)

            return new_values

        original_language = validate_language("original_language")
        excluded_original_language = validate_language("excluded_original_language")
        available_translated_language = validate_language("available_translated_language")

        _pub_demo_values = [
            'shounen',
            'shoujo',
            'josei',
            'seinen',
            'none'
        ]
        validate_values_from_list("publication_demographic", _pub_demo_values)

        _content_rating_values = [a.value for a in ContentRating]
        validate_values_from_list("content_rating", _content_rating_values)

        if has_available_chapters:
            try:
                validate_boolean(has_available_chapters)
            except ConfigTypeError as e:
                raise SearchFilterError("has_available_chapters", e)

        # Validate orders
        def validate_order(order):
            new_order = {}
            ascending = ['asc', 'ascending']
            descending = ['desc', 'descending']
            for key, value in order.items():
                # Validate order keys
                re_order_key = r'order\[(' \
                               r'title|' \
                               r'year|' \
                               r'createdAt|' \
                               r'updatedAt|' \
                               r'latestUploadedChapter|' \
                               r'followedCount|' \
                               r'relevance|' \
                               r'rating|' \
                               r')\]'
                match = re.match(re_order_key, key)
                if match is None:
                    raise SearchFilterError(
                        key,
                        "Invalid order key"
                    )

                if value in ascending:
                    new_order[key] = ascending[0]
                elif value in descending:
                    new_order[key] = descending[0]
                else:
                    raise SearchFilterError(
                        key,
                        f"invalid value must be one of {ascending} or {descending}"
                    )

            return new_order

        order = validate_order(order)

        self._param_init = {
            "authors[]": authors,
            "artists[]": artists,
            "year": year,
            "includedTags[]": included_tags,
            "includedTagsMode": included_tags_mode,
            "excludedTags[]": excluded_tags,
            "excludedTagsMode": excluded_tags_mode,
            "status[]": status,
            "originalLanguage[]": original_language,
            "excludedOriginalLanguage[]": excluded_original_language,
            "availableTranslatedLanguage[]": available_translated_language,
            "publicationDemographic[]": publication_demographic,
            "contentRating[]": content_rating or _default_content_ratings,
            "createdAtSince": created_at_since,
            "updatedAtSince": updated_at_since,
            "hasAvailableChapters": has_available_chapters,
            "group": group,
        }

        self._param_init.update(**order)

    def _get_tags(self):
        tags = {}
        r = Net.mangadex.get(f'{base_url}/manga/tag')
        data = r.json()

        for item in data['data']:
            _id = item['id']
            attr = item['attributes']
            name = get_local_attr(attr['name']).lower()
            tags[name] = _id

        return tags

    def _get_params(self):
        includes = ['author', 'artist', 'cover_art']

        params = {
            'includes[]': includes,
            'title': self.title,
            'limit': self.limit,
            'offset': self.offset,
        }
        params.update(self._param_init.copy())

        return params

    def fill_data(self):
        params = self._get_params()
        url = f'{base_url}/manga'
        r = Net.mangadex.get(url, params=params)
        data = r.json()

        if r.status_code >= 400:
            err = data['errors'][0]['detail']
            raise MangaDexException(err)

        items = data['data']

        for item in items:
            self.queue.put(Manga(data=item))

        self.offset += len(items)

class IteratorUserLibraryManga(BaseIterator):
    statuses = [
        'reading',
        'on_hold',
        'plan_to_read',
        'dropped',
        're_reading',
        'completed'
    ]

    def __init__(self, status=None):
        super().__init__()

        self.limit = 100
        self.offset = 0

        if status is not None and status not in self.statuses:
            raise MangaDexException(f"{status} are not valid status, choices are {set(self.statuses)}")

        self.status = status

        lib = {}
        for stat in self.statuses:
            lib[stat] = []
        self.library = lib

        logged_in = Net.mangadex.check_login()
        if not logged_in:
            raise NotLoggedIn("Retrieving user library require login")

        self._parse_reading_status()

    def _parse_reading_status(self):
        r = Net.mangadex.get(f'{base_url}/manga/status')
        data = r.json()

        for manga_id, status in data['statuses'].items():
            self.library[status].append(manga_id)

    def _check_status(self, manga):
        if self.status is None:
            return True

        manga_ids = self.library[self.status]
        return manga.id in manga_ids

    def next(self) -> Manga:
        while True:
            manga = super().next()

            if not self._check_status(manga):
                # Filter is used
                continue

            return manga

    def fill_data(self):
        includes = [
            'artist', 'author', 'cover_art'
        ]
        params = {
            'includes[]': includes,
            'limit': self.limit,
            'offset': self.offset,
        }
        url = f'{base_url}/user/follows/manga'
        r = Net.mangadex.get(url, params=params)
        data = r.json()

        items = data['data']

        for item in items:
            self.queue.put(Manga(data=item))

        self.offset += len(items)

class IteratorMangaFromList(BaseIterator):
    def __init__(self, _id=None, data=None):
        if _id is None and data is None:
            raise ValueError("atleast provide _id or data")
        elif _id and data:
            raise ValueError("_id and data cannot be together")

        super().__init__()

        self.id = _id
        self.data = data
        self.limit = 100
        self.name = None # type: str
        self.user = None # type: User

        self.manga_ids = []

        self._parse_list()

    def _parse_list(self):
        if self.id:
            data = get_list(self.id)['data']
        else:
            data = self.data

        self.name = data['attributes']['name']

        for rel in data['relationships']:
            _type = rel['type']
            _id = rel['id']
            if _type == 'manga':
                self.manga_ids.append(_id)
            elif _type == 'user':
                self.user = User(_id)

    def fill_data(self):
        ids = self.manga_ids
        includes = ['author', 'artist', 'cover_art']
        content_ratings = [
            'safe',
            'suggestive',
            'erotica',
            'pornographic' # Filter porn content will be done in next()
        ]

        limit = self.limit
        if ids:
            param_ids = ids[:limit]
            del ids[:len(param_ids)]
            params = {
                'includes[]': includes,
                'limit': limit,
                'contentRating[]': content_ratings,
                'ids[]': param_ids
            }
            url = f'{base_url}/manga'
            r = Net.mangadex.get(url, params=params)
            data = r.json()

            notexist_ids = param_ids.copy()
            copy_data = data.copy()
            for manga_data in copy_data['data']:
                manga = Manga(data=manga_data)
                if manga.id in notexist_ids:
                    notexist_ids.remove(manga.id)

            if notexist_ids:
                for manga_id in notexist_ids:
                    log.warning(f'There is ghost (not exist) manga = {manga_id} in list {self.name}')

            for manga_data in data['data']:
                self.queue.put(Manga(data=manga_data))

class IteratorUserLibraryList(BaseIterator):
    def __init__(self):
        super().__init__()

        self.limit = 100
        self.offset = 0

        logged_in = Net.mangadex.check_login()
        if not logged_in:
            raise NotLoggedIn("Retrieving user library require login")

    def fill_data(self):
        params = {
            'limit': self.limit,
            'offset': self.offset,
        }
        url = f'{base_url}/user/list'
        r = Net.mangadex.get(url, params=params)
        data = r.json()

        items = data['data']

        for item in items:
            self.queue.put(MangaDexList(data=item))

        self.offset += len(items)

class IteratorUserList(BaseIterator):
    def __init__(self, _id=None):
        super().__init__()

        self.limit = 100
        self.user = User(_id)

    def fill_data(self):
        params = {
            'limit': self.limit,
            'offset': self.offset,

        }
        url = f'{base_url}/user/{self.user.id}/list'
        try:
            r = Net.mangadex.get(url, params=params)
        except HTTPException:
            # Some users are throwing server error (Bad gateway)
            # MD devs said it was cache and headers issues
            # Reference: https://api.mangadex.org/user/10dbf775-1935-4f89-87a5-a1f4e64d9d94/list
            # For now the app will throw error and tell the user cannot be fetched until it's get fixed

            # HTTPException from session only giving "server throwing ... code" message
            raise HTTPException(
                f"An error occured when getting mdlists from user \"{self.user.id}\". " \
                f"The app cannot fetch all MangaDex lists from user \"{self.user.id}\" " \
                "because of server error. The only solution is to wait until this get fixed " \
                "from MangaDex itself."
            ) from None

        data = r.json()

        items = data['data']

        for item in items:
            self.queue.put(MangaDexList(data=item))

        self.offset += len(items)

class IteratorUserLibraryFollowsList(BaseIterator):
    def __init__(self):
        super().__init__()

        self.limit = 100

        logged_in = Net.mangadex.check_login()
        if not logged_in:
            raise NotLoggedIn("Retrieving user library require login")

    def fill_data(self):
        params = {
            'limit': self.limit,
            'offset': self.offset,
        }
        url = f'{base_url}/user/follows/list'
        r = Net.mangadex.get(url, params=params)
        data = r.json()

        items = data['data']

        for item in items:
            self.queue.put(MangaDexList(data=item))

        self.offset += len(items)

class IteratorSeasonalManga(IteratorMangaFromList):
    owner_list = 'd2ae45e0-b5e2-4e7f-a688-17925c2d7d6b'

    def __init__(self, season):
        seasons = self._get_seasons()

        try:
            mdlist = seasons[season]
        except KeyError:
            raise MangaDexException(f"invalid season, available choices are {list(seasons.keys())}")

        super().__init__(mdlist.id)

    @classmethod
    def _get_seasons(self):
        seasons = {}
        for mdlist in IteratorUserList(self.owner_list):
            name = mdlist.name.lower().replace('seasonal: ', '')
            seasons[name] = mdlist

        return seasons

def iter_random_manga(content_ratings):
    ids = []
    while True:
        params = {
            'includes[]': ['author', 'artist', 'cover_art'],
            "contentRating[]": content_ratings
        }
        r = Net.mangadex.get(f'{base_url}/manga/random', params=params)
        data = r.json()['data']
        manga = Manga(data=data)

        if manga.id not in ids:
            # Make sure it's not duplicated manga
            ids.append(manga.id)
            yield manga

        continue


from aws_cdk import (
    # Duration,
    Stack, Fn,
    aws_rds,
    aws_iam,
    aws_ec2,
    aws_eks,
    RemovalPolicy, SecretValue,
    aws_lambda,
    aws_apigateway as apigw
)

from constructs import Construct
cidr="10.10.0.0/16"
key_name = "my-key-pair"

class SaasSolutionCdkStack(Stack):

    def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
        super().__init__(scope, construct_id, **kwargs)

        azs = Fn.get_azs()
        self.vpc = aws_ec2.Vpc(self, "VPC",
                           max_azs=2,
                           ip_addresses=aws_ec2.IpAddresses.cidr(cidr),
                           nat_gateways=2,
                           enable_dns_hostnames=True,
                           enable_dns_support=True,
                           subnet_configuration=[
                               aws_ec2.SubnetConfiguration(
                                   name="public",
                                   subnet_type=aws_ec2.SubnetType.PUBLIC,
                                   cidr_mask=24),
                               aws_ec2.SubnetConfiguration(
                                   subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_EGRESS,
                                   name="private",
                                   cidr_mask=24) # could be /16 to have more instances, but this is a demo scope.
                           ]
                           )

        postgres = aws_rds.DatabaseInstance(self, "PostgresqlInstance",
                                database_name="tenantdb",
                                engine=aws_rds.DatabaseInstanceEngine.postgres(version=aws_rds.PostgresEngineVersion.VER_14_5),
                                vpc_subnets=aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_EGRESS),
                                vpc=self.vpc,
                                port=5432,
                                removal_policy=RemovalPolicy.DESTROY,
                                deletion_protection=False,
                                max_allocated_storage=200,
                                publicly_accessible=True
                        )

        # Create an IAM role for worker groups and kubernetes RBAC configuration
        self.eks_admin_role = aws_iam.Role(self, 'eksAdmin',
                                    assumed_by=aws_iam.ServicePrincipal(service='ec2.amazonaws.com'),
                                    role_name='eks-cluster-role',
                                    managed_policies=
                                        [aws_iam.ManagedPolicy.from_aws_managed_policy_name(managed_policy_name='AdministratorAccess')])
        self.eks_instance_profile = aws_iam.CfnInstanceProfile(self, 'instanceprofile',
                                                      roles=[self.eks_admin_role.role_name],
                                                      instance_profile_name='eks-cluster-role')

        cluster = aws_eks.Cluster(self, 'demo-cluster',
                                  masters_role=self.eks_admin_role,
                                  vpc=self.vpc,
                                  default_capacity=2,
                                  vpc_subnets=[aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_EGRESS)],
                                  version=aws_eks.KubernetesVersion.V1_24,
                                  output_cluster_name=True
                                  )


import json

from Products.ZenRRD.CommandParser import CommandParser

class table(CommandParser):
    def processResults(self, cmd, result):
        data = None
        try:
            data = json.loads(cmd.result.output)
        except ValueError:
            return

        if 'databases' not in data:
            return result

        for point in cmd.points:
            component = cmd.points[0].component

            table = None
            for dbName, dbStats in data['databases'].items():
                if table is not None:
                    break

                for tableName, tableStats in dbStats['tables'].items():
                    component_id = '{0}_{1}'.format(dbName, tableName)
                    if component_id == component:
                        table = tableStats
                        break

            if table is None:
                # No matching table found.
                continue

            if point.id in table:
                result.values.append((point, table[point.id]))

        return result


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.14.5
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# %% [markdown]
# # Count peptides over all files

# %%
import os
import sys
import logging
from pathlib import Path
import random
import yaml
import json

import pandas as pd
import ipywidgets as widgets

### Logging setup ######
from vaep.logging import setup_nb_logger
setup_nb_logger()

### vaep imports ######
from vaep.io.mq import MaxQuantOutputDynamic
from vaep.io.data_objects import MqAllSummaries
from vaep.io.data_objects import PeptideCounter
import vaep.pandas

##################
##### CONFIG #####
##################
from config import FOLDER_MQ_TXT_DATA, FOLDER_PROCESSED

from config import FOLDER_DATA # project folder for storing the data
logging.info(f"Search Raw-Files on path: {FOLDER_MQ_TXT_DATA}")

# %% [markdown]
# Use samples previously loaded.

# %%
ELIGABLE_FILES_YAML = Path('config/eligable_files.yaml')
MAP_FOLDER_PATH = Path('config/file_paths')

with open(ELIGABLE_FILES_YAML) as f:
    files = set(yaml.safe_load(f)['files'])
    logging.info(f"Found a total of {len(files):,d} eligable files.")
with open(MAP_FOLDER_PATH) as f:
    folders_dict = yaml.safe_load(f)
    folders_dict = {folder: folders_dict[folder] for folder in files}  # only select folders selected

folders = [Path(folders_dict[folder]) for folder in files]
assert len(files) == len(folders_dict) == len(folders)

# %%
fn_id_old_new: str = 'data/rename/selected_old_new_id_mapping.csv' # selected samples with pride and original id
df_ids = pd.read_csv(fn_id_old_new)
df_ids

# %%
folders_dict = { sample_id: FOLDER_MQ_TXT_DATA / sample_id for sample_id in df_ids['Sample ID']}
# folders_dict = {p.stem : p.parent / p.stem for p in folders_dict}
# folders_dict

# %%
OVERWRITE = False

from config import FNAME_C_PEPTIDES, FNAME_C_EVIDENCE, FNAME_C_PG, FNAME_C_GENES

FNAME_C_PEPTIDES, FNAME_C_EVIDENCE, FNAME_C_PG, FNAME_C_GENES

# %% [markdown]
# ## Random example

# %%
import random
pd.set_option('display.max_columns', 60)
random_folder, random_path = random.sample(folders_dict.items(), 1)[0]
mq_output = MaxQuantOutputDynamic(random_path)
print(f"peptides.txt from {random_folder!s}")
mq_output.peptides

# %%
use_columns = mq_output.peptides.columns[33:45]
df = mq_output.peptides[use_columns].convert_dtypes() #.to_json('test.json')
df

# %%
df_json_string = df.to_json(orient='index', indent=4)
df_json_string[:1000]

# %%
df_csv = df.to_csv()
df_csv[:1000]

# %%
pd.read_json(df_json_string, orient='index')

# %%
mq_output.peptides.Intensity # as is in peptides.txt, comma seperated thousands

# %% [markdown]
# ## Count aggregated peptides

# %%
peptide_counter = PeptideCounter(FNAME_C_PEPTIDES, overwrite=OVERWRITE)
peptide_counter

# %%
if peptide_counter.loaded:
    print(peptide_counter.counter.most_common(10),
          len(peptide_counter.loaded),
          sep='\n')
else:
    print('New file created.')

# %% [markdown]
# - creates peptide intensity dumps for each MQ outputfolder per default `count_peptides` function (default processing function for `PeptideCounter`)

# %%
# %%time
# folders = [Path(folder_path) for folder_path in folders_dict.values()]
c = peptide_counter.sum_over_files(folders=folders)

# %%
c.most_common(10) # peptide_counter.counter.most_common(10)

# %%
# To share as python file
N = 1000
with open(FOLDER_PROCESSED / f'most_common_{10}_peptides.py', 'w') as f:
    f.write('import pandas as pd\n\n')

    #pprint.pformat list -> do this using standardlibrary
    # https://docs.python.org/3/library/pprint.html
    f.write(f"most_common = [\n  ")
    f.write(',\n  '.join(f"{str(t)}" for t in c.most_common(N)))
    f.write("\n]\n\n")

    #peptide_counter.loaded()

    f.write("pd.DataFrame.from_records(most_common, index='Sequence', columns=['Sequence', 'counts'])\n")

# %% [markdown] Collapsed="false"
# ## Peptides by charge
#
# - count peptides by charge state (which are aggregated in `peptides.txt`)

# %%
evidence_cols = vaep.pandas.get_columns_accessor(mq_output.evidence.reset_index())
evidence_cols # vaep.mq get this list

# %%
evidence = mq_output.evidence.set_index(evidence_cols.Charge, append=True)
evidence

# %% [markdown]
# Modifikationen könnten noch zum index hinzugefügt werden

# %%
evidence.Modifications.value_counts()

# %%
vaep.pandas.prop_unique_index(evidence)

# %% [markdown]
# Using the protein AA sequence and it's charge as identifiers, does not yield a unique index.
#
# First potential contaminants and peptides with zero intensity (or missing intensity) can be removed from the table.
#
# These are apparently peptides identified by an MS2 spectrum but which could not be quantified by a MS1 scans

# %%
mask =  evidence[evidence_cols.Intensity].isna()
evidence.loc[mask, evidence_cols.Type].value_counts()

# %%
evidence_cols = vaep.io.data_objects.evidence_cols
use_cols = [evidence_cols.mz, evidence_cols.Protein_group_IDs, evidence_cols.Intensity, evidence_cols.Score, evidence_cols.Potential_contaminant]

evidence_selected = vaep.io.data_objects.select_evidence(evidence[use_cols])
evidence_selected

# %%
evidence_selected = evidence_selected.sort_values(by=['Sequence', 'Charge', 'Score'], ascending=False)
evidence_selected

# %%
evidence_selected = vaep.pandas.select_max_by(evidence_selected.reset_index(), [evidence_cols.Sequence, evidence_cols.Charge], evidence_cols.Score)
evidence_selected

# %%
from collections import Counter
c = Counter()
c.update(evidence.index)
c.most_common(10)

# %%
example = evidence.loc[c.most_common(10)[0][0]]

vaep.pandas.show_columns_with_variation(example)

# %% [markdown]
# - `Type`: only `MULTI-MSMS` and `MULIT-SECPEP` are quantified (does this mean a matching MS1 spectrum?)

# %%
evidence[evidence_cols.Type].value_counts()

# %% [markdown]
# Some peptides can be assigned to different protein group IDs (razor peptides)
#  - option: discared non-unique peptides (and Protein group IDs can be already a combination of several isotopes)
#  - option: select on `Score` or `Intensity` (is there a relationship?)
#  - option: select based on `Number of isotopic peaks`

# %%
evidence[evidence_cols.Protein_group_IDs].value_counts()

# %% [markdown]
# ## Count peptides based on evidence files

# %%
evidence_counter = vaep.io.data_objects.EvidenceCounter(FNAME_C_EVIDENCE, overwrite=OVERWRITE)
c = evidence_counter.sum_over_files(folders=folders)

# %% [markdown]
# ## Protein Groups
#
# - protein groups between files
#     - aggregate by GENE ?
#     -

# %%
mq_output.proteinGroups.describe(include='all')

# %%
pg_cols = vaep.pandas.get_columns_accessor(mq_output.proteinGroups.reset_index())
pg_cols

# %%
use_cols = [
# pg_cols.Protein_IDs,
 pg_cols.Majority_protein_IDs,
 pg_cols.Gene_names,
 pg_cols.Evidence_IDs,
 pg_cols.Q_value,
 pg_cols.Score,
 pg_cols.Only_identified_by_site,
 pg_cols.Reverse,
 pg_cols.Potential_contaminant,
 pg_cols.Intensity,
]

pd.options.display.max_rows = 100
pd.options.display.min_rows = 40
mask = mq_output.proteinGroups[[pg_cols.Only_identified_by_site, pg_cols.Reverse, pg_cols.Potential_contaminant]].notna().sum(axis=1) > 0
mq_output.proteinGroups.loc[mask, use_cols]

# %%
msg = "Omitting the data drops {0:.3f} % of the data."
print(msg.format(
mask.sum() / len(mask) * 100
))

# %%
selection = mq_output.proteinGroups.loc[~mask, use_cols]
gene_counts = selection[pg_cols.Gene_names].value_counts() # Gene Names not unique
msg = 'proportion of entries with non-unique genes: {:.3f}'
print(msg.format(gene_counts.loc[gene_counts > 1].sum() / gene_counts.sum()))
gene_counts.head(20)

# %%
mask = selection.Intensity > 0
msg = "Proportion of non-zero Intensities: {:.3f} (zero_ count = {})"
print(msg.format(mask.sum() / len(mask), (~mask).sum()))
selection.loc[~mask]

# %%
selection = selection.loc[mask]

# %% [markdown]
# Some Proteins have no gene annotation
#   - P56181 -> mitochondrial
#
# In the online version of Uniprot these seems to be annotated (brief check).
# So latest version probably has a gene annotation, so therefore these files are kept

# %%
gene_set = selection[pg_cols.Gene_names].str.split(';')

col_loc_gene_names = selection.columns.get_loc(pg_cols.Gene_names)
_ = selection.insert(col_loc_gene_names+1, 'Number of Genes', gene_set.apply(vaep.pandas.length))

mask = gene_set.isna()
selection.loc[mask]

# %%
cols = vaep.pandas.get_columns_accessor(selection)
gene_counts = vaep.pandas.counts_with_proportion(selection[cols.Number_of_Genes])
gene_counts

# %% [markdown]
# Most `proteinGroups` have single genes assigned to them. If one only looks at gene sets,
# one can increase uniquely identified `proteinGroups` further.
#
# > Can `geneGroups` (sets of `Gene Names`) be used instead of `proteinGroups`?

# %%
gene_sets_counts = selection[cols.Gene_names].value_counts()
gene_sets_counts.value_counts()

# %% [markdown]
# Potential solutions:
# - summarize intensity per gene. One of the isoforms seems to have the major proportion of intensity assigned.
# - select maximum by score (higher scores seem to be related to higher intensity)

# %%
non_unique_genes = gene_sets_counts.loc[gene_sets_counts > 1].index

mask = selection[cols.Gene_names].isin(non_unique_genes)
selection.loc[mask].reset_index().set_index(cols.Gene_names).sort_index()

# %% [markdown]
# Protein Groups with Gene set with three and more genes:

# %%
selection.loc[selection[cols.Number_of_Genes] > 2]

# %%
logging.info(f"Selection shape before dropping duplicates by gene: {selection.shape}")
mask_no_gene = selection[pg_cols.Gene_names].isna()
selection_no_gene = selection.loc[mask_no_gene]
logging.info(f"Entries without any gene annotation: {len(selection_no_gene)}")
selection_no_gene

# %%
selection = vaep.pandas.select_max_by(df=selection.loc[~mask_no_gene].reset_index(), grouping_columns=[pg_cols.Gene_names], selection_column=pg_cols.Score)
logging.info(f"Selection shape after  dropping duplicates by gene: {selection.shape}")
selection = selection.set_index(pg_cols.Protein_IDs)
mask = selection[cols.Gene_names].isin(non_unique_genes)
selection.loc[mask]

# %%
selection = selection.append(selection_no_gene)

# %%
protein_groups_counter = vaep.io.data_objects.ProteinGroupsCounter(FNAME_C_PG, overwrite=OVERWRITE)
c = protein_groups_counter.sum_over_files(folders=folders)

# %%
vaep.pandas.counts_with_proportion(pd.Series(c)) # Most proteinGroups are unique

# %% [markdown]
# ### Count genes
# Genes sets could be used to identify common features.
#
# > The assignment of isoforms to one proteinGroup or another might be volatile.
# > A single (unique) peptide could lead to different assignments.
# > Imputation on the evidence level could be a way to alleviate this problem
#
# - If genes set are not unique for a single run, one would have to decide which to take

# %%
gene_counter = vaep.io.data_objects.GeneCounter(FNAME_C_GENES, overwrite=OVERWRITE)

if not gene_counter.dumps:
    #empty dict, replace
    gene_counter.dumps = dict(protein_groups_counter.dumps) # prot proteinGroups files to GeneCounter
pg_dumps = list(gene_counter.dumps.values())

c_genes = gene_counter.sum_over_files(folders=pg_dumps)

c_genes = pd.Series(c_genes)
vaep.pandas.counts_with_proportion(c_genes) # Most proteinGroups are unique

# %% [markdown] Collapsed="false"
# ## Theoretial Peptides from used fasta-file
#
# > `01_explore_FASTA.ipynb` (formely `misc_FASTA_tryptic_digest.ipynb`)

# %% [markdown]
#


# (C) 2021 GoodData Corporation
""" Module containing a class that provides access to metadata and afm services.
"""

from __future__ import annotations

from typing import Optional

import gooddata_afm_client as afm_client
import gooddata_metadata_client as metadata_client
import gooddata_scan_client as scan_client
from gooddata_sdk import __version__

USER_AGENT = f"gooddata-python-sdk/{__version__}"

class GoodDataApiClient:
    """Provide access to metadata and afm services."""

    def __init__(
        self,
        host: str,
        token: str,
        custom_headers: Optional[dict[str, str]] = None,
        extra_user_agent: Optional[str] = None,
    ) -> None:
        """Take url, token for connecting to GoodData.CN.

        HTTP requests made by this class may be enriched by `custom_headers` dict
        containing header names as keys and header values as dict values.

        `extra_user_agent` is optional string to be added to default http User-Agent
        header. This takes precedence over custom_headers setting.
        """
        self._hostname = host
        self._token = token
        self._custom_headers = custom_headers or {}

        user_agent = f"{USER_AGENT} {extra_user_agent}" if extra_user_agent is not None else USER_AGENT

        self._metadata_config = metadata_client.Configuration(host=host)
        self._metadata_client = metadata_client.ApiClient(
            configuration=self._metadata_config,
            header_name="Authorization",
            header_value=f"Bearer {token}",
        )
        self._set_default_headers(self._metadata_client.default_headers)
        for header_name, header_value in self._custom_headers.items():
            self._metadata_client.default_headers[header_name] = header_value
        self._metadata_client.user_agent = user_agent

        self._scan_config = scan_client.Configuration(host=host)
        self._scan_client = scan_client.ApiClient(
            configuration=self._scan_config,
            header_name="Authorization",
            header_value=f"Bearer {token}",
        )
        self._set_default_headers(self._scan_client.default_headers)
        for header_name, header_value in self._custom_headers.items():
            self._scan_client.default_headers[header_name] = header_value
        self._scan_client.user_agent = user_agent

        self._afm_config = afm_client.Configuration(host=host)
        self._afm_client = afm_client.ApiClient(
            configuration=self._afm_config,
            header_name="Authorization",
            header_value=f"Bearer {token}",
        )
        self._set_default_headers(self._afm_client.default_headers)
        for header_name, header_value in self._custom_headers.items():
            self._afm_client.default_headers[header_name] = header_value
        self._afm_client.user_agent = user_agent

    @staticmethod
    def _set_default_headers(headers: dict) -> None:
        headers["X-Requested-With"] = "XMLHttpRequest"
        headers["X-GDC-VALIDATE-RELATIONS"] = "true"

    @property
    def afm_client(self) -> afm_client.ApiClient:
        return self._afm_client

    @property
    def metadata_client(self) -> metadata_client.ApiClient:
        return self._metadata_client

    @property
    def scan_client(self) -> scan_client.ApiClient:
        return self._scan_client


from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.orm import relationship
from .db import db
class Mall(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(50))
    account_id=db.Column(db.Integer, db.ForeignKey('account.id'))

    unit = relationship("Unit", uselist=False, back_populates="mall") # one to one

    def __repr__(self):
        return '<Mall %s>' % self.name


import pandas
MARKER = b'PAR1'
import struct
import msgpack
from io import BytesIO as cStringIO
from array import array
import zlib
import numpy
def differ(first, second):
    return [item for item in first if item not in second]

def globalorder(f,gdict, chunk, ordering):
    output = cStringIO()
    output.truncate(0)
    headindex = [0]*4
    minmax = [None]*2
    minmax[0] = min(chunk)
    minmax[1] = max(chunk)
    type = 'i'*len(headindex)
    output.write(struct.pack(type, *headindex))
    l1 = output.tell()
    output.write(msgpack.dumps(minmax))
    l2 = output.tell()
    headindex[0] = l2-l1
    v = len(gdict)
    newvals = differ(set(list(chunk)), gdict)
    for val in newvals:
          gdict[val] = v
          v+=1

    l3 = output.tell()
    if minmax[0] != minmax[1]:
        offsets = [gdict[y] for y in chunk]
        ll = max(offsets)
        if ll<256:
            type1 = 'B'
            headindex[2] = 2
        elif ll<65536:
            type1 = 'H'
        #type1 = np.int16
            headindex[2] = 1
        else:
            type1 = 'i'
        #type1 = np.int32
        output.write(struct.pack(type1*len(offsets), *offsets))
    else:
    	output.write(struct.pack('i',gdict[minmax[0]]))
    #output.write(array(type1,[coldict[y] for y in chunk]).tostring())
    l4 = output.tell()
    headindex[1] = l4-l3
    headindex[3] = len(chunk)
    output.seek(0)
    output.write(struct.pack(type, *headindex))
    return output.getvalue()

def write_global(data,filen, ordering, row_group_offsets):
    gdict = {}
    blocksdump = []
    f = open(filen,"w+b")
    f.write(MARKER)
    headindex = [0]*4
    l = len(data)
    headindex[0] = l
    headindex[1] = len(data.columns)
    type = 'i'*len(headindex)
    f.write(struct.pack(type, *headindex))
    nparts = max((l - 1) // row_group_offsets + 1, 1)
    chunksize = max(min((l - 1) // nparts + 1, l), 1)
    blocks = [0]*(nparts+1)
    blocks[nparts] = chunksize
    headindex[3] = nparts
    blocksind = f.tell()
    f.write(struct.pack('L'*len(blocks),*blocks))
    for col in data:
        i=0
        j=1
        for part in range(nparts):
            chunk = data[col][chunksize*i:chunksize*j]
            blocksdump.append(globalorder(f,gdict, chunk, ordering))
            i+=1
            j+=1
    l1 = f.tell()
    f.write(msgpack.dumps(sorted(gdict.keys(), key=gdict.get)))
    l2 = f.tell()
    headindex[2] = l2-l1
    f.seek(4)
    f.write(struct.pack(type, *headindex))
    f.seek(l2)
    i = 0
    for bl in blocksdump:
        blocks[i] = f.tell()
        f.write(bl)
        i+=1
    f.seek(blocksind)
    f.write(struct.pack('L'*len(blocks),*blocks))

import numpy as np

def globalindirect(f,global_dict,ordering,chunk):
    output = cStringIO()
    indirect = 0
    output.truncate(0)
    headindex = [0]*7
    minmax = [None]*2
    minmax[0] = min(chunk)
    minmax[1] = max(chunk)
    local_dict = np.sort(chunk.unique())

    type = 'i'*len(headindex)
    output.write(struct.pack(type, *headindex))
    l1 = output.tell()
    output.write(msgpack.dumps(minmax))
    l2 = output.tell()
    headindex[0] = l2-l1
    coldict = {}

    size_1 = len(local_dict)
    size_2 = len(global_dict)

    res2 = [None]*size_1
    i, j = 0, 0
    if not ordering:
      res2 = list(np.intersect1d(local_dict,global_dict,assume_unique=True,return_indices=True)[2])
    else:
      while i < size_1:
        if local_dict[i] > global_dict[j]:
            j+=1
        elif local_dict[i] == global_dict[j]:
          res2[i] = j
          i += 1
          j += 1

    ### chose if indirect
    locallen = len(local_dict)
    maxres = max(res2)
    globallen = len(global_dict)
    datalen = len(chunk)
    if locallen < 256:
            s1 = 1
    elif locallen < 65536:
            s1 = 2
    else:
            s1 = 4
    if globallen < 256:
            s2 = 1
    elif globallen < 65536:
            s2 = 2
    else:
            s2 = 4
    if maxres < 256:
            s3 = 1
    elif maxres < 65536:
            s3 = 2
    else:
            s3 = 4

    if datalen*s1 + locallen*s3 < datalen*s2:
        indirect = 1
    #######################

    if indirect:
        coldict = dict(((x, y) for y, x in enumerate(local_dict)))
    else:
        coldict = dict(((x, y) for y, x in enumerate(global_dict)))

    l3 = output.tell()
    if minmax[0] != minmax[1]:
        offsets = [coldict[y] for y in chunk]
        ll = max(offsets)
        if ll<256:
            type1 = 'B'
            headindex[2] = 2
        elif ll<65536:
            type1 = 'H'
        #type1 = np.int16
            headindex[2] = 1
        else:
            type1 = 'i'
        #type1 = np.int32
        globall = max(res2)
        if globall<256:
            type2 = 'B'
            headindex[5] = 2
        elif globall<65536:
            type2 = 'H'
        #type1 = np.int16
            headindex[5] = 1
        else:
            type2 = 'i'
        l4 = output.tell()
        if indirect:
            output.write(struct.pack(type2*len(res2), *res2))
            l4 = output.tell()
            headindex[4] = l4-l3 # here is the size of the indirect mapping stored after minmax
            headindex[6] = len(res2)
        output.write(struct.pack(type1*len(offsets), *offsets))
    else:
        output.write(struct.pack('i',int(np.where(global_dict == minmax[0])[0])))
        headindex[6] = 1
        l4 = output.tell()
        headindex[4] = l4-l3
        output.write(struct.pack('i',coldict[minmax[0]]))
    #output.write(array(type1,[coldict[y] for y in chunk]).tostring())
    l5 = output.tell()
    headindex[1] = l5-l4
    headindex[3] = len(chunk)
    output.seek(0)
    output.write(struct.pack(type, *headindex))
    f.write(output.getvalue())

def write_indirect(data,filen, ordering, row_group_offsets):
    f = open(filen,"w+b")
    f.write(MARKER)
    headindex = [0]*4
    l = len(data)
    headindex[0] = l
    headindex[1] = len(data.columns)
    type = 'i'*len(headindex)
    f.write(struct.pack(type, *headindex))
    nparts = max((l - 1) // row_group_offsets + 1, 1)
    chunksize = max(min((l - 1) // nparts + 1, l), 1)
    blocks = [0]*(nparts+1)
    blocks[nparts] = chunksize
    headindex[3] = nparts
    blocksind = f.tell()
    f.write(struct.pack('L'*len(blocks),*blocks))

    headindex[3] = nparts
    for col in data:
        l1 = f.tell()
        if ordering == 1:
            global_dict = sorted(data[col].unique())
            f.write(msgpack.dumps(global_dict))
        else:
            global_dict = data[col].unique()
            f.write(msgpack.dumps(list(global_dict)))
        l2 = f.tell()
        headindex[2] = l2-l1
        f.seek(4)
        f.write(struct.pack(type, *headindex))
        f.seek(l2)
        i=0
        j=1
        for part in range(nparts):
            blocks[i] = f.tell()
            chunk = data[col][chunksize*i:chunksize*j]
            globalindirect(f,global_dict,ordering,chunk)
            i+=1
            j+=1
    f.seek(blocksind)
    f.write(struct.pack('L'*len(blocks),*blocks))

import pyarrow as pa
import pyarrow.parquet as pq

def write_parquet(data,filen, row_group_offsets):
    if row_group_offsets == -1:
        table = pa.Table.from_pandas(data)
        pq.write_table(table, filen)
        return

    l = len(data)
    nparts = max((l - 1) // row_group_offsets + 1, 1)
    chunksize = max(min((l - 1) // nparts + 1, l), 1)

    ## write default no row groups
    #pq.write_table(table, 'example.parquet',compression={'10.1145/3025453.3025878': 'snappy', '10.1145/2317956.2318088': 'gzip'},use_dictionary=['10.1145/3025453.3025878','10.1145/2317956.2318088'])

    c = 0
    table = pa.Table.from_pandas(data)
    writer = pq.ParquetWriter(filen, table.schema, use_dictionary = True)
    for i in range(nparts):
        if i!=nparts:
            writer.write_table(table[c*chunksize:chunksize+c*chunksize])
            c+=1
        else:
            writer.write_table(table[:len(data)%chunksize])
    writer.close()


import collections
import csv
from dm_control import mujoco
from dm_control.rl import control
from dm_control.suite import base
from dm_control.suite import common
from dm_control.utils import containers
from dm_control.utils import rewards
import numpy as np
import os

_DEFAULT_TIME_LIMIT = 40
_CONTROL_TIMESTEP = .04
_JOINTS = ['root',
           'tail1',
           # 'tail_twist',
           'tail2']
           # 'finright_roll',
           # 'finright_pitch',
           # 'finleft_roll',
           # 'finleft_pitch']
SUITE = containers.TaggedTasks()

def get_model_and_assets():
  """Returns a tuple containing the model XML string and a dict of assets."""
  # return common.read_model(os.path.join(os.path.abspath('./'),'fishD.xml')), common.ASSETS
  return common.read_model(os.path.join(os.path.dirname(os.path.realpath(__file__)),'fishD.xml')), common.ASSETS
  # return common.read_model(sys.a'fishD.xml'), common.ASSETS

@SUITE.add('benchmarking')
def swim(time_limit=_DEFAULT_TIME_LIMIT, random=None, environment_kwargs=None):
  """Returns the Fish Swim task."""
  physics = Physics.from_xml_string(*get_model_and_assets())
  task = Swim(random=random)
  environment_kwargs = environment_kwargs or {}
  return control.Environment(physics, task, control_timestep=_CONTROL_TIMESTEP, time_limit=time_limit,
      **environment_kwargs)

class Physics(mujoco.Physics):
  """Physics simulation with additional features for the Fish domain."""

  def upright(self):
    """Returns projection from z-axes of torso to the z-axes of worldbody."""
    return self.named.data.xmat['torso', 'zz']

  def torso_velocity(self):
    """Returns velocities and angular velocities of the torso."""
    return self.data.sensordata

  def joint_velocities(self):
    """Returns the joint velocities."""
    return self.named.data.qvel[_JOINTS]

  def joint_angles(self):
    """Returns the joint positions."""
    return self.named.data.qpos[_JOINTS]

  def mouth_to_target(self):
    """Returns a vector, from mouth to target in local coordinate of mouth."""
    data = self.named.data
    mouth_to_target_global = data.geom_xpos['target'] - data.geom_xpos['mouth']
    return mouth_to_target_global.dot(data.geom_xmat['mouth'].reshape(3, 3))

class Swim(base.Task):
  """A Fish `Task` for swimming with smooth reward."""

  def __init__(self, random=None,reward_mode='speed'):
    """Initializes an instance of `Swim`.

    Args:
      random: Optional, either a `numpy.random.RandomState` instance, an
        integer seed for creating a new `RandomState`, or None to select a seed
        automatically (default).
    """
    self.reward_mode = reward_mode
    self.init=True
    f1 = open('./log-fish.csv', "w") #erase
    super().__init__(random=random)

  def initialize_episode(self, physics):
    """Sets the state of the environment at the start of each episode."""

    # quat = self.random.randn(4)
    # physics.named.data.qpos['root'][3:7] = quat / np.linalg.norm(quat)
    # for joint in _JOINTS:
    #   physics.named.data.qpos[joint] = self.random.uniform(-.2, .2)
    # # Randomize target position.
    # physics.named.model.geom_pos['target', 'x'] = self.random.uniform(-.4, .4)
    # physics.named.model.geom_pos['target', 'y'] = self.random.uniform(-.4, .4)
    # physics.named.model.geom_pos['target', 'z'] = self.random.uniform(.1, .3)
    super().initialize_episode(physics)

  def get_observation(self, physics):
    """Returns an observation of joints, target direction and velocities."""
    obs = collections.OrderedDict()
    obs['joint_angles'] = physics.joint_angles()
    # obs['upright'] = physics.upright()
    # obs['target'] = physics.mouth_to_target()
    obs['velocity'] = physics.joint_velocities()
    return obs

  def get_reward(self, physics):
    """Returns a smooth reward."""
    radii = physics.named.model.geom_size[['mouth', 'target'], 0].sum()
    in_target = rewards.tolerance(np.linalg.norm(physics.mouth_to_target()), bounds=(0, radii), margin=2*radii)
    # is_upright = 0.5 * (physics.upright() + 1)
    if self.reward_mode=='speed':
      # return np.array([physics.named.data.xpos['torso', 'y']])#good
      # return physics.named.data.qpos['root']
      # t=physics.named.data.qvel['root']
      # print(physics.named.data.qvel['root'])
      return physics.named.data.qvel['root'][0]
    return in_target

  # def after_step(self,physics):
  #     row = {'j_p': physics.named.data.qpos[_JOINTS[0]][0], 'j_v': physics.named.data.qvel[_JOINTS[0]][0], 'j_acc': physics.named.data.qacc[_JOINTS[0]][0],
  #            's_acc': physics.named.data.qacc["root"][0], 's_p': physics.named.data.qpos["root"][0], 's_v': physics.named.data.qvel["root"][0], 'F_x':physics.named.data.sensordata[6],'F_y':physics.named.data.sensordata[7],'F_z':physics.named.data.sensordata[8]}
  #
  #     with open('./log-fish-results.csv', 'a', newline='') as f:
  #     # with open('./log.csv', 'a', newline='') as f:
  #         # create the csv writer
  #         writer = csv.DictWriter(f, fieldnames=row.keys())
  #         if self.init:
  #             writer.writeheader()
  #             self.init = False
  #         # write a row to the csv file
  #         writer.writerow(row)
  #         f.close()


import copy
import csv
import random
from typing import List, Tuple

def create_base(
    trainSamples: int,
) -> Tuple[List[List[float]], List[List[float]], List[int], List[int]]:
    X_test = []
    X_train = []
    with open("src/datasets/jm1.csv", newline="") as csvfile:
        spamreader = csv.reader(csvfile, delimiter=",", quotechar="|")
        jm1 = [row for nr, row in enumerate(spamreader)]
        jm1_true = [j for j in jm1 if j[21] == "true"]
        jm1_false = [j for j in jm1 if j[21] == "false"]
        random.shuffle(jm1_true)
        random.shuffle(jm1_false)
        trainPart = int(
            trainSamples / 2
        )  # Test sample divided between true answers and false answers
        jm1_train = jm1_true[:trainPart]
        jm1_train = jm1_train + jm1_false[:trainPart]
        # jm1_test = jm1_true[trainPart:trainSamples]
        # jm1_test = jm1_test + jm1_false[trainPart:trainSamples]
        jm1_test = jm1_true[trainPart:]
        jm1_test = jm1_test + jm1_false[trainPart:]
        random.shuffle(jm1_train)
        random.shuffle(jm1_test)
        jm1_test = jm1_test[:trainSamples]

    Y_train_str = [j.pop(-1) for j in jm1_train]
    Y_train = [1 if x == "true" else 0 for x in Y_train_str]

    for jt in jm1_train:
        X_train.append([float(j) for j in jt])

    Y_test_str = [j.pop(-1) for j in jm1_test]
    Y_test = [1 if x == "true" else 0 for x in Y_test_str]
    for jt in jm1_test:
        X_test.append([float(j) for j in jt])

    return X_test, X_train, Y_train, Y_test

def save_classifier_if_diverse(
    classif_score: float,
    classif_predict: List[int],
    predicts: List[List[int]],
    params: List[str],
    nr_items: int,
    string_parameters: str,
) -> None:
    if classif_score >= 0.6:
        if len(predicts) > 0:
            flag = False
            for pred in predicts:
                s = sum(
                    x != y
                    for x, y in zip(
                        classif_predict,
                        pred,
                    )
                )
                if s < (0.05 * nr_items):
                    flag = True
                    break

            if not flag:
                predicts.append(copy.deepcopy(classif_predict))
                params.append(string_parameters)
        else:
            predicts.append(copy.deepcopy(classif_predict))
            params.append(string_parameters)

def save_quality_classifier(
    f1_score: float,
    params: List[str],
    string_parameters: str,
):
    if f1_score > 0.3:
        params.append(string_parameters)


# coding=utf-8
# created by msg on 2019/11/24 2:30 下午

import re
import unicodedata
import tensorflow as tf
from sklearn.model_selection import train_test_split

# 第一个参数指定字符串标准化的方式。
# NFC表示字符应该是整体组成(比如可能的话就使用单一编码)
# NFD表示字符应该分解为多个组合字符表示。
# unicodedata.category(chr) 把一个字符返回它在UNICODE里分类的类型
def unicode_to_ascii(s):
    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')

# 预处理
def preprocess_sentence(w):
    # 转小写换编码
    w = unicode_to_ascii(w.lower().strip())

    # 标点符号前后加上空格
    #  eg: "he is a boy." => "he is a boy ."
    w = re.sub(r"([?.!,¿])", r" \1 ", w)

    # 去掉多余空格
    w = re.sub(r'[" ]+', " ", w)

    # 去掉前后空格
    w = w.rstrip().strip()

    # 加开始和结束字符
    w = '<start> ' + w + ' <end>'
    return w

# 创建数据集
def parse_data(filename):
    lines = open(filename, encoding='utf-8').read().strip().split('\n')
    sentence_pairs = [line.split('\t') for line in lines]
    preprocessed_sentence_pairs = [(preprocess_sentence(en), preprocess_sentence(spa)) for en, spa in sentence_pairs]
    return zip(*preprocessed_sentence_pairs)

# 分词和词语到索引的转换
def tokenizer(lang):
    # 用空格分词
    lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=None, filters='', split=' ')
    # 先分词
    lang_tokenizer.fit_on_texts(lang)
    # 再创建词到索引的映射
    tensor = lang_tokenizer.texts_to_sequences(lang)
    # 不够长度的在后边补0
    tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor, padding='post')
    return tensor, lang_tokenizer

# 获取句子的最大长度
def max_length(tensor):
    return max(len(t) for t in tensor)

# 拆分训练测试数据集
def split(input_tensor, output_tensor, test_size=0.2):
    return train_test_split(input_tensor, output_tensor, test_size=test_size)

# 构建训练集
def make_dataset(input_tensor, output_tensor, batch_size=64, epochs=20, shuffle=True):
    dataset = tf.data.Dataset.from_tensor_slices((input_tensor, output_tensor))
    if shuffle:
        dataset = dataset.shuffle(30000)
    dataset = dataset.repeat(epochs).batch(batch_size, drop_remainder=True)
    return dataset

if __name__ == '__main__':
    spa_eng_path = 'spa-eng/spa.txt'
    en_dataset, spa_dataset = parse_data(spa_eng_path)
    print(en_dataset[-1])
    print(spa_dataset[-1])

    # 西班牙语到英语的训练
    input_tensor, input_tokenizer = tokenizer(spa_dataset[:30000])
    output_tensor, output_tokenizer = tokenizer(en_dataset[:30000])

    print(input_tensor)
    print(output_tensor)

    # 查看输入和输出的最大长度
    print(max_length(input_tensor))
    print(max_length(output_tensor))

    # 拆分数据集为训练集和验证集
    input_train, input_eval, output_train, output_eval = split(input_tensor, output_tensor)
    print(len(input_train), len(input_eval), len(output_train), len(output_eval))

    # 构建tf.data格式
    train_dataset = make_dataset(input_train, output_train)
    eval_dataset = make_dataset(input_eval, output_eval)

    # 打印一下
    for x, y in train_dataset.take(1):
        print(x.shape)
        print(y.shape)
        print(x)
        print(y)


#!/usr/bin/env python
from __future__ import absolute_import
import json
import logging

from . import IndicatorTypes

API_V1_ROOT = "{}/api/v1/"
PULSES_ROOT = "{}/pulses".format(API_V1_ROOT)
SUBSCRIBED = "{}/subscribed".format(PULSES_ROOT)
EVENTS = "{}/events".format(PULSES_ROOT)

try:
    # For Python2
    from urllib2 import URLError, HTTPError, build_opener, ProxyHandler
except ImportError:
    # For Python3
    from urllib.error import URLError, HTTPError
    from urllib.request import build_opener, ProxyHandler

logger = logging.getLogger("OTXv2")

class InvalidAPIKey(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class BadRequest(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class OTXv2(object):
    """
    Main class to interact with the AlienVault OTX API.
    """

    def __init__(self, api_key, proxy=None, server="https://otx.alienvault.com", project="SDK"):
        self.key = api_key
        self.server = server
        self.proxy = proxy
        self.sdk = 'OTX Python {}/1.0'.format(project)

    def get(self, url):
        """
        Internal API for GET request on a OTX URL
        :param url: URL to retrieve
        :return: response in JSON object form
        """
        if self.proxy:
            proxy = ProxyHandler({'http': self.proxy})
            request = build_opener(proxy)
        else:
            request = build_opener()
        request.addheaders = [
            ('X-OTX-API-KEY', self.key),
            ('User-Agent', self.sdk)
        ]
        response = None
        try:
            response = request.open(url)
        except URLError as e:
            if isinstance(e, HTTPError):
                if e.code == 403:
                    raise InvalidAPIKey("Invalid API Key")
                elif e.code == 400:
                    raise BadRequest("Bad Request")
            else:
                raise e
        data = response.read().decode('utf-8')
        json_data = json.loads(data)
        return json_data

    def create_url(self, url_path, **kwargs):
        uri = url_path.format(self.server)
        uri += "?"
        for parameter, value in kwargs.items():
            uri += parameter
            uri += "="
            uri += str(value)
            uri += "&"
        return uri

    def getall(self, limit=20):
        """
        Get all pulses user is subscribed to.
        :param limit: The page size to retrieve in a single request
        :return: the consolidated set of pulses for the user
        """
        pulses = []
        next = self.create_url(SUBSCRIBED, limit=limit)
        while next:
            json_data = self.get(next)
            for r in json_data["results"]:
                pulses.append(r)
            next = json_data["next"]
        return pulses

    def getall_iter(self, limit=20):
        """
        :param limit:
        :return:
        """
        pulses = []
        next = self.create_url(SUBSCRIBED, limit=limit)
        while next:
            json_data = self.get(next)
            for r in json_data["results"]:
                yield r
            next = json_data["next"]

    def getsince(self, mytimestamp, limit=20):
        """
        Get all pulses created or updated since a timestamp
        :param mytimestamp: timestamp to filter returned pulses
        :param limit: The page size to retrieve in a single request
        :return: the consolidated set of pulses for the user
        """
        pulses = []
        next = self.create_url(SUBSCRIBED, limit=limit, modified_since=mytimestamp)
        while next:
            json_data = self.get(next)
            for r in json_data["results"]:
                pulses.append(r)
            next = json_data["next"]
        return pulses

    def getsince_iter(self, mytimestamp, limit=20):
        pulses = []
        next = self.create_url(SUBSCRIBED, limit=limit, modified_since=mytimestamp)
        while next:
            json_data = self.get(next)
            for r in json_data["results"]:
                yield r
            next = json_data["next"]

    def get_all_indicators(self, indicator_types=IndicatorTypes.all_types):
        """
        Get all the indicators contained within your pulses of the IndicatorTypes passed.
        By default returns all IndicatorTypes.
        :param indicator_types: IndicatorTypes to return
        :return: yields the indicator object for use
        """
        name_list = IndicatorTypes.to_name_list(indicator_types)
        for pulse in self.getall_iter():
            for indicator in pulse["indicators"]:
                if indicator["type"] in name_list:
                    yield indicator

    def getevents_since(self, mytimestamp, limit=20):
        """
        Get all events (activity) created or updated since a timestamp
        :param mytimestamp: timestamp to filter returned activity
        :param limit: The page size to retrieve in a single request
        :return: the consolidated set of pulses for the user
        """
        events = []
        next = self.create_url(EVENTS, limit=limit, since=mytimestamp)
        while next:
            json_data = self.get(next)
            for r in json_data["results"]:
                events.append(r)
            next = json_data["next"]
        return events


from typing import List

def intcode(program: List[int], inputs: List[int] = [], perpetual_input: bool = False,
            loop_mode: bool = False, print_outputs: bool = True, extend: int = 0,
            i: int = 0):
    assert len(program) > 0

    p = program.copy()
    p.extend([0] * extend)

    relative_base = 0
    outputs = []

    while p[i] != 99:
        # Extract operation
        op = int(str(p[i])[-2:])
        # Extract modes after zero filling it & inverting
        modes = [int(char) for char in str(p[i])[:-2].zfill(3)][::-1]

        # Map variables/locations based on operation
        if op in [1, 2, 7, 8]:
            # Two parameters, 1 target
            variables = [
                p[i + 1] if modes[0] == 1 else p[p[i + 1] + relative_base] if modes[0] == 2 else p[
                    p[i + 1]],
                p[i + 2] if modes[1] == 1 else p[p[i + 2] + relative_base] if modes[1] == 2 else p[
                    p[i + 2]],
                i + 3 if modes[2] == 1 else p[i + 3] + relative_base if modes[2] == 2 else p[i + 3]
            ]
        elif op in [5, 6]:
            # Two parameters
            variables = [
                p[i + 1] if modes[0] == 1 else p[p[i + 1]] if modes[0] == 0 else p[
                    p[i + 1] + relative_base],
                p[i + 2] if modes[1] == 1 else p[p[i + 2]] if modes[1] == 0 else p[
                    p[i + 2] + relative_base],
            ]
        elif op in [3, 4]:
            # One location?
            variables = [
                i + 1 if modes[0] == 1 else p[i + 1] + relative_base if modes[0] == 2 else p[i + 1]
            ]
        elif op in [9]:
            # One parameter
            variables = [
                p[i + 1] if modes[0] == 1 else p[p[i + 1]] if modes[0] == 0 else p[
                    p[i + 1] + relative_base],
            ]

        # Perform operation
        if op == 1:  # Sum
            p[variables[2]] = variables[0] + variables[1]
            i += 4
        elif op == 2:  # Multiplication
            p[variables[2]] = variables[0] * variables[1]
            i += 4
        elif op == 3:  # Input
            p[variables[0]] = inputs[0] if perpetual_input and len(inputs) == 0 else inputs.pop(0)
            i += 2
        elif op == 4:  # Output
            outputs.append(p[variables[0]])
            if print_outputs:
                print(p[variables[0]])
            i += 2
            if loop_mode:
                return outputs, p.copy(), i
        elif op == 5:  # Jump-if-true
            i = variables[1] if variables[0] else i + 3
        elif op == 6:  # Jump-if-false
            i = i + 3 if variables[0] else variables[1]
        elif op == 7:  # Less-than
            p[variables[2]] = int(variables[0] < variables[1])
            i += 4
        elif op == 8:  # Equals
            p[variables[2]] = int(variables[0] == variables[1])
            i += 4
        elif op == 9:  # Relative base adjustment
            relative_base += variables[0]
            i += 2
        else:  # ERROR!!
            print("Oops...")

    if loop_mode:
        return None, None, -1
    return outputs, p, i

def chain_intercodes(program: List[int], phases: List[int] = []):
    output = 0
    for phase in phases:
        outputs, _, _ = intcode(program, [phase, output])
        output = outputs[-1]
    return output

def looping_intcodes(program: List[int], phases: List[int]):
    amps = [program.copy() for i in range(len(phases))]
    amps_i = [0] * len(phases)
    amps_active = list(range(len(phases)))

    output = 0
    result = None

    while len(amps_active) > 0:
        for amp in amps_active:
            program = amps[amp].copy()
            i = amps_i[amp]
            phase = phases[amp]
            program_input = [phase, output] if i == 0 else [output]
            outputs, amp_p, amp_i = intcode(program, program_input, perpetual_input=True,
                                            loop_mode=True, print_outputs=False, i=i)
            if outputs is not None:
                output = outputs[-1]
                result = output
                # print(result)
            if amp_i == -1:
                amps_active.remove(amp)
            else:
                amps[amp] = amp_p.copy()
                amps_i[amp] = amp_i

    return result


#Convert XML to TXT
import xml.etree.ElementTree as ET
import glob

#list of xml files
listOfFiles = sorted(glob.glob('/Users/shiva.hnf/Documents/IIT/visual-targets/Demo/VT_DataAnnotation/VT_InputData/Annotations/TwoPeople_ThreeObjects(2)/*.xml'))

#opening a txt file
f = open('/Users/shiva.hnf/Documents/IIT/visual-targets/Demo/VT_DataAnnotation/bndBox_TXT/TwoPeople_ThreeObjects(2).txt','w') #Creates a new file
#f.write('name, left, bottom, right, top\n')
f.close()

#parsing the xml files
for i in range(len(listOfFiles)):
  tree = ET.parse(listOfFiles[i])
  root = tree.getroot()
  #extracting info from xml files
  for ann in root.iter('annotation'):
    filename = ann.find('filename').text
    left = ann.find('object/bndbox/xmin').text
    right = ann.find('object/bndbox/xmax').text
    bottom = ann.find('object/bndbox/ymin').text
    top = ann.find('object/bndbox/ymax').text
    line_to_write = filename + ',' + left + ',' + bottom + ',' + right + ',' + top + '\n'
    #writing to the txt file
    with open('/Users/shiva.hnf/Documents/IIT/visual-targets/Demo/VT_DataAnnotation/bndBox_TXT/TwoPeople_ThreeObjects(2).txt', 'a') as f:
      f.write(line_to_write)


import xml.etree.ElementTree as ET

def read_anntation(xml_file: str):
    tree = ET.parse(xml_file)
    root = tree.getroot()

    bounding_box_list = []

    file_name = root.find('filename').text
    for obj in root.iter('object'):

        object_label = obj.find("name").text
        for box in obj.findall("bndbox"):
            x_min = int(box.find("xmin").text)
            y_min = int(box.find("ymin").text)
            x_max = int(box.find("xmax").text)
            y_max = int(box.find("ymax").text)

        bounding_box = [object_label, x_min, y_min, x_max, y_max]
        bounding_box_list.append(bounding_box)

    return bounding_box_list, file_name


from bottle import get, response
import json
import mysql.connector
from g import (
    DATABASE_CONFIG
)

###################################################################
# FETCH ALL BOOKING DATA
###################################################################
@get('/api-fetch-booking_data')
def _():
    try:

        ################  CONNECT TO DATABASE  ###################
        db_connection = mysql.connector.connect(**DATABASE_CONFIG)
        cursor = db_connection.cursor(dictionary=True)
        ##########################################################

        sql_fetchAll_customer = """ SELECT * FROM participants """
        cursor.execute(sql_fetchAll_customer)
        participants = cursor.fetchall()

        sql_fetchAll_booking_options =      """
                                                SELECT bkg_option_id, options, prices.duration, prices.price
                                                FROM booking_options
                                                JOIN prices ON booking_options.fk_price_id = prices.price_id
                                            """
        cursor.execute(sql_fetchAll_booking_options)
        booking_options = cursor.fetchall()

        sql_fetchAll_booking_date_times =    """
                                                SELECT * FROM booking_date_times
                                             """
        cursor.execute(sql_fetchAll_booking_date_times)
        booking_date_times = cursor.fetchall()

        sql_fetchAll_booking_dates_with_times =    """
                                                        SELECT DISTINCT booking_dates.available_dates
                                                        FROM booking_date_times
                                                        INNER JOIN booking_dates ON booking_date_times.fk_bkg_date_id = booking_dates.bkg_date_id
                                                   """
        cursor.execute(sql_fetchAll_booking_dates_with_times)
        booking_dates = cursor.fetchall()

        response.content_type = 'application/json; charset=UTF-8'
        return json.dumps(dict(
            participants=participants,
            booking_options=booking_options,
            booking_date_times=booking_date_times,
            booking_dates=booking_dates
            ), default=str)
    except Exception as ex:
        print(ex)

#### test1
@get('/test/<date>')
def _(date):

    try:

        ################  CONNECT TO DATABASE  ###################
        db_connection = mysql.connector.connect(**DATABASE_CONFIG)
        cursor = db_connection.cursor(dictionary=True)
        ##########################################################

        sql_fetchAll_booking_dates_with_times =    f"""
                                                        SELECT DISTINCT bkg_date_time_id, booking_dates.available_dates, available_times
                                                        FROM booking_date_times
                                                        INNER JOIN booking_dates ON booking_date_times.fk_bkg_date_id = booking_dates.bkg_date_id
                                                        WHERE booking_dates.available_dates = {date}
                                                   """
        cursor.execute(sql_fetchAll_booking_dates_with_times)
        booking_dates_times = cursor.fetchall()

        print('#'*100)
        print(booking_dates_times)

        response.content_type = 'application/json; charset=UTF-8'
        return json.dumps(dict(
            booking_dates_and_times=booking_dates_times
            ), default=str)

    except Exception as ex:
        print(ex)


import torch
import torchvision
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms

# Define hyperparameters
batch_size = 32
learning_rate = 0.001
num_epochs = 10
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Define dataset and dataloader classes
class VideoDataset(Dataset):
    def __init__(self, video_path, label_path):
        self.video_path = video_path
        self.label_path = label_path
        self.video_names, self.labels = self._load_labels()

    def __getitem__(self, index):
        # Load and preprocess video frames
        video_name = self.video_names[index]
        frames = self._load_video_frames(video_name)

        # Load and preprocess ground truth labels
        label = self.labels[index]
        label = self._preprocess_label(label)

        return frames, label

    def __len__(self):
        return len(self.video_names)

    def _load_labels(self):
        with open(self.label_path, 'r') as f:
            lines = f.readlines()
            labels = []
            video_names = []
            for line in lines:
                parts = line.split(',')
                video_name = parts[0]
                label = parts[7].strip()
                video_names.append(video_name)
                labels.append(label)
        return video_names, labels

    def _load_video_frames(self, video_name):
        # Load video frames using a library such as OpenCV or PyAV
        # Preprocess frames using transforms as necessary
        return frames

    def _preprocess_label(self, label):
        # Preprocess label to convert it into a format that can be used for training
        return label
def decode_outputs(outputs, idx_to_word):
    # Convert the output tensor into a sequence of predicted word indices
    _, indices = torch.max(outputs, dim=2)
    indices = indices.squeeze()

    # Convert the predicted word indices into a sequence of words
    predicted_words = [idx_to_word[idx.item()] for idx in indices]

    # Remove <START> and <END> tokens from the sequence
    predicted_words = predicted_words[1:-1]

    return predicted_words
# Define model architecture
class VideoCaptioningModel(nn.Module):
    def __init__(self, hidden_size):
        super().__init__()
        self.conv = nn.Sequential(...)
        self.rnn = nn.LSTM(input_size=..., hidden_size=hidden_size, num_layers=1, batch_first=True)
        self.fc = nn.Linear(hidden_size, ...)

    def forward(self, x):
        # Apply convolutional layers
        x = self.conv(x)

        # Apply recurrent layers
        h0 = torch.zeros(1, x.size(0), self.rnn.hidden_size).to(x.device)
        c0 = torch.zeros(1, x.size(0), self.rnn.hidden_size).to(x.device)
        x, _ = self.rnn(x, (h0, c0))

        # Apply fully connected layer
        x = self.fc(x)

        return x

# Define loss function and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

# Load data and train model
train_dataset = VideoDataset(video_path='../data/video', label_path='../data/labels.txt')
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)

model = VideoCaptioningModel(hidden_size=...)
model.to(device)

for epoch in range(num_epochs):
    for i, (frames, label) in enumerate(train_dataloader):
        frames = frames.to(device)
        label = label.to(device)

        # Forward pass
        outputs = model(frames)

        # Compute loss and gradients
        loss = criterion(outputs, label)
        loss.backward()

        # Update parameters
        optimizer.step()
        optimizer.zero_grad()

        # Print progress
        if (i + 1) % 10 == 0:
            print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
                  .format(epoch + 1, num_epochs, i + 1, len(train_dataloader), loss.item()))

# Test the model
with torch.no_grad():
    model.eval()
    for i, (frames, _) in enumerate(train_dataloader):
        frames = frames.to(device)
        outputs = model(frames)
        predicted_words = decode_outputs(outputs)
        print('Video {}: {}'.format(i+1, predicted_words))

def decode_outputs(outputs):
    # Decode the output of the model into a sequence of words
    return predicted_words

torch.save(model.state_dict(), 'situation.pth')


# coding:utf-8
from LineApi import *
from LineApi.hooks import HooksTracer
from collections    import OrderedDict
import random, requests, json, random, sqlite3, sys

with open("Tokens.json","r",encoding="utf8") as f:
    tokens = json.loads(f.read(),object_pairs_hook=OrderedDict)

with open("Help.txt","r",encoding="utf8") as f:
    helpText = f.read()

yukino_1 = LINE(tokens["Main"],type="IOSIPAD")
yukino_2 = LINE(tokens["Sub1"],type="IOSIPAD")
yukino_3 = LINE(tokens["Sub2"],type="IOSIPAD")
yukino_4 = LINE(tokens["Sub3"],type="IOSIPAD")
yukino_5 = LINE(tokens["Sub4"],type="IOSIPAD")

db                 = sqlite3.connect("protect.db", check_same_thread=False)
tracer             = HooksTracer(yukino_1,prefix=["／","/","!","?",".","#"],db=db)
tracer.bots        = [yukino_1, yukino_2, yukino_3, yukino_4, yukino_5]
tracer.kickers     = [yukino_2, yukino_3, yukino_4, yukino_5]
tracer.botMids     = [b.getProfile().mid for b in tracer.bots]
tracer.kickerMids  = [k.getProfile().mid for k in tracer.kickers]
tracer.helptext    = helpText

class OpPs(object):
    @tracer.Operation(26)
    def OPERATION_MESSAGE(self,cl,op):
        msg = op.message
        self.trace(msg,"Content")
    @tracer.Content(0)
    def CONTENT_MESSAGE(self,cl,msg):
        self.log("[MESSAGE] "+msg.text)
        self.trace(msg,"Command")

    @tracer.Operation(19)
    def NOTIFIED_KICKOUT_FROM_GROUP(self,cl,op):
        self.log("NOTIFIED_KICKOUT_FROM_GROUP")
        gid    = op.param1
        kicker = op.param2
        gotban = op.param3
        try:
            if self.getGroup(gid,"normalProtect"):
                if  kicker not in self.kickerMids\
                and kicker not in self.getPermissionByName("White")\
                and kicker not in self.getPermissionByName("Admin"):
                    self.kk_kick(gid,gotban,kicker)
                    self.kk_invite(gid,gotban,kicker)
                elif gotban in self.botMids:
                    self.kk_invite(gid,gotban,kicker)
        except Exception as error:
            print(error)

    @tracer.Operation(13)
    def NOTIFIED_INVITE_INTO_GROUP(self,cl,op):
        self.log("NOTIFIED_INVITE_INTO_GROUP")
        gid        = op.param1
        inviter    = op.param2
        got_inv    = op.param3
        try:
            if cl.mid in got_inv:
                if inviter in self.getPermissionByName("White")\
                or inviter in self.getPermissionByName("Admin"):
                    cl.acceptGroupInvitation(gid)
                    for k in self.kickers:
                        try:
                            cl.inviteIntoGroup(gid, k.mid)
                            k.acceptGroupInvitation(gid)
                        except:
                            pass
                    cl.sendMessage(op.param1,"こんにちは! 私はグループ保護Botです。\nコマンド一覧は '/ヘルプ' で確認できます。")
            elif self.getGroup(gid,"normalProtect")\
            and (inviter not in self.getPermissionByName("White")\
                 or inviter in self.getPermissionByName("Admin")):
                gotinvs = got_inv.split("")
                for b in gotinvs:
                    cl.cancelGroupInvitation(gid, b)
        except Exception as error:
            print(error)

    @tracer.Operation(11)
    def NOTIFIED_UPDATE_GROUP(self,cl,op):
        self.log("NOTIFIED_UPDATE_GROUP")
        gid = op.param1
        usr = op.param2
        ctp = op.param3
        try:
            #グループ名変更
            if ctp == "1":
                gr = cl.getGroup(gid)
                if self.getGroup(gid,"protectGroupName"):
                    if  usr not in self.botMids\
                    and usr not in self.getPermissionByName("White")\
                    and usr not in self.getPermissionByName("Admin"):
                        gr.name = self.getGroup(gid,"groupName")
                        cl.updateGroup(gr)
                self.postGroup(gid,"groupName",gr.name)
            #グループ画変更
            elif ctp == "2":
                if self.getGroup(gid,"protectGroupImage"):
                    if  usr not in self.botMids\
                    and usr not in self.getPermissionByName("White")\
                    and usr not in self.getPermissionByName("Admin"):
                        cl.updateGroupPicture(gid,"./%s.jpg"%(gid))
            #グループURL変更
            elif ctp == "4":
                if self.getGroup(gid,"protectGroupUrl"):
                    if  usr not in self.botMids\
                    and usr not in self.getPermissionByName("White")\
                    and usr not in self.getPermissionByName("Admin"):
                        gr = cl.getGroup(gid)
                        if gr.preventedJoinByTicket:
                            gr.preventedJoinByTicket = False
                        else:
                            gr.preventedJoinByTicket = True
                        cl.updateGroup(gr)
                        self.kk_kick(gid,None,usr)
            else:
                print(ctp)
        except Exception as error:
            print(error)

class CmdPs(object):
    @tracer.Command()
    def mid(self,cl,msg):
        cl.replyMessage(msg,msg._from)

    @tracer.Command(sources=["Group"])
    def gid(self,cl,msg):
        cl.replyMessage(msg,msg.to)

    @tracer.Command(sources=["Group"],permissions=["Admin","White"])
    def さよなら(self,cl,msg):
        cl.replyMessage(msg,"さよなら :/")
        for b in self.bots:
            b.leaveGroup(msg.to)

    @tracer.Command(ignoreCase=True,alt=["help","?","へるぷ"])
    def ヘルプ(self,cl,msg):
        cl.replyMessage(msg, self.helptext)

    @tracer.Command()
    def 権限(self,cl,msg):
        cl.replyMessage(msg, "あなたは以下の権限を持っています\n"+("\n".join(self.getPermissionById(msg._from))))

    @tracer.Command(sources=["Group"])
    def グル情報(self,cl,msg):
        group = cl.getGroup(msg.to)
        md =  "[グループ名]:\n"+group.name+"\n\n"
        md += "[gid]:\n"+group.id+"\n\n"
        md += "[アイコン画像]:\n"+"http://dl.profile.line-cdn.net/"
        md += group.pictureStatus+"\n"
        if group.preventedJoinByTicket:
            md += "招待URLから参加: 拒否\n"
        else:
            md += "招待URLから参加: 許可\n"
        md += "メンバー数: " + str(len(group.members)) + "人\n"
        if group.invitee is None:
            md += "招待中: 0人\n"
        else:
            md += "招待中: " + str(len(group.invitee)) + "人\n"
        if self.getGroup(msg.to,"normalProtect"):
            md += "保護: オン\n"
        else:
            md += "保護: オフ\n"
        if self.getGroup(msg.to,"protectGroupUrl"):
            md += "保護URL: オン\n"
        else:
            md += "保護URL: オフ\n"
        if self.getGroup(msg.to,"protectGroupName"):
            md += "保護グル名: オン\n"
        else:
            md += "保護グル名: オフ\n"
        if self.getGroup(msg.to,"protectGroupImage"):
            md += "保護 グループ画像: オン\n"
        else:
            md += "保護 グループ画像: オフ\n"
        cl.replyMessage(msg, md)

    @tracer.Command(sources=["Group"],inpart=True,permissions=["Admin","White"])
    def ホワリス追加(self,cl,msg):
        mts = self.getMention(msg.contentMetadata)
        added  = ""
        for m in mts:
            if "White" not in cl.getPermissionById(m):
                self.addPermission(m,"White")
                added  += cl.getContact(m).displayName + "\n"
        if added != "":
            cl.replyMessage(msg, "%s\nこれらのユーザーをホワイトリストに追加しました"%(added))
        else:
            cl.replyMessage(msg, "メンションされたユーザーは既にホワイトリストに入っています")

    @tracer.Command(sources=["Group"],inpart=True,permissions=["Admin"])
    def ホワリス削除(self,cl,msg):
        mts = self.getMention(msg.contentMetadata)
        removed  = ""
        for m in mts:
            if "White" in cl.getPermissionById(m):
                self.removePermission(m,"White")
                removed  += cl.getContact(m).displayName + "\n"
        if removed != "":
            cl.replyMessage(msg, "%s\nこれらのユーザーをホワイトリストから削除しました"%(removed))
        else:
            cl.replyMessage(msg, "メンションされたユーザーは既にホワイトリストに入っていません")

    @tracer.Command(sources=["Group"],permissions=["Admin","White"])
    def 保護オン(self,cl,msg):
        if not self.getGroup(msg.to,"normalProtect"):
            self.postGroup(msg.to,'normalProtect',True)
            cl.replyMessage(msg, "保護機能を有効にしました。")
        else:
            cl.replyMessage(msg, "保護機能はすでに有効になっています。")

    @tracer.Command(sources=["Group"],permissions=["Admin","White"])
    def 保護オフ(self,cl,msg):
        if self.getGroup(msg.to,"normalProtect"):
            self.postGroup(msg.to,'normalProtect',False)
            cl.replyMessage(msg, "保護機能を解除しました。")
        else:
            cl.replyMessage(msg, "保護機能はすでに無効になっています。")

    @tracer.Command(alt=["保護グル名オン"],sources=["Group"],permissions=["Admin","White"])
    def 保護名前オン(self,cl,msg):
        if not self.getGroup(msg.to,"protectGroupName"):
            self.postGroup(msg.to,'protectGroupName',True)
            self.postGroup(msg.to,'groupName',cl.getGroup(msg.to).name)
            cl.replyMessage(msg, "グループ名の保護機能を有効にしました。")
        else:
            cl.replyMessage(msg, "グループ名の保護機能はすでに有効になっています。")

    @tracer.Command(alt=["保護グル名オフ"],sources=["Group"],permissions=["Admin","White"])
    def 保護名前オフ(self,cl,msg):
        if self.getGroup(msg.to,"protectGroupName"):
            self.postGroup(msg.to,'protectGroupName',False)
            cl.replyMessage(msg, "グループ名の保護機能を解除しました。")
        else:
            cl.replyMessage(msg, "グループ名の保護機能はすでに無効になっています。")

    @tracer.Command(alt=["保護グル画オン"],sources=["Group"],permissions=["Admin","White"])
    def 保護画像オン(self,cl,msg):
        if not self.getGroup(msg.to,"protectGroupImage"):
            gp = cl.getGroup(msg.to).pictureStatus
            resp = requests.get("http://dl.profile.line-cdn.net/"+gp)
            with open(msg.to+".jpg","wb") as f:
                f.write(resp.content)
            self.postGroup(msg.to,"protectGroupImage",True)
            cl.replyMessage(msg, "グループ画像の保護機能を有効にしました。")
        else:
            cl.replyMessage(msg, "グループ画像の保護機能はすでに有効になっています。")

    @tracer.Command(alt=["保護グル画オフ"],sources=["Group"],permissions=["Admin","White"])
    def 保護画像オフ(self,cl,msg):
        if self.getGroup(msg.to,"protectGroupImage"):
            self.postGroup(msg.to,"protectGroupImage",False)
            cl.replyMessage(msg, "グループ画像の保護機能を解除しました。")
        else:
            cl.replyMessage(msg, "グループ画像の保護機能はすでに無効になっています。")

    @tracer.Command(alt=["保護うあるオン"],ignoreCase=True,sources=["Group"],permissions=["Admin","White"])
    def 保護URLオン(self,cl,msg):
        if not self.getGroup(msg.to,"protectGroupUrl"):
            self.postGroup(msg.to,'protectGroupUrl',True)
            cl.replyMessage(msg, "保護機能を有効にしました。")
        else:
            cl.replyMessage(msg, "保護機能はすでに有効になっています。")

    @tracer.Command(alt=["保護うあるオフ"],ignoreCase=True,sources=["Group"],permissions=["Admin","White"])
    def 保護URLオフ(self,cl,msg):
        if self.getGroup(msg.to,"protectGroupUrl"):
            self.postGroup(msg.to,'protectGroupUrl',False)
            cl.replyMessage(msg, "グループurlの保護機能を無効にしました。")
        else:
            cl.replyMessage(msg, "グループurlの保護機能はすでに無効になっています。")

    @tracer.Command(alt=["招待うある許可"],ignoreCase=True,sources=["Group"],permissions=["Admin","White"])
    def 招待URL許可(self,cl,msg):
        gr = cl.getGroup(msg.to)
        if gr.preventedJoinByTicket:
            gr.preventedJoinByTicket = False
            cl.updateGroup(gr)
            cl.replyMessage(msg, "招待URLからの参加を許可にしました。")
        else:
            cl.replyMessage(msg, "既に許可されています。")

    @tracer.Command(alt=["招待うある拒否"],ignoreCase=True,sources=["Group"],permissions=["Admin","White"])
    def 招待URL拒否(self,cl,msg):
        gr = cl.getGroup(msg.to)
        if not gr.preventedJoinByTicket:
            gr.preventedJoinByTicket = True
            cl.updateGroup(gr)
            cl.replyMessage(msg, "招待URLからの参加を拒否にしました。")
        else:
            cl.replyMessage(msg, "既に拒否されています。")

class AdminCmdPs(object):
    @tracer.Command(permissions=["Admin"],alt=["exec"],inpart=True)
    def execute_message(self,cl,msg):
        '''Execute Message as Python Script'''
        with open("temp.txt","w") as t:
            sys.stdout = t
            try:
                exec(msg.text.replace(self.getPrefix(msg.text)+"exec",""))
            except:
                print(traceback.format_exc())
        sys.stdout = sys.__stdout__
        with open("temp.txt","r") as r:
            cl.replyMessage(msg,r.read())

tracer.addClass(FuncPs())
tracer.addClass(OpPs())
tracer.addClass(CmdPs())
tracer.addClass(AdminCmdPs())
tracer.run()

販売保護キッカー５.py

販売保護キッカー５.py


from io import BytesIO
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from pydub import AudioSegment
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
import tempfile
import os
from django.conf import settings
import soundfile as sf

from core.logic import audio_to_text

# Create your views here.

def index(request):
    return  render(request, "realindex.html")

def mp3_to_flac(mp3_path, output_flac_path):
    audio = AudioSegment.from_mp3(mp3_path)
    temp_wav_path = 'temp.wav'
    audio.export(temp_wav_path, format='wav')
    pcm_audio, sample_rate = sf.read(temp_wav_path)
    sf.write(output_flac_path, pcm_audio, samplerate=sample_rate, subtype='FLAC')
    os.remove(temp_wav_path)

@csrf_exempt
def processAudio(request):
    try:
        if request.method == 'POST':
            audio_file = request.FILES.get('audio')

            # Determine the path to the "resources" folder
            resources_folder = os.path.join(settings.BASE_DIR, 'resources')

            # Create the folder if it doesn't exist
            if not os.path.exists(resources_folder):
                os.makedirs(resources_folder)

            # Save the uploaded MP3 file to the "resources" folder
            mp3_path = os.path.join(resources_folder, 'temp.mp3')
            with open(mp3_path, 'wb') as f:
                for chunk in audio_file.chunks():
                    f.write(chunk)

            # Convert the MP3 to FLAC
            flac_path = os.path.join(resources_folder, 'temp.flac')
            mp3_to_flac(mp3_path, flac_path)

            # You can now process the converted FLAC file
            text = audio_to_text('resources/temp.flac')
            print(text)

            # Load and play the MP3 file
            #mixer.music.load(mp3_path)
            #mixer.music.play()
    except Exception as e:
        print(e)

def processAudio1(request):
   if request.method == "POST":
      # Retrieve the audio data from the POST request
      audio_data = request.FILES.get("audio")
      # Process the audio data (e.g., save it to a file, analyze, etc.)
      # Here, we'll just print the size of the audio data
      if audio_data:
        # Load the audio blob using pydub
        audio_data = AudioSegment.from_file(BytesIO(audio_data.read()))
        flac_audio = audio_data.export(format="")
        text = audio_to_text(flac_audio)
        print("Text:_____:", text)
        return aboutUS(request)

def aboutUS(request):
   return render(request,'about_us.html')

def contactUS(request):
   return render(request,'contact_us.html')
def blogPost(request):
   return render(request,'blog.html')

def google(request):
    audio =request.POST.get("audio")
    text = audio_to_text(audio)
    if text:
      return HttpResponseRedirect("https://www.google.com")