from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import json
import os

from googlecloudsdk.calliope import cli_tree
from googlecloudsdk.core import module_util
from googlecloudsdk.core.util import files
import six

def _ParameterizePath(path):
  """Return path with $HOME prefix replaced by ~."""
  home = files.GetHomeDir() + os.path.sep
  if path.startswith(home):
    return '~' + os.path.sep + path[len(home):]
  return path

class CliTreeInfo(object):
  """Info for one CLI tree. A list of these is returned by ListAll()."""

  def __init__(self, command, path, version, cli_version, command_installed,
               error):
    self.command = command
    self.path = path
    self.version = version
    self.cli_version = cli_version
    self.command_installed = command_installed
    self.error = error

def ListAll(directory=None):
  """Returns the CliTreeInfo list of all available CLI trees.

  Args:
    directory: The config directory containing the CLI tree modules.

  Raises:
    CliTreeVersionError: loaded tree version mismatch
    ImportModuleError: import errors

  Returns:
    The CLI tree.
  """
  # List all CLIs by searching directories in order. .py, .pyc, and .json
  # files are treated as CLI modules/data, where the file base name is the name
  # of the CLI root command.
  directories = [
      directory,  # Explicit caller override dir
      cli_tree.CliTreeConfigDir(),  # Config dir shared across installations
      cli_tree.CliTreeDir(),  # Installation dir controlled by the updater
  ]

  trees = []
  for directory in directories:
    if not directory or not os.path.exists(directory):
      continue
    for (dirpath, _, filenames) in os.walk(six.text_type(directory)):
      for filename in sorted(filenames):  # For stability across runs.
        base, extension = os.path.splitext(filename)
        if base == '__init__' or '.' in base:
          # Ignore Python droppings and names containing more than one dot.
          continue
        path = os.path.join(dirpath, filename)
        error = ''
        tree = None
        if extension in ('.py', '.pyc'):
          try:
            module = module_util.ImportPath(path)
          except module_util.ImportModuleError as e:
            error = six.text_type(e)
          try:
            tree = module.TREE
          except AttributeError:
            tree = None
        elif extension == '.json':
          try:
            tree = json.loads(files.ReadFileContents(path))
          except Exception as e:  # pylint: disable=broad-except, record all errors
            error = six.text_type(e)
        if tree:
          version = tree.get(cli_tree.LOOKUP_VERSION, 'UNKNOWN')
          cli_version = tree.get(cli_tree.LOOKUP_CLI_VERSION, 'UNKNOWN')
          del tree
        else:
          version = 'UNKNOWN'
          cli_version = 'UNKNOWN'
        trees.append(CliTreeInfo(
            command=base,
            path=_ParameterizePath(path),
            version=version,
            cli_version=cli_version,
            command_installed=bool(files.FindExecutableOnPath(base)),
            error=error))
      # Don't search subdirectories.
      break
  return trees


from pygame.rect import *
from Pkg.Denovo_Learning.Parameters import Parameters

import pygame as pg

class Target:

    def __init__(self):
        self.rect = Rect(Parameters.target_x, Parameters.target_y, Parameters.target_diameter, Parameters.target_diameter)

    def move(self):
        self.rect.move_ip(Parameters.speed)

        if self.rect.left < 0:
            Parameters.speed[0] *= -1
        if self.rect.right > Parameters.width:
            Parameters.speed[0] *= -1
        if self.rect.top < 0:
            Parameters.speed[1] *= -1
        if self.rect.bottom > Parameters.height:
            Parameters.speed[1] *= -1

    def update(self, mode, Screen):
        if mode == 'base':
            self.target = pg.draw.ellipse(Screen.screen, Parameters.gray, self.rect, width = 1)
        elif mode == 'adapt':
            self.target = pg.draw.ellipse(Screen.screen, Parameters.blue, self.rect, width = 1)
        elif mode == 'reverse':
            self.target = pg.draw.ellipse(Screen.screen, Parameters.yellow, self.rect, width = 1)

    def get_pos(self):
        return [list(self.rect.copy())[0],list(self.rect.copy())[1]]


# -*-coding：utf-8-*-
from bs4 import BeautifulSoup
import requests

if __name__ == '__main__':
    target = 'http://www.xiaoshuotxt.org/mingzhu/18892/'
    req = requests.get(url=target)
    html = req.text
    bf = BeautifulSoup(html)
    texts = bf.find_all('div', class_='neir')
    print(texts)


import requests
import bs4
import glob

id = 25505347

while True:
    url = f"https://news.ycombinator.com/item?id={id}"
    print(url)
    resp = requests.get(url)
    open(f"data/stories/{id}", 'w').write(resp.text)
    id -= 1


# TODO should check if the config file exists before trying to use it.

import simplejson
# import nose # removing as I don't think it's used. - GDB 18-11-05
import os
from jsonschema import validate, ValidationError, Draft4Validator
import argparse
import configparser
import logging
# import glob
import fnmatch
from datetime import date
import importlib
import zipfile # to decompress
script_dir = os.path.dirname(os.path.realpath(__file__))
try:
    spec = importlib.util.spec_from_file_location("veris_logger", script_dir + "/veris_logger.py")
    veris_logger = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(veris_logger)
    # veris_logger = imp.load_source("veris_logger", script_dir + "/veris_logger.py")
except:
    print("Script dir: {0}.".format(script_dir))
    raise

#logging.basicConfig(level=logging.INFO, format=FORMAT.format(""), datefmt='%m/%d/%Y %H:%M:%S')

#defaultSchema = "../verisc.json"
#defaultEnum = "../verisc-enum.json"
defaultMerged = "../verisc-merged.json"

def checkMalwareIntegrity(inDict):
    if 'malware' in inDict.get("action", {}):
        if 'Software installation' not in inDict.get('attribute',{}).get('integrity',{}).get('variety',[]):
          yield ValidationError("Malware present, but no Software installation in attribute.integrity.variety")

def checkSocialIntegrity(inDict):
  if 'social' in inDict.get("action", {}):
    if 'Alter behavior' not in inDict.get('attribute',{}).get('integrity',{}).get('variety',[]):
      yield ValidationError("acton.social present, but Alter behavior not in attribute.integrity.variety")

def checkSQLiRepurpose(inDict):
  if 'SQLi' in inDict.get('action',{}).get('hacking',{}).get('variety',[]):
    if 'Repurpose' not in inDict.get('attribute',{}).get('integrity',{}).get('variety',[]):
      yield ValidationError("action.hacking.SQLi present but Repurpose not in attribute.integrity.variety")

def checkSecurityIncident(inDict):
  if 'security_incident' not in inDict:
    yield ValidationError("security_incident not in incident.")
  else:
    if inDict['security_incident'] == "Confirmed" and 'attribute' not in inDict:
      yield ValidationError("security_incident Confirmed but attribute section not present")

def checkLossTheftAvailability(inDict):
  expectLoss = False
  if 'Theft' in inDict.get('action',{}).get('physical',{}).get('variety',[]):
    expectLoss = True
  if 'Loss' in inDict.get('action',{}).get('error',{}).get('variety',[]):
    expectLoss = True
  if expectLoss:
    if 'Loss' not in inDict.get('attribute',{}).get('availability',{}).get('variety',[]):
      yield ValidationError("action.physical.theft or action.error.loss present but attribute.availability.loss not present")

def checkPlusAttributeConsistency(inDict):
  if 'confidentiality' in inDict.get('plus', {}).get('attribute', {}):
    if 'confidentiality' not in inDict.get('attribute', {}):
      yield ValidationError("plus.attribute.confidentiality present but confidentiality is not an affected attribute.")

def checkYear(inDict):
    if inDict.get('plus', {}).get('dbir_year', None):
        dbir_year = inDict['plus']['dbir_year']
        nyear = inDict.get('plus', {}).get('timeline', {}).get('notification', {}).get('year', None)
        nmonth = inDict.get('plus', {}).get('timeline', {}).get('notification', {}).get('month', None)
        nday = inDict.get('plus', {}).get('timeline', {}).get('notification', {}).get('day', None)
        iyear = inDict.get('timeline', {}).get('incident', {}).get('year', None)
        imonth = inDict.get('timeline', {}).get('incident', {}).get('month', None)
        iday = inDict.get('timeline', {}).get('incident', {}).get('day', None)
        discovered = inDict.get('timeline', {}).get('discovered', {}).get('unit', "(no discovery unit)")
        if nyear is not None:
            source = "notification"
            tyear = nyear
            tmonth = nmonth
        else:
            tyear = iyear
            tmonth = imonth
            source = "incident"
        if tyear >= dbir_year:
            yield ValidationError("DBIR year of {0} from {5} runs from Nov 1, {1} to Oct 31, {2}. Incident year {3} and month {4} is too late to be in this DBIR year.".format(
                dbir_year, dbir_year - 2, dbir_year - 1, tyear, tmonth, source))
        if tyear == dbir_year - 1:
            if tmonth is not None and tmonth > 10:
                yield ValidationError("DBIR year of {0} from {5} runs from Nov 1, {1} to Oct 31, {2}. Incident year {3} and month {4} is too late to be in this DBIR year.".format(
                    dbir_year, dbir_year - 2, dbir_year - 1, tyear, tmonth, source))
        elif tyear == dbir_year - 2:
            if tmonth is not None and tmonth < 11:
                if discovered in ["Months", "Years"]:
                    yield ValidationError("DBIR year of {0} from {5} runs from Nov 1, {1} to Oct 31, {2}. Incident year {3}, month {4}, and discovery unit {6} is before this range.".format(
                        dbir_year, dbir_year - 2, dbir_year - 1, tyear, tmonth, source, discovered))
        else:
            if discovered != "Years":
                yield ValidationError("DBIR year of {0} from {4} runs from Nov 1, {1} to Oct 31, {2}. Incident year {3} and discovery unit {5} is before this range.".format(
                    dbir_year, dbir_year - 2, dbir_year - 1, tyear, source, discovered))
        # check if incident or notification dates are in future
        if nyear is not None:
            try:
                ndate = date(*[x if x else 1 for x in [nyear, nmonth, nday]])
            except ValueError as e:
                yield ValidationError("Problem with notification date: {0}".format(e))
            if ndate > date.today():
                yield ValidationError("Notification date {0} is greater than today's date {1}.".format(ndate, date.today()))
        try:
            idate = date(*[x if x else 1 for x in [iyear, imonth, iday]])
        except ValueError as e:
            yield ValidationError("Problem with incident date: {0}".format(e))
        if idate > date.today():
            yield ValidationError("Incident date {0} is greater than today's date {1}.".format(idate, date.today()))
        if nyear is not None and idate > ndate:
            yield ValidationError("Notification date {0} appears to be earlier than incident date {1}. This may be due to incomplete dates.".format(ndate, idate))

### sanity check impact.overall_amount to ensure it's at least the sum of losses
## VERIS issue #142
def checkImpactTotal(inDict):
    if 'loss' in inDict.get('impact', {}):
        sum_of_amounts = sum([k.get('amount', 0) for k in inDict['impact']['loss']])
        if sum_of_amounts > inDict['impact'].get('overall_amount', 0):
            yield ValidationError("The amounts in impact.loss sum to {0}, but impact.overall_amount is {1}.  ".format(sum_of_amounts, inDict['impact'].get('overall_amount', "Not Present")) +
                                  "impact.overall_amount should at least be as much as the sum of individual losses.")

### sanity check attribute.confidentiality.data_total to ensure it's at least the max of data lost
## VERIS issue #143
def checkDataTotal(inDict):
    if 'data' in inDict['attribute'].get('confidentiality', {}):
        max_of_amounts = max([k.get('amount', 0) for k in inDict['attribute']['confidentiality']['data']])
        if max_of_amounts > inDict['attribute']['confidentiality'].get('data_total', 0):
            yield ValidationError("The maximum amount of attribute.confidentiality.data.amount is {0}, but attribute.confidentiality.data_total is {1}.  ".format(max_of_amounts, inDict['attribute']['confidentiality'].get('data_total', "Not Present")) +
                                  "attribute.confidentiality.data_total should at least be as much as the max of individual data amounts.")

### sanity check. if 'misuse', 'actor' should include 'external'
### VERIS issue #229
def checkMisuseActor(inDict):
    if 'misuse' in inDict['action'] and 'internal' not in inDict['actor'] and 'partner' not in inDict['actor']:
        yield ValidationError("Misuse in action, but no internal or partner actor defined.  Per VERIS issue #229, there should always be an internal or partner actor if there is a misuse action.")

### Validate victim region field
### VERIS issue #180
def checkRegion(inDict):
    regions = {
        "002": ["011", "014", "017", "018", "015", "000"], # Africa
        "010": ["000"], # Antarctica
        "019": ["021", "005", "013", "029", "419", "000"], # America
        "419": ["005", "013", "029"], # Latin America and Caribbean
        "142": ["030", "034", "035", "143", "145", "000"], # Asia
        "150": ["039", "151", "154", "830", "155", "000"], # Europe
        "009": ["053", "054", "057", "061", "000"], # Oceania
        "000": ["000"] # Unknown
    }
    non_used_regions = {
        "001": "000", # world
        "020": "002", # sub-saharan africa
        "003": "019" # americas"
    }

    for region in inDict.get('victim', {}).get('region', []):
        if type(region) != str or len(region) != 6:
            yield ValidationError("Victim.region {0} is not a six character string.".format(region))
        else:
            super_region = region[:3]
            sub_region = region[3:]
            if super_region in non_used_regions.keys():
                yield ValidationError("Region {2} is incorrect. Replace first half of victim.region ('{0}') with '{1}'.".format(super_region, non_used_regions[super_region], region))
#            elif sub_region in non_used_regions.keys():
#                yield ValidationError("Region {1} is incorrect. Replace second half of victim.region ('{0}') with '000' unless you know the correct region.".format(sub_region, region))
            elif sub_region not in regions.get(super_region, []):
                error_text = "victim.region second half, ('{0}') does not match first half '{1}'".format(sub_region, super_region)
                if super_region in regions:
                    error_text += "  Please replace the second half with one of {0}.".format(", ".join(regions[super_region]))
                else:
                    error_text += "  Since {0} is not a super region, please check the whole region.".format(super_region)
                yield ValidationError(error_text)

### Validate that secondary.victim.amount is > 0 if victim.secondary.victim_id is not empty
### VERIS issue #407
def checkSecondaryVictimAmount(inDict):
    secondary_victims = inDict.get('victim', {}).get('secondary', {}).get('victim_id', [])
    secondary_victim_amount = inDict.get('victim', {}).get('secondary', {}).get('amount', 0)
    if len(secondary_victims) > secondary_victim_amount:
        yield ValidationError("victim.secondary.victim_id lists {0} victims, however victim.secondary.amount is less or missing. Check that the amount is correct.  The amount should be the number of known victims (even if unknown victims may exist.)".format(len(secondary_victims)))

### Check value_chain enumerations that are likely based on actions taken
### VERIS issue #400
def checkValueChain(inDict):
    any_value_chain_recommendation = False
    ### Check enrichments from rules.py
    if 'Phishing' in inDict['action'].get('social', {}).get('variety', []):
        if "Email" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
            yield ValidationError("Because incident includes 'action.social.variety.Phishing', 'value_chain.deveopment.variety.Email' should also. Please update.")
        if "Email addresses" not in inDict.get('value_chain', {}).get('targeting', {}).get('variety', []):
            yield ValidationError("Because incident includes 'action.social.variety.Phishing', 'value_chain.targeting.variety.Email addresses' should also. Please update.")
    if 'C2' in inDict['action'].get('malware', {}).get('vector', []) and "C2" not in inDict.get('value_chain', {}).get('non-distribution services', {}).get('variety', []):
        yield ValidationError("Because there is action.malware.variety.C2, value_chain.non-distribution services.variety.C2 should also.  Please update.")
    if 'Ransomware' in inDict['action'].get('malware', {}).get('variety', []):
        if "Cryptocurrency" not in inDict.get('value_chain', {}).get('cash-out', {}).get('variety', []):
            yield ValidationError("Because there is action.malware.variety.Ransomware, value_chain.cash-out.variety.Cryptocurrency should also.  Please update.")
        # note: this is a recommendation, not enrichment
        if "Ransomware" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
            yield ValidationError("Because there is action.malware.variety.Ransomware, consider adding value_chain.distribution.variety.Ransomware.")
            any_value_chain_recommendation = True
### This is recommend only
#    if 'Trojan' in inDict['action'].get('malware', {}).get('variety', []) and "Trojan" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
#        yield ValidationError("Because there is action.malware.variety.Trojan, value_chain.development.variety.Trojan should also.  Please update.")
    if 'Email' in inDict['action'].get('social', {}).get('vector', []):
        if "Email" not in inDict.get('value_chain', {}).get('distribution', {}).get('variety', []):
            yield ValidationError("Because there is action.social.variety.Email, value_chain.distribution.variety.Email should also.  Please update.")
        # note: this is a recommendation, not enrichment
        if "Email addresses" not in inDict.get('value_chain', {}).get('targeting', {}).get('variety', []):
            yield ValidationError("Because there is action.social.variety.Email addresses, consider adding value_chain.targeting.variety.Email.")
            any_value_chain_recommendation = True

    ### Recommended Only
    if ('malware' in inDict['action'].keys()) & (len(inDict.get('value_chain', {}).get('development', {}).get('variety', [])) == 0):
        yield ValidationError("Because there is a malware action, consider adding a value_chain development variety, even if it is 'Unknown'.")
        any_value_chain_recommendation = True
    if 'Trojan' in inDict['action'].get('malware', {}).get('variety', []) and "Trojan" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
        yield ValidationError("Because there is action.malware.variety.Trojan, consider adding value_chain.development.variety.Trojan.")
        any_value_chain_recommendation = True
    if 'Pretexting' in inDict['action'].get('social', {}).get('variety', []) and "Persona" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
        yield ValidationError("Because there is action.social.variety.Pretexting, consider adding value_chain.development.variety.Persona.")
        any_value_chain_recommendation = True
    if 'Use of stolen creds' in inDict['action'].get('hacking', {}).get('variety', []) and "Lost or stolen credentials" not in inDict.get('value_chain', {}).get('targeting', {}).get('variety', []):
        yield ValidationError("Because there is action.hacking.variety.Use of stolen creds, consider adding value_chain.targeting.variety.Lost or stolen credentials, but only if the creds are the beginning of the attack and not stolen mid attack.")
        any_value_chain_recommendation = True
    if 'Exploit vuln' in inDict['action'].get('hacking', {}).get('variety', []):
        if "Vulnerabilities" not in inDict.get('value_chain', {}).get('targeting', {}).get('variety', []):
            yield ValidationError("Because there is action.hacking.variety.Exploit vuln, consider adding value_chain.targeting.variety.Vulnerabilities.")
            any_value_chain_recommendation = True
        if "Exploit" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
            yield ValidationError("Because there is action.hacking.variety.Exploit vuln, consider adding value_chain.development.variety.Exploit.")
            any_value_chain_recommendation = True
    if 'Downloader' in inDict['action'].get('malware', {}).get('variety', []) and "Loader" not in inDict.get('value_chain', {}).get('distribution', {}).get('variety', []):
        yield ValidationError("Because there is action.malware.variety.Downloader, consider adding value_chain.distribution.variety.Loader.")
        any_value_chain_recommendation = True
    if 'Exploit misconfig' in inDict['action'].get('hacking', {}).get('variety', []) and "Misconfigurations" not in inDict.get('value_chain', {}).get('targeting', {}).get('variety', []):
        yield ValidationError("Because there is action.hacking.variety.Exploit misconfig, consider adding value_chain.targeting.variety.Misconfigurations.")
        any_value_chain_recommendation = True
    if 'Exploit misconfig' in inDict['action'].get('malware', {}).get('variety', []) and "Misconfigurations" not in inDict.get('value_chain', {}).get('targeting', {}).get('variety', []):
        yield ValidationError("Because there is action.malware.variety.Exploit misconfig, consider adding value_chain.targeting.variety.Misconfigurations.")
        any_value_chain_recommendation = True
    if 'Web application' in inDict['action'].get('malware', {}).get('vector', []) and "Website" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
        yield ValidationError("Because there is action.malware.vector.Web application, consider adding value_chain.development.variety.Website.")
        any_value_chain_recommendation = True
    if 'Web application' in inDict['action'].get('social', {}).get('vector', []) and "Website" not in inDict.get('value_chain', {}).get('development', {}).get('variety', []):
        yield ValidationError("Because there is action.social.vector.Web application, consider adding value_chain.development.variety.Website.")
        any_value_chain_recommendation = True
    if any_value_chain_recommendation:
        yield ValidationError("Some value_chain changes are recommendations only.  This could be for one of three reasons: The actor could have gotten it free.  The actor could have obtained it during the incident (e.g. creds).  Website describes building a full website, not just uploading a file or such.  Please take these into account when updating the incident.")

#https://github.com/vz-risk/veris/issues/429 Interruption should be for defacement
def checkDefacementInterruption(incident):
    if 'Defacement' in incident.get('attribute',{}).get('integrity',{}).get('variety', {}):
        if 'availability' not in incident.get('attribute',{}):
            yield ValidationError("Because there is a defacement, consider adding Interruption to Availability")
        elif 'Interruption' not in incident.get('attribute',{}).get('availability'):
            yield ValidationError("Because there is a defacement, consider adding Interruption to Availability")

def main(incident):
  for e in checkMalwareIntegrity(incident):
    yield e
  for e in checkSocialIntegrity(incident):
    yield e
  for e in checkSQLiRepurpose(incident):
    yield e
  for e in checkSecurityIncident(incident):
    yield e
  for e in checkLossTheftAvailability(incident):
    yield e
  for e in checkPlusAttributeConsistency(incident):
    yield e
  for e in checkYear(incident):
    yield e
  for e in checkImpactTotal(incident):
    yield e
  for e in checkMisuseActor(incident):
    yield e
  for e in checkDataTotal(incident):
    yield e
# Added with 1.3.6
  for e in checkRegion(incident):
    yield e
  for e in checkSecondaryVictimAmount(incident):
    yield e
  for e in checkValueChain(incident):
    yield e
 #Add with 1.3.7
  for e in checkDefacementInterruption(incident):
    yield e

if __name__ == '__main__':
    # TODO: implement config file options for all of these
    parser = argparse.ArgumentParser(description="Checks a set of json files to see if they are valid VERIS incidents")
    parser.add_argument("-m","--mergedfile", help="The fully merged json schema file.")
    #parser.add_argument("-s", "--schema", help="schema file to validate with")
    #parser.add_argument("-e", "--enum", help="enumeration file to validate with")
    parser.add_argument("-l", "--log_level", choices=["error", "critical", "warning", "info", "debug"],
                        help="Minimum logging level to display", default="warning")
    parser.add_argument('--log_file', help='Location of log file')
    parser.add_argument("-i", "--input", nargs='+', help="list of paths to search for incidents")
    #parser.add_argument("-u", "--plus", help="optional schema for plus section")
    parser.add_argument('--conf', help='The location of the config file', default="../user/data_flow.cfg")
    args = parser.parse_args()
    args = {k:v for k,v in vars(args).items() if v is not None}

    # Parse the config file
    cfg = {}
    try:
        config = configparser.ConfigParser()
        config.readfp(open(args["conf"]))
        cfg_key = {
            'GENERAL': ['report', 'input', 'output', 'analysis', 'year', 'force_analyst', 'version', 'database', 'check'],
            'LOGGING': ['log_level', 'log_file'],
            'REPO': ['veris', 'dbir_private'],
            'VERIS': ['mergedfile', 'enumfile', 'schemafile', 'labelsfile', 'countryfile']
        }
        for section in cfg_key.keys():
            if config.has_section(section):
                for value in cfg_key[section]:
                    if value.lower() in config.options(section):
                        cfg[value] = config.get(section, value)
        cfg["year"] = int(cfg["year"])
        veris_logger.updateLogger(cfg)
        logging.debug("config import succeeded.")
    except Exception as e:
        logging.warning("config import failed.")
        #raise e
        pass

    cfg.update(args)
    dateFmt = '%m/%d/%Y %H:%M:%S'
    veris_logger.updateLogger(cfg, None, dateFmt)

    logging.debug(args)
    logging.debug(cfg)

    if cfg.get("mergedfile", ""):
        if type(cfg['mergedfile']) == dict:
            schema = cfg['mergedfile']
        else:
            try:
                schema = simplejson.loads(open(cfg["mergedfile"]).read())
            except IOError:
                logging.critical("ERROR: mergedfile not found. Cannot continue.")
                raise
                # exit(1)
            except simplejson.scanner.JSONDecodeError:
                logging.critical("ERROR: mergedfile is not parsing properly. Cannot continue.")
                raise
                # exit(1)
    # removed schema joining.  If you need a merged schema, use mergeSchema.py to generate one. - gdb 061416
    else:
      IOError("ERROR: mergedfile not found.  Cannot continue.")
      # exit(1)

    # Create validator
    validator = Draft4Validator(schema)

    # data_paths = []
    # if args.path:
    #     data_paths = args.path
    # else:  # only use config option if nothing is specified on the command line
    #     try:
    #         path_to_parse = cfg.get('input')
    #         data_paths = path_to_parse.strip().split('\n')
    #     except ConfigParser.Error:
    #         logging.warning("No path specified in config file. Using default")
    #         data_paths = ['.']
    #         pass

    # if "input" in cfg:
    #     cfg["input"] = [l.strip() for l in cfg["input"].split(" ,")]  # spit to list
    # else:
    #     raise ValueError("No input director or file provided to validate.")

    # files_to_validate = set()
    incident_counter = 0
    for src in cfg["input"]:
        if os.path.isfile(src) and src.lower().endswith(".json"):
            logging.debug("Now validating {0}.".format(src))
            # errors in json
            try:
                incident = simplejson.load(open(src))
                if type(incident) == list:
                    logging.warning("ERROR: %s is a list (presumably of VERIS json).  If so, it must be zipped to be processed. Skipping" % inFile)
                    continue
            except simplejson.scanner.JSONDecodeError:
                logging.warning("ERROR: %s did not parse properly. Skipping" % src)
                continue
            # replacing vakudate() with iterating errors - 171206 - GDB
            for e in validator.iter_errors(incident):
                offendingPath = '.'.join(str(x) for x in e.path)
                logging.warning("ERROR in %s. %s %s" % (src, offendingPath, e.message))
            # capture errors from main
            # try:
            #     # validator.validate(incident) # replacing with iterating errors - 171206 GDB
            #     main(incident)
            # except ValidationError as e:
            #     offendingPath = '.'.join(str(x) for x in e.path)
            #     logging.warning("ERROR in %s. %s %s" % (src, offendingPath, e.message))
            for e in main(incident):
                offendingPath = '.'.join(str(x) for x in e.path)
                logging.warning("ERROR in %s. %s %s" % (src, offendingPath, e.message))

            incident_counter += 1
            if incident_counter % 100 == 0:
                logging.info("%s incident validated" % incident_counter)
        elif os.path.isfile(src) and src.lower().endswith(".zip"):
            with zipfile.ZipFile(src, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
                for jfile in zf.namelist():
                    with zf.open(jfile) as filehandle:
                        try:
                            incidents = simplejson.load(filehandle)
                        except simplejson.scanner.JSONDecodeError:
                            logging.warning("ERROR: %s in %s did not parse properly. Skipping" % (jfile, src))
                            continue
                        for incident in incidents:
                            for e in validator.iter_errors(incident):
                                offendingPath = '.'.join(str(x) for x in e.path)
                                logging.warning("ERROR in incident %s in file %s in zipfile %s. %s %s" % (incident.get("plus", {}).get("master_id", "Unknown plus.master_id"), jfile, src, offendingPath, e.message))
                            for e in main(incident):
                                offendingPath = '.'.join(str(x) for x in e.path)
                                logging.warning("ERROR in incident %s in file %s in zipfile %s. %s %s" % (incident.get("plus", {}).get("master_id", "Unknown plus.master_id"), jfile, src, offendingPath, e.message))

                            incident_counter += 1
                            if incident_counter % 100 == 0:
                                logging.info("%s incident validated" % incident_counter)
        elif os.path.isdir(src):
            logging.debug("Now validating files in {0}.".format(src))
            src = src.rstrip("/")
            # for inFile in glob.iglob(src + "/*/*.json"):
            for root, dirnames, filenames in os.walk(src):
                for filename in fnmatch.filter(filenames, '*.[Jj][Ss][Oo][Nn]'):
                    inFile = os.path.join(root, filename)
                    # files_to_validate.add(inFile)
                    try:
                        incident = simplejson.load(open(inFile))
                        if type(incident) == list:
                            logging.warning("ERROR: %s is a list (presumably of VERIS json).  If so, it must be zipped to be processed.  Skipping." % inFile)
                            continue
                    except simplejson.scanner.JSONDecodeError:
                        logging.warning("ERROR: %s did not parse properly. Skipping" % inFile)
                        continue
                    for e in validator.iter_errors(incident):
                        offendingPath = '.'.join(str(x) for x in e.path)
                        logging.warning("ERROR in %s. %s %s" % (inFile, offendingPath, e.message))
                    for e in main(incident):
                        offendingPath = '.'.join(str(x) for x in e.path)
                        logging.warning("ERROR in %s. %s %s" % (inFile, offendingPath, e.message))

                    incident_counter += 1
                    if incident_counter % 100 == 0:
                        logging.info("%s incident validated" % incident_counter)
                for filename in fnmatch.filter(filenames, '*.zip'):
                    inFile = os.path.join(root, filename)
                    with zipfile.ZipFile(inFile, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
                        for jfile in zf.namelist():
                            with zf.open(jfile) as filehandle:
                                try:
                                    incidents = simplejson.load(filehandle)
                                except simplejson.scanner.JSONDecodeError:
                                    logging.warning("ERROR: %s in %s did not parse properly. Skipping" % (jfile, inFile))
                                for incident in incidents:
                                    for e in validator.iter_errors(incident):
                                        offendingPath = '.'.join(str(x) for x in e.path)
                                        logging.warning("ERROR in incident %s in file %s in zipfile %s. %s %s" % (incident.get("plus", {}).get("master_id", "Unknown plus.master_id"), jfile, inFile, offendingPath, e.message))
                                    for e in main(incident):
                                        offendingPath = '.'.join(str(x) for x in e.path)
                                        logging.warning("ERROR in incident %s in file %s in zipfile %s. %s %s" % (incident.get("plus", {}).get("master_id", "Unknown plus.master_id"), jfile, inFile, offendingPath, e.message))

                                    incident_counter += 1
                                    if incident_counter % 100 == 0:
                                        logging.info("%s incident validated" % incident_counter)
        else:
            logging.warning("%s did not match a known veris format.  Skipping." % src)

    logging.info("schema assembled successfully.")
    # logging.debug(simplejson.dumps(schema,indent=2,sort_keys=True))

    # data_paths = [x + '/*.json' for x in data_paths]
    # incident_counter = 0
    # for eachDir in data_paths:
    #     for eachFile in glob(eachDir):
    #       logging.debug("Now validating %s" % eachFile)
    #       try:
    #           incident = simplejson.loads(open(eachFile).read())
    #       except simplejson.scanner.JSONDecodeError:
    #           logging.warning("ERROR: %s did not parse properly. Skipping" % eachFile)
    #           continue

          # try:
          #     #validate(incident, schema)
          #     validator.validate(incident)
          #     main(incident)
          # except ValidationError as e:
          #     offendingPath = '.'.join(str(x) for x in e.path)
          #     logging.warning("ERROR in %s. %s %s" % (eachFile, offendingPath, e.message))

          # incident_counter += 1
          # if incident_counter % 100 == 0:
          #     logging.info("%s incident validated" % incident_counter)

    logging.info("checkValidity complete")


#!/usr/bin/env python

from pyfirmata import Arduino, util
from time import sleep
import os

port = '/dev/ttyUSB0'
board = Arduino(port)
sleep(2)

it = util.Iterator(board)
it.start()

analog0 = board.get_pin('a:0:i')
analog1 = board.get_pin('a:1:i')
analog2 = board.get_pin('a:2:i')
digital9 = board.get_pin('d:9:p')
digital10 = board.get_pin('d:10:p')
digital11 = board.get_pin('d:11:p')

def rgbcolor(red, green, blue):
	digital9.write(red)
	digital10.write(blue)
	digital11.write(green)

try:
	while True:
		sleep(.020)
		analog_0 = analog0.read()
		analog_1 = analog1.read()
		analog_2 = analog2.read()
		rgbcolor(analog_0,analog_1,analog_2)
except KeyboardInterrupt:
	board.exit()
	os._exit


import numpy as np
import casadi as ca

# from five_link_model import Biped1
# from five_link_model import Biped2

from hopper_model import Hopper

from terrain import Terrain
# import helper_functions as hf

##################################
#### Remove ddpos ###############
#################################

class NLP():
    def __init__(self, knot_points_per_phase, steps, total_duration, model='hopper', terrain='flat'):
        super().__init__()
        self.knot_points_per_phase = knot_points_per_phase
        self.num_phases            = steps
        self.total_duration        = total_duration

        if model == 'hopper':
            self.model = Hopper()
            self.time_phases = {}

        self.terrain = Terrain(type=terrain)

        self.opti   = ca.Opti()

        self.q    = {}
        self.qdot = {}
        self.u    = {}
        self.p0   = {} # i = 0
        self.dp0  = {} # i = 0
        self.f10  = {} # i = 0

        self.ceq = []
        self.ciq = []

        self.setVariables()
        self.setConstraints()
        self.setBounds()

    def setVariables(self):
        for step in range(self.num_phases):

            self.time_phases.update({str(step) : self.opti.variable(1)})
            self.ceq.append(0<=self.time_phases[str(step)])

            self.q.update(   {str(step) : []})
            self.qdot.update({str(step) : []})

            self.p0.update(  {str(step) : []})
            self.dp0.update( {str(step) : []})

            self.u.update(   {str(step) : []})
            self.f10.update( {str(step) : []})

            for knot_point in range(self.knot_points_per_phase):
                self.q   [str(step)].append(self.opti.variable(3))
                self.qdot[str(step)].append(self.opti.variable(3))

                self.p0  [str(step)].append(self.opti.variable(2))
                self.dp0 [str(step)].append(self.opti.variable(2))

                self.u   [str(step)].append(self.opti.variable(2))
                self.f10 [str(step)].append(self.opti.variable(2))

                q    = self.q   [str(step)][-1]
                qdot = self.qdot[str(step)][-1]
                u    = self.u   [str(step)][-1]
                p0   = self.p0  [str(step)][-1]
                dp0  = self.dp0 [str(step)][-1]
                f10  = self.f10 [str(step)][-1]

                self.model.setFullState(q=q, dq=qdot,
                                        p0=p0, dp0=dp0,
                                        u=u, f10=f10)

                ###--Add Model Constraints--###
                # self.opti.subject_to(self.model.p       ['Constraint'])
                # self.opti.subject_to(self.model.dynamics['Constraint'])

                self.ceq.append(self.model.p['Constraint'])
                # self.ceq.append(self.model.dynamics['Constraint'])

                # self.opti.minimize(p0[1]**2)
                # self.opti.minimize(ca.sumsqr(self.model.p['Leg'][1,:]))

                for i in range(3):
                    self.opti.subject_to((self.model.p['Leg'][1,i]) >= self.terrain.heightMap(self.model.p['Leg'][0,:]))

                # if j==0 and n==0:
                #     self.setInitialBoundary()
                #     com_x = (self.model.c['Left Leg'][0,:]) + (self.model.c['Right Leg'][0,:]) + (self.model.c['Torso'][0,:])
                #     self.opti.subject_to(com_x==0)

                # if j==self.num_phases-1 and n==self.num_phases-1:
                # com_y = ca.sum1(self.model.c['Leg'][1,:])
                # self.opti.subject_to(com_y<=self.model.b)

    def setConstraints(self):
        # self.setBoundary()
        self.setContactConstraints()
        self.setCollocationContraints()

        self.opti.subject_to(self.ceq)

    # def setBoundary(self):
        # self.opti.subject_to(self.q [str(0)][0] == self.model.initial_q)
        # self.opti.subject_to(self.p0[str(0)][0] == np.zeros((2,1)))
        # self.opti.subject_to(self.q[str(self.num_phases-1)][-1] == self.model.final_q)
        # self.opti.subject_to(self.p0[str(self.num_phases-1)][-1][0] == 1)

        # self.ceq.append(self.q [str(0)][0] == self.model.initial_q)
        # self.ceq.append(self.p0[str(0)][0] == np.zeros((2,1)))

    def setContactConstraints(self):

        # self.opti.subject_to(sum(self.time_phases.values()) == self.total_duration)
        self.ceq.append(sum(self.time_phases.values()) == self.total_duration)

        for step in range(self.num_phases):

            # if step is even ==> Contact
            # if step is odd  ==> No Contact
            for knot_point in range(self.knot_points_per_phase):

                # if step > 0:
                #     # self.opti.subject_to(self.p0 [str(step-1)][-1] - self. p0[str(step)][0] == 0)
                #     # self.opti.subject_to(self.f10[str(step-1)][-1] - self.f10[str(step)][0] == 0)

                #     self.ceq.append(self.p0 [str(step-1)][-1] - self. p0[str(step)][0] == 0)
                #     self.ceq.append(self.f10[str(step-1)][-1] - self.f10[str(step)][0] == 0)

                if step%2 != 0:
                    f10 = self.f10[str(step)][knot_point]
                    p0  = self.p0 [str(step)][knot_point]

                    # self.opti.bounded(0,self.terrain.heightMap(p0[0])-p0[1],ca.inf)
                    # self.opti.subject_to(f10==0)

                    self.ceq.append(self.terrain.heightMap(p0[0])<=p0[1])
                    self.ceq.append(f10==0)
                else:
                    p0  = self.p0 [str(step)][knot_point]
                    dp0 = self.dp0[str(step)][knot_point]
                    f10 = self.f10[str(step)][knot_point]

                    # self.opti.subject_to((self.terrain.mu*f10[0])**2 - f10[1]**2 >= 0)
                    # self.opti.subject_to(ca.dot(f10,p0) >= 0)
                    # self.opti.subject_to(p0[1]==self.terrain.heightMap(p0[0]))
                    # self.opti.subject_to(dp0==0)

                    self.ceq.append((self.terrain.mu*f10[0])**2 - f10[1]**2 >= 0)
                    self.ceq.append(ca.dot(f10,p0) >= 0)
                    self.ceq.append(p0[1]==self.terrain.heightMap(p0[0]))
                    self.ceq.append(dp0==0)

    def setCollocationContraints(self):
        for step in range(self.num_phases):

            h = self.time_phases[str(step)]/self.knot_points_per_phase
            # h = 0.01
            for n in range(self.knot_points_per_phase-1):

                if step>0 and n==0:
                    q_1   , q_2    = self.q   [str(step-1)][-1], self.q   [str(step)][n]
                    qdot_1, qdot_2 = self.qdot[str(step-1)][-1], self.qdot[str(step)][n]

                    p0_1 , p0_2  = self.p0 [str(step-1)][-1], self.p0 [str(step)][n]
                    dp0_1, dp0_2 = self.dp0[str(step-1)][-1], self.dp0[str(step)][n]

                    u_1  , u_2   = self.u  [str(step-1)][-1], self.u  [str(step)][n]
                    f10_1, f10_2 = self.f10[str(step-1)][-1], self.f10[str(step)][n]

                    self.ceq.append(q_2 - q_1== 0);self.ceq.append(qdot_2 - qdot_1== 0)
                    self.ceq.append(p0_2-p0_1== 0);self.ceq.append(dp0_2-dp0_1== 0)
                    self.ceq.append(u_2-u_1== 0);self.ceq.append(f10_2-f10_1== 0)

                    # m_1 = Hopper()
                    # m_1.setFullState(q=q_1, dq=qdot_1,
                    #                 p0=p0_1, dp0=dp0_1,
                    #                 u=u_1, f10=f10_1)

                    # qddot_1 = m_1.dynamics['Leg']

                    # m_2 = Hopper()
                    # m_2.setFullState(q=q_2, dq=qdot_2,
                    #                 p0=p0_2, dp0=dp0_2,
                    #                 u=u_2, f10=f10_2)

                    # qddot_2 = m_2.dynamics['Leg']

                    # self.ceq.append((h/2) * (qdot_2  + qdot_1)  == (q_2 - q_1))
                    # self.ceq.append((h/2) * (qddot_2 + qddot_1) == (qdot_2 - qdot_1))
                    # self.ceq.append((h/2) * (dp0_2  + dp0_1) == (p0_2 - dp0_1))
                    # self.ceq.append((h)*(self.model.gravity) == (dp0_2[1] - dp0_1[1]))
                # else:
                q_1   , q_2    = self.q   [str(step)][n], self.q   [str(step)][n+1]
                qdot_1, qdot_2 = self.qdot[str(step)][n], self.qdot[str(step)][n+1]

                p0_1 , p0_2  = self.p0 [str(step)][n], self.p0 [str(step)][n+1]
                dp0_1, dp0_2 = self.dp0[str(step)][n], self.dp0[str(step)][n+1]

                u_1  , u_2   = self.u  [str(step)][n], self.u  [str(step)][n+1]
                f10_1, f10_2 = self.f10[str(step)][n], self.f10[str(step)][n+1]

                m_1 = Hopper()
                m_1.setFullState(q=q_1, dq=qdot_1,
                                p0=p0_1, dp0=dp0_1,
                                u=u_1, f10=f10_1)

                qddot_1 = m_1.dynamics['Leg']

                m_2 = Hopper()
                m_2.setFullState(q=q_2, dq=qdot_2,
                                p0=p0_2, dp0=dp0_2,
                                u=u_2, f10=f10_2)

                qddot_2 = m_2.dynamics['Leg']

                self.ceq.append((h/2) * (qdot_2  + qdot_1)  == (q_2 - q_1))
                self.ceq.append((h/2) * (qddot_2 + qddot_1) == (qdot_2 - qdot_1))
                self.ceq.append((h/2) * (dp0_2  + dp0_1) == (p0_2 - dp0_1))
                self.ceq.append((h)*(self.model.gravity) == (dp0_2[1] - dp0_1[1]))

    def setBounds(self):
        # traj=0
        for step in range(self.num_phases):
            for knot_point in range(self.knot_points_per_phase):
                q    = self.q   [str(step)][knot_point]
                qdot = self.qdot[str(step)][knot_point]

                p0  = self.p0 [str(step)][knot_point]
                dp0 = self.dp0[str(step)][knot_point]

                u   = self.u  [str(step)][knot_point]
                f10 = self.f10[str(step)][knot_point]

                self.opti.bounded([-np.pi/2]*3, q,[np.pi/2]*3)
                self.opti.bounded([-1.5]*2, u,[1.5]*2)

                # traj+=(p0[1]**2)

        # self.opti.minimize(traj)
# test check for sanity
# test_problem = NLP(knot_points_per_phase=40, steps=3, total_duration=2, model='hopper')
# print((test_problem.time_phases))


from context import functions
from enum import Enum
import unittest

class TestHooks(unittest.TestCase):
    def tearDown(self) -> None:
        for event in functions.Event:
            functions.unset_handler(event)
        return super().tearDown()

    def test_functions_contains_handlers_dict_and_Event_enum(self):
        assert hasattr(functions, '_handlers')
        assert type(functions._handlers) is dict
        assert hasattr(functions, 'Event')
        assert issubclass(functions.Event, Enum)

    def test_functions_contains_handler_set_and_unset_functions(self):
        assert hasattr(functions, '_set_handler')
        assert callable(functions._set_handler)
        assert hasattr(functions, 'set_before_handler')
        assert callable(functions.set_before_handler)
        assert hasattr(functions, 'set_after_handler')
        assert callable(functions.set_after_handler)
        assert hasattr(functions, 'unset_handler')
        assert callable(functions.unset_handler)

    def test_AFTER_HEADER_event(self):
        before = functions._header('hello')
        assert before == '# hello\n\n'
        functions.set_after_handler(
            functions.Event.AFTER_HEADER,
            lambda doc: doc + 'world'
        )
        after = functions._header('hello')
        assert after == '# hello\n\nworld'

    def test_AFTER_PARAGRAPH_event(self):
        before = functions._paragraph('hello')
        assert before == 'hello\n\n'
        functions.set_after_handler(
            functions.Event.AFTER_PARAGRAPH,
            lambda doc: doc + 'world'
        )
        after = functions._paragraph('hello')
        assert after == 'hello\n\nworld'

    def test_AFTER_LIST_event(self):
        before = functions._list('hello')
        assert before == '- hello\n'
        functions.set_after_handler(
            functions.Event.AFTER_LIST,
            lambda doc: doc + 'world'
        )
        after = functions._list('hello')
        assert after == '- hello\nworld'

    def test_BEFORE_VALUE_event(self):
        signal = False
        def handle(*args):
            nonlocal signal
            signal = True
            return args
        _ = functions.dox_a_value(signal)
        assert not signal
        functions.set_before_handler(
            functions.Event.BEFORE_VALUE,
            handle
        )
        _ = functions.dox_a_value(signal)
        assert signal

    def test_AFTER_VALUE_event(self):
        val = 'some str'
        before = functions.dox_a_value(val)
        functions.set_after_handler(
            functions.Event.AFTER_VALUE,
            lambda doc: doc + 'AFTER'
        )
        after = functions.dox_a_value(val)
        assert after == before + 'AFTER'

    def test_BEFORE_FUNCTION_event(self):
        signal = False
        def handle(*args):
            nonlocal signal
            signal = True
            return args
        _ = functions.dox_a_function(handle)
        assert not signal
        functions.set_before_handler(
            functions.Event.BEFORE_FUNCTION,
            handle
        )
        _ = functions.dox_a_function(handle)
        assert signal

    def test_AFTER_FUNCTION_event(self):
        handler = lambda doc: doc + 'AFTER'
        before = functions.dox_a_function(handler)
        functions.set_after_handler(
            functions.Event.AFTER_FUNCTION,
            handler
        )
        after = functions.dox_a_function(handler)
        assert after == before + 'AFTER'

    def test_BEFORE_CLASS_event(self):
        signal = False
        def handle(*args):
            nonlocal signal
            signal = True
            return args
        _ = functions.dox_a_class(TestHooks)
        assert not signal
        functions.set_before_handler(
            functions.Event.BEFORE_CLASS,
            handle
        )
        _ = functions.dox_a_class(TestHooks)
        assert signal

    def test_AFTER_CLASS_event(self):
        before = functions.dox_a_class(TestHooks)
        functions.set_after_handler(
            functions.Event.AFTER_CLASS,
            lambda doc: doc + 'AFTER'
        )
        after = functions.dox_a_class(TestHooks)
        assert after == before + 'AFTER'

    def test_BEFORE_MODULE_event(self):
        signal = False
        def handle(*args):
            nonlocal signal
            signal = True
            return args
        _ = functions.dox_a_module(functions)
        assert not signal
        functions.set_before_handler(
            functions.Event.BEFORE_MODULE,
            handle
        )
        _ = functions.dox_a_module(functions)
        assert signal

    def test_AFTER_MODULE_event(self):
        before = functions.dox_a_module(functions)
        functions.set_after_handler(
            functions.Event.AFTER_MODULE,
            lambda doc: doc + 'AFTER'
        )
        after = functions.dox_a_module(functions)
        assert after == before + 'AFTER'

    def test_event_handler_chaining(self):
        val = 'some str'
        before = functions.dox_a_value(val)
        functions.set_after_handler(
            functions.Event.AFTER_VALUE,
            lambda doc: doc + 'AFTER1 '
        )
        functions.set_after_handler(
            functions.Event.AFTER_VALUE,
            lambda doc: doc + 'AFTER2'
        )
        after = functions.dox_a_value(val)
        assert after == before + 'AFTER1 AFTER2'
        functions.set_after_handler(
            functions.Event.AFTER_VALUE,
            lambda doc: doc + ' AFTER3'
        )
        after = functions.dox_a_value(val)
        assert after == before + 'AFTER1 AFTER2 AFTER3'

if __name__ == '__main__':
    unittest.main()


# This files contains your custom actions which can be used to run
# custom Python code.
#
# See this guide on how to implement these action:
# https://rasa.com/docs/rasa/custom-actions

from typing import Any, Text, Dict, List

from rasa_sdk import Action, Tracker
from rasa_sdk.executor import CollectingDispatcher
import random

DATABASE = ["bún đậu mắm tôm",
            "bún đậu nước mắm",
            "bún cá",
            "bún hải sản",
            "cơm văn phòng",
            "cơm sườn",
            "xôi",
            "bún ốc",
            "mì vằn thắn",
            "hủ tiếu",
            "bún chả",
            "bún ngan",
            "ngan xào tỏi",
            "bún bò huế",
            "mì tôm hải sản",
            "bánh mì trứng xúc xích rắc thêm ít ngải cứu",
            "bánh mì trứng",
            "bánh mì xúc xích",
            "bánh mì pate"]

class ActionRecommend(Action):

    def name(self) -> Text:
        return "action_recommend"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        food = []
        for i in range(2):
            food_number = random.randrange(len(DATABASE))
            food.append(DATABASE[food_number])

        dispatcher.utter_message(
            text="Em nghĩ hôm nay anh chị có thể thử món '{}' hoặc bên cạnh đó cũng có thể là món '{}' ạ".format(food[0], food[1]))

        return []

class action_get_lottery(Action):
	def name(self):
            # Doan nay khai bao giong het ten ham ben tren la okie
		    return 'action_get_lottery'
	def run(self, dispatcher, tracker, domain):
            # Khai bao dia chi luu tru ket qua so xo. O day lam vi du nen minh lay ket qua SX Mien Bac
            url = 'https://xskt.com.vn/rss-feed/mien-bac-xsmb.rss'
            # Tien hanh lay thong tin tu URL
            feed_cnt = feedparser.parse(url)
            # Lay ket qua so xo moi nhat
            first_node = feed_cnt['entries']
            # Lay thong tin ve ngay va chi tiet cac giai
            return_msg = first_node[0]['title'] + "\n" + first_node[0]['description']
            # Tra ve cho nguoi dung
            dispatcher.utter_message(return_msg)
            return []


# -*- coding: utf-8 -*-
from __future__ import unicode_literals, division, print_function, absolute_import
import re
import logging
import time
import datetime

import pydub # https://github.com/jiaaro/pydub
from google.cloud import storage
from google.cloud.exceptions import NotFound
from google.cloud import speech
from google.cloud.speech import enums
from google.cloud.speech import types

from .path import Path, TempPath
from .utils import Time, String

logger = logging.getLogger(__name__)

class SpeechPath(Path):
    regex = re.compile(r"\.(?:mp3|wav|flac)$", re.I)

    def is_type(self):
        return True if self.regex.search(self) else False

class Speech(object):
    """Wrapper around Google's speech api

    https://cloud.google.com/speech
    https://cloud.google.com/speech/docs/reference/libraries#client-libraries-usage-python
    https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/speech/
    https://cloud.google.com/speech/support
    https://cloud.google.com/speech/docs/best-practices
    """

    TIMEOUT = 10800

    @property
    def text(self):
        return self._text

    @property
    def lines(self):
        return self.text.splitlines(False)

    @property
    def words(self):
        return self._words

    def __init__(self, path, lang):
        self.path = path
        self.lang = lang

    def _convert(self, path, start, stop):
        audio_format = "ogg"
        # https://github.com/jiaaro/pydub/blob/master/pydub/audio_segment.py
        audio = pydub.AudioSegment.from_file(path)
        if bool(start) or bool(stop):
            start = Time(start)
            stop = Time(stop)
            if start and not stop:
                logger.info("Starting audio at {}".format(start))
                audio = audio[start.total_ms():]
            elif stop and not start:
                logger.info("Stopping audio at {}".format(stop))
                audio = audio[:stop.total_ms():]
            else:
                logger.info("Slicing audio {} - {}".format(start, stop))
                audio = audio[start.total_ms():stop.total_ms()]

            tp = TempPath(
                "{}-{}-{}".format(
                    path.name,
                    start.total_seconds(),
                    stop.total_seconds()
                ),
                audio_format
            )

        else:
            tp = TempPath(path.name, audio_format)

        logger.info("Converting {} to {} at {}".format(path, audio_format, tp))

        # https://cloud.google.com/speech/docs/basics#audio-encodings
        # https://superuser.com/questions/516806/how-to-encode-audio-with-opus-codec
        #audio.export(tp, format=audio_format, codec="opus", parameters=["-strict", "-2"])
        ogg_audio = pydub.AudioSegment.from_file(
            audio.export(tp, format=audio_format, codec="libopus", parameters=["-ac", "1"])
        )

        return tp, ogg_audio

    def _upload(self, path):
        # https://cloud.google.com/python/getting-started/using-cloud-storage
        # https://github.com/GoogleCloudPlatform/google-cloud-python/
        # https://cloud.google.com/storage/docs/object-basics
        # https://googlecloudplatform.github.io/google-cloud-python/latest/storage/client.html
        storage_client = storage.Client()
        bucket_name = storage_client.project
        logger.info("Uploading {} to Google cloud storage bucket {}".format(path, bucket_name))

        try:
            bucket = storage_client.get_bucket(bucket_name)

        except NotFound:
            bucket = storage_client.create_bucket(bucket_name)

        filename = path.safename
        blob = bucket.blob(filename)
        blob.upload_from_string(path.contents(), path.mimetype)
        #public_url = blob.public_url
        # super convenient google doesn't bother with a .uri property for a
        # thing that seems to be pretty commonly needed
        # https://stackoverflow.com/a/25373496/5006
        uri = "gs://{}/{}".format(bucket_name, blob.name)
        logger.info("Google cloud storage uri {}".format(uri))
        return uri, blob

    def _transcribe(self, uri, audio):
        client = speech.SpeechClient()

        # NOTE -- according to https://cloud.google.com/speech/quotas  we are
        # limited to ~180 Minutes
        recog = types.RecognitionAudio(uri=uri)
        # https://cloud.google.com/speech/reference/rest/v1/RecognitionConfig
        config = types.RecognitionConfig(
            encoding=enums.RecognitionConfig.AudioEncoding.OGG_OPUS,
            sample_rate_hertz=audio.frame_rate,
            language_code=self.lang,
            enable_word_time_offsets=True
        )

        # https://cloud.google.com/speech/docs/async-recognize
        # https://googlecloudplatform.github.io/google-cloud-python/latest/speech/index.html#asynchronous-recognition
        operation = client.long_running_recognize(config, recog)

        start = time.time()
        logger.info('Beginning transcription')
        response = operation.result(timeout=self.TIMEOUT)
        stop = time.time()
        logger.info("Transcription finished in {}".format(Time(stop - start)))
        return response

    def scan(self, start=0, stop=0):
        path = self.path
        path, audio = self._convert(path, start, stop)
        uri, blob = self._upload(path)
        self.response = self._transcribe(uri, audio)
        logger.info("Deleting file from Google cloud storage {}".format(uri))
        blob.delete() # https://cloud.google.com/storage/docs/object-basics
        return self.response

    def __iter__(self):
        for result in self.response.results:
            alternative = result.alternatives[0]
            text = String(alternative.transcript).flow()

            # https://cloud.google.com/speech/docs/async-time-offsets
            word_info = alternative.words[0]
            # TODO -- we have a nanos property also, do we care about
            # nanoseconds?
            start_time = Time(word_info.start_time.seconds)
            yield start_time, text

#         for result in response.results:
#             # The first alternative is the most likely one for this portion.
#             print(result.alternatives[0].transcript)
#             #print('Confidence: {}'.format(result.alternatives[0].confidence))


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from openstack.network.v2 import network
from openstack.network.v2 import subnet
from openstack.network.v2 import subnet_pool
from openstack.tests.functional import base

class TestSubnetFromSubnetPool(base.BaseFunctionalTest):
    IPV4 = 4
    CIDR = "10.100.0.0/28"
    MINIMUM_PREFIX_LENGTH = 8
    DEFAULT_PREFIX_LENGTH = 24
    MAXIMUM_PREFIX_LENGTH = 32
    SUBNET_PREFIX_LENGTH = 28
    IP_VERSION = 4
    PREFIXES = ["10.100.0.0/24"]
    NET_ID = None
    SUB_ID = None
    SUB_POOL_ID = None

    def setUp(self):
        super(TestSubnetFromSubnetPool, self).setUp()
        self.NET_NAME = self.getUniqueString()
        self.SUB_NAME = self.getUniqueString()
        self.SUB_POOL_NAME = self.getUniqueString()

        sub_pool = self.user_cloud.network.create_subnet_pool(
            name=self.SUB_POOL_NAME,
            min_prefixlen=self.MINIMUM_PREFIX_LENGTH,
            default_prefixlen=self.DEFAULT_PREFIX_LENGTH,
            max_prefixlen=self.MAXIMUM_PREFIX_LENGTH,
            prefixes=self.PREFIXES,
        )
        self.assertIsInstance(sub_pool, subnet_pool.SubnetPool)
        self.assertEqual(self.SUB_POOL_NAME, sub_pool.name)
        self.SUB_POOL_ID = sub_pool.id
        net = self.user_cloud.network.create_network(name=self.NET_NAME)
        self.assertIsInstance(net, network.Network)
        self.assertEqual(self.NET_NAME, net.name)
        self.NET_ID = net.id
        sub = self.user_cloud.network.create_subnet(
            name=self.SUB_NAME,
            ip_version=self.IPV4,
            network_id=self.NET_ID,
            prefixlen=self.SUBNET_PREFIX_LENGTH,
            subnetpool_id=self.SUB_POOL_ID,
        )
        self.assertIsInstance(sub, subnet.Subnet)
        self.assertEqual(self.SUB_NAME, sub.name)
        self.SUB_ID = sub.id

    def tearDown(self):
        sot = self.user_cloud.network.delete_subnet(self.SUB_ID)
        self.assertIsNone(sot)
        sot = self.user_cloud.network.delete_network(
            self.NET_ID, ignore_missing=False
        )
        self.assertIsNone(sot)
        sot = self.user_cloud.network.delete_subnet_pool(self.SUB_POOL_ID)
        self.assertIsNone(sot)
        super(TestSubnetFromSubnetPool, self).tearDown()

    def test_get(self):
        sot = self.user_cloud.network.get_subnet(self.SUB_ID)
        self.assertEqual(self.SUB_NAME, sot.name)
        self.assertEqual(self.SUB_ID, sot.id)
        self.assertEqual(self.CIDR, sot.cidr)
        self.assertEqual(self.IPV4, sot.ip_version)
        self.assertEqual("10.100.0.1", sot.gateway_ip)
        self.assertTrue(sot.is_dhcp_enabled)


from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from justchat.models import Room, ChatHistory
from django.contrib.auth import get_user_model
from django.db.models import Q

User = get_user_model()

# def index(request):
#     return render(request, 'chat/index.html')
#
#
# def login(request):
#     return render(request, 'chat/login.html')
#
#
# def room(request, room_name):
#     return render(request, 'chat/room.html', {
#         'room_name': room_name
#     })

# @login_required
# def omchat(request):
#     rooms = Room.objects.order_by("title")
#     return render(request, 'chat/omchat.html', {
#         'rooms': rooms
#     })

@login_required
def userchat(request):
    print(f"user: ", request.user)
    a_user = User.objects.get(username=request.user)
    users = User.objects.exclude(username=a_user.username).order_by('id')
    print("a_user:", a_user.id)
    return render(request, 'userchat/user_chat.html', {
        'users': users,
        'a_user': a_user
    })

def chat_history(request):
    a_user_id = request.GET.get('a_user_id')
    b_user_id = request.GET.get('b_user_id')
    print(f"a_user_id {a_user_id}")
    print(f"b_user_id {b_user_id}")
    a_user = User.objects.get(pk=a_user_id)
    b_user = User.objects.get(pk=b_user_id)
    messages = ChatHistory.objects.filter(Q(a_party_user=a_user) | Q(a_party_user=b_user), Q(b_party_user=a_user) | Q(b_party_user=b_user)).order_by('created_at')
    return render(request, 'userchat/chat_history.html', {
        'a_user': a_user,
        'b_user': b_user,
        'messages': messages
    })

def send_chat(request):
    a_user_id = request.GET.get('a_user_id')
    b_user_id = request.GET.get('b_user_id')
    message = request.GET.get('message')
    print(f"a_user_id : {a_user_id}")
    print(f"b_user_id : {b_user_id}")
    print(f"message : {message}")

    ChatHistory.objects.create(a_party_user_id=a_user_id,
                               b_party_user_id=b_user_id,
                               message=message
                               )
    return HttpResponse("success")


#!/usr/bin/env python3
import leveldb
import msgpack
import csv
from util.misc import Benchmark, open_file

REQUIRED_KEYS = {'title', 'paper_id', 'date'}

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="Creates a LevelDB of TSV metadata in infile")
    parser.add_argument('infile')
    parser.add_argument('leveldb_path')
    parser.add_argument('--benchmark-freq', default=100000, type=int)
    parser.add_argument('--delimiter', '-d', default='\t')
    args = parser.parse_args()

    db = leveldb.LevelDB(args.leveldb_path,
                         write_buffer_size=100 << 20,  # 100MB
                         block_cache_size=400 << 20)  # 400MB

    with open_file(args.infile) as ifs:
        b = Benchmark(args.benchmark_freq)
        reader = csv.DictReader(ifs, delimiter=args.delimiter)
        for row in reader:
            if not REQUIRED_KEYS.issubset(row.keys()):
                print(row)
                raise KeyError("Not all required keys present")
            db.Put(row["paper_id"].encode(), msgpack.packb(row))
            b.increment()

    b.print_freq()
    print(db.GetStats())


from typing import List
from collections import deque

class Solution:
    def maxNumber(self, nums1: List[int], nums2: List[int], k: int) -> List[int]:
        def maxArray(nums, k):
            if k == len(nums):
                return nums
            arr = []
            le = len(nums)
            for i, n in enumerate(nums):
                while arr and arr[-1] < n and len(arr) - 1 + le - i >= k:
                    arr.pop()
                if len(arr) < k:
                    arr.append(n)
            return arr

        def merge(a1, a2):
            a1, a2 = deque(a1), deque(a2)
            return [max(a1, a2).popleft() for _ in range(len(a1) + len(a2))]

        res = [0] * k
        for i in range(max(k - len(nums2), 0), min(k, len(nums1)) + 1):
            a1 = maxArray(nums1, i)
            a2 = maxArray(nums2, k - i)
            arr = merge(a1, a2)
            if arr > res:
                res = arr
        return res

print(Solution().maxNumber([], [1], 1))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('flavourdb2', '0002_chef_chefbiofoodimage_comuna_menu_menuimage'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='comuna',
            name='chefs',
        ),
        migrations.AddField(
            model_name='chef',
            name='comunas',
            field=models.ManyToManyField(to='flavourdb2.Comuna'),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='menu',
            name='name',
            field=models.CharField(default=b'', max_length=200),
            preserve_default=True,
        ),
    ]


import shutil
import tempfile
from http import HTTPStatus

from django import forms
from django.conf import settings
from django.core.cache import cache
from django.core.files.uploadedfile import SimpleUploadedFile
from django.shortcuts import get_object_or_404
from django.test import Client, TestCase, override_settings
from django.urls import reverse
from posts.forms import PostForm
from posts.models import Follow, Group, Post, User

TEMP_MEDIA_ROOT = tempfile.mkdtemp(dir=settings.BASE_DIR)

@override_settings(MEDIA_ROOT=TEMP_MEDIA_ROOT)
class PostViewTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        # Создаем неавторизованный клиент
        cls.guest_client = Client()
        # Создаем пользователя
        cls.user = User.objects.create_user(username='HasNoName')
        cls.user2 = User.objects.create_user(username='HasNoName2')

        small_gif = (
            b'\x47\x49\x46\x38\x39\x61\x02\x00'
            b'\x01\x00\x80\x00\x00\x00\x00\x00'
            b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
            b'\x00\x00\x00\x2C\x00\x00\x00\x00'
            b'\x02\x00\x01\x00\x00\x02\x02\x0C'
            b'\x0A\x00\x3B'
        )
        uploaded = SimpleUploadedFile(
            name='small.gif',
            content=small_gif,
            content_type='image/gif'
        )

        cls.group = Group.objects.create(
            title='Тестовая Группа',
            slug='test-slug',
            description='тестовое описание группы'
        )

        cls.post = Post.objects.create(
            text='Тестовый текст',
            author=cls.user,
            group=cls.group,
            image=uploaded
        )

    def setUp(self):
        # Создаем пользователя
        self.authorized_client = Client()
        # Создаем второй клиент
        self.authorized_client2 = Client()
        # Авторизуем пользователя
        self.authorized_client.force_login(self.user)
        # Авторизуем пользователя2
        self.authorized_client2.force_login(self.user2)
        # выборка всей ленты новостей
        author_querry = Follow.objects.filter(user=self.user)
        author_values_list = author_querry.values_list('author')
        self.post_list = Post.objects.filter(author_id__in=author_values_list)
        # добавление подписки
        self.follow = Follow.objects.create(
            user=self.user,
            author=self.post.author
        )
        cache.clear()

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()
        shutil.rmtree(TEMP_MEDIA_ROOT, ignore_errors=True)

    def test_pages_posts_correct_template(self):
        """views функция использует соответствующий шаблон."""
        # Собираем в лист пары "имя_html_шаблона, reverse(name)"
        templates_pages_names = [
            (
                'posts/index.html',
                reverse('posts:index')
            ),
            (
                'posts/group_list.html', reverse(
                    'posts:group_list', kwargs={'slug': self.group.slug}
                )
            ),
            (
                'posts/profile.html', reverse(
                    'posts:profile',
                    args=[self.user]
                )
            ),
            (
                'posts/post_detail.html', reverse(
                    'posts:post_detail', kwargs={'post_id': self.post.pk}
                )
            ),
            (
                'posts/create_post.html', reverse(
                    'posts:create_post'
                )
            ),
            (
                'posts/create_post.html',
                reverse(
                    'posts:post_edit',
                    kwargs={'post_id': self.post.pk})
            )

        ]

        for template, reverse_name in templates_pages_names:
            with self.subTest(reverse_name=reverse_name):
                response = self.authorized_client.get(reverse_name)
                self.assertTemplateUsed(response, template)

    def test_pages_show_correct_context(self):
        """Шаблон index сформирован с правильным контекстом."""

        for post in Post.objects.select_related("group"):
            response = self.authorized_client.get(reverse('posts:index'))
            self.assertEqual(response.context.get('post'), post)

    def test_group_list_pages_show_correct_context(self):
        """Шаблон group_list сформирован с правильным контекстом."""
        response = self.authorized_client.get(reverse(
            'posts:group_list', kwargs={'slug': self.group.slug})
        )

        group = get_object_or_404(Group, slug=self.group.slug)
        first_objects = group.posts.all()
        for post in first_objects:
            self.assertEqual(response.context.get('post'), post)

        second_object = response.context.get('page_obj').object_list
        for post in second_object:
            self.assertEqual(response.context.get('post'), post)

    def test_profile_pages_show_correct_context(self):
        """Шаблон profile сформирован с правильным контекстом."""
        response = self.authorized_client.get(reverse(
            'posts:profile',
            args=[self.user])
        )

        author = self.user
        first_objects = author.posts.all()
        for post in first_objects:
            self.assertEqual(response.context.get('post'), post)

        second_object = response.context.get('page_obj').object_list
        for post in second_object:
            self.assertEqual(response.context.get('post'), post)

    def test_post_detail_pages_show_correct_context(self):
        """Шаблон post_detail сформирован с правильным контекстом."""
        response = self.authorized_client.get(reverse(
            'posts:post_detail',
            kwargs={'post_id': self.post.pk})
        )
        self.assertEqual(response.context.get('post'), self.post)

    def test_post_edit_show_correct_context(self):
        """Шаблон post_edit сформирован с правильным контекстом."""
        response = (self.authorized_client.get(reverse(
            'posts:post_edit', kwargs={'post_id': self.post.pk}))
        )

        form_field = response.context.get('form').fields.get('text')
        self.assertIsInstance(form_field, forms.fields.CharField)

        form_field = response.context.get('form').fields.get('group')
        self.assertIsInstance(form_field, forms.fields.ChoiceField)

        form = response.context.get('form')
        self.assertIsInstance(form, PostForm)

    def test_create_post_show_correct_context(self):
        """Шаблон post_create сформирован с правильным контекстом."""
        response = self.authorized_client.get(reverse(
            'posts:create_post')
        )
        form_fields = {
            'text': forms.fields.CharField,
            'group': forms.fields.ChoiceField,
        }
        for value, expected in form_fields.items():
            with self.subTest(value=value):
                form_field = response.context.get('form').fields.get(value)
                self.assertIsInstance(form_field, expected)

    def test_show_correct_img_context(self):

        """Шаблоны сформированы с правильным контекстом

         проверка на наличие картинки"""
        url_names = [
            reverse('posts:index'),
            reverse('posts:group_list', args=['test-slug']),
            reverse('posts:profile', args=[self.user]),
            reverse('posts:post_detail', args=[self.post.pk]),
        ]

        for url in url_names:
            with self.subTest(value=url):
                response = self.authorized_client.get(url)
                self.assertEqual(
                    response.context.get('post').image,
                    self.post.image
                )

    def test_add_comment_correct_context(self):
        """Проверка add_comment

        комментарий появляется на странице поста
        комментировать посты может только авторизованный пользователь
        """
        url = reverse('posts:add_comment', args=[self.post.pk])
        response = self.authorized_client.get(url,)

        if self.post.author == self.authorized_client:
            self.assertEqual(response.status_code, HTTPStatus.OK)

        elif self.user == self.authorized_client:
            self.assertRedirects(response, url)

        else:
            response = self.guest_client.get(url)
            self.assertRedirects(
                response, reverse(
                    'users:login'
                ) + "?next=" + reverse(
                    'posts:add_comment', args=[self.post.pk])
            )

    def test_post_right_group_exists(self):
        """Проверка Создания Поста ,

        что если при создании поста указать группу то этот пост появляется."""

        response = self.authorized_client.get(
            reverse('posts:index')
        )
        object = self.group.posts.filter(
            group=response.context.get('post').group
        )
        self.assertTrue(object.exists())

        response = self.authorized_client.get(reverse(
            'posts:group_list', kwargs={'slug': self.group.slug})
        )

        object = self.group.posts.filter(
            group=response.context.get('post').group
        )
        self.assertTrue(object.exists())

    def test_post_right_group(self):
        response = self.authorized_client.get(
            reverse('posts:index')
        )

        for post in Post.objects.select_related("group"):
            self.assertEqual(response.context.get('post').group, post.group)

        response = self.authorized_client.get(reverse(
            'posts:group_list', kwargs={'slug': self.group.slug})
        )

        object = self.group.posts.all()
        for post in object:
            self.assertEqual(response.context.get('post').group, post.group)

    def test_follow(self):
        """Проверка авторизованный пользователь может

        подписываться на других пользователей """

        self.authorized_client.get(f'/profile/{self.user}/follow/')
        response = self.authorized_client.get('/follow/')

        for post in self.post_list:
            self.assertEqual(response.context.get('post'), post)

    def test_unfollow(self):
        """Проверка авторизованный пользователь может

        удалять других пользователей из подписок """

        self.authorized_client.get(f'/profile/{self.user}/unfollow/')
        response = self.authorized_client.get('/follow/')

        for post in self.post_list:
            self.assertEqual(response.context.get('post'), post)

    def test_check_correct_followed(self):
        """Проверка Ленты постов авторов

        Новая запись пользователя появляется в ленте
        тех, кто на него подписан"""

        response = self.authorized_client.get('/follow/')

        for post in self.post_list:
            self.assertEqual(response.context.get('post'), post)

    def test_check_correct_unfollowed(self):
        """Проверка Ленты постов авторов

        В ленте подписок нет лишних постов"""

        response = self.authorized_client2.get('/follow/')

        self.assertEqual(response.context.get('post'), None)

class PaginatorViewsTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        # Создаем пользователя
        # Создаем неавторизованный клиент
        cls.guest_client = Client()
        # Создаем пользователя
        cls.user = User.objects.create_user(username='HasNoName')
        # Создаем второй клиент
        cls.authorized_client = Client()
        # Авторизуем пользователя
        cls.authorized_client.force_login(cls.user)

        cls.group = Group.objects.create(
            title='Тестовая Группа',
            slug='test-slug',
            description='тестовое описание группы'
        )

        cls.post = Post.objects.bulk_create(
            [
                Post(
                    text=f'Тестовый текст{i}',
                    author=cls.user,
                    group=cls.group
                )
                for i in range(0, 13)
            ]
        )

    def test_first_page_contains(self):
        """Тест Пагинатора для Первой странцы"""
        PAGE_LIMIT = 10
        url_names = {
            reverse('posts:index'): PAGE_LIMIT,
            reverse(
                'posts:group_list',
                kwargs={'slug': self.group.slug}
            ): PAGE_LIMIT,
            reverse(
                'posts:profile',
                args=[self.user]
            ): PAGE_LIMIT,
        }

        for value, expected in url_names.items():
            with self.subTest(value=value):
                response = self.client.get(value + '?page=1')
                self.assertEqual(len(response.context['page_obj']), expected)

    def test_second_page_contains_three_records(self):
        """Тест Пагинатора для Второй странцы"""
        PAGE_LIMIT_SECOND_PAGE = 3

        url_names = {
            reverse(
                'posts:index'
            ): PAGE_LIMIT_SECOND_PAGE,

            reverse(
                'posts:group_list',
                kwargs={'slug': self.group.slug}
            ): PAGE_LIMIT_SECOND_PAGE,

            reverse(
                'posts:profile',
                args=[self.user]
            ): PAGE_LIMIT_SECOND_PAGE,
        }

        for value, expected in url_names.items():
            with self.subTest(value=value):
                response = self.client.get(value + '?page=2')
                self.assertEqual(len(response.context['page_obj']), expected)


# from django.contrib.auth.models import User
from django.db import models

from rest_framework.serializers import (
        HyperlinkedIdentityField,
        ModelSerializer,
        SerializerMethodField,
        )

from directories.models import  PhoneNumber, Department, Faculty
# from users.api.serializers import UserDetailSerializer

#Creating an ApplyMedical
class PhoneNumberCreateUpdateSerializer(ModelSerializer):
    department  = models.ForeignKey(Department, default=1, on_delete = models.CASCADE)
    class Meta:
        model = PhoneNumber
        fields = [
            'department',
            'phone',
        ]

apply_detail_url = HyperlinkedIdentityField(
        view_name='directories-api:detail',
        lookup_field='pk'#or primary key <pk>
    )

class FacultyDetailSerializer(ModelSerializer):

    class Meta:
        model = Faculty
        fields = [
            'id',
            'shortname',
            'fullname',
            'dean',
            'location',
            'lat',
            'lng',
            'timestamp',
            'updated',
            ]

class DepartmentDetailSerializer(ModelSerializer):
    faculty = FacultyDetailSerializer(read_only=True)
    class Meta:
        model = Department
        fields = [

            'id',
            'shortname',
            'fullname',
            'location',
            'chairman',
            'email',
            'lat',
            'lng',
            'timestamp',
            'updated',
            'faculty',
            ]

class PhoneNumberDetailSerializer(ModelSerializer):
    url        = apply_detail_url
    department = DepartmentDetailSerializer(read_only=True)

    class Meta:
        model = PhoneNumber
        fields = [
            'url',
            'id',
            'phone',
            'timestamp',
            'updated',
            'department',
        ]

class PhoneNumberListSerializer(ModelSerializer):
    url = apply_detail_url
    department = DepartmentDetailSerializer(read_only=True)
    delete_url = HyperlinkedIdentityField(
        view_name='directories-api:delete',
        lookup_field='pk'#or primary key <pk>
    )
    class Meta:
        model = PhoneNumber
        fields = [
            'url',
            'id',
            'phone',
            'timestamp',
            'updated',
            'delete_url',
            'department',
        ]


# Python script to convert between data formats.
# It takes as input the json outputs of Alphapose and converts them to the appropriate dictionary form for
# the SLT methods. This script assumes that AlphaPose extraction has already been performed.
# Developed in the context of the aiD project
import gzip
import pickle
import torch
import json

def save(Data,name):

  with gzip.open(name, 'wb') as handle:
    pickle.dump(Data, handle, protocol=pickle.HIGHEST_PROTOCOL)

directory = 'data/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/test/'
text = 'data/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/annotations/manual/PHOENIX-2014-T.test.corpus.csv'
count = 0
all_data = []
with open(text, 'r') as csvfile:
    for line in csvfile:
        if count == 0:
            count +=1
            continue
        #print(line.split('|'))
        dir, _, _, _, signer,gloss, text = line.split('|')
        outputpath = directory + dir + '/'

        with open(outputpath + 'AlphaPose_out.json') as f:
            data = json.load(f)

        data_tensor = torch.Tensor([ x['keypoints'] for x in data[3:]])

        dict_data = {
            'name': dir,
            'signer': signer,
            'gloss': gloss.strip(),
            'text': text.strip(),
            'sign': data_tensor
            }

        all_data.append(dict_data)
        count += 1

        if count % 200 == 0:
            print(count)

save(all_data, 'alphapose_phoenix.test')


import json

import pytest

from src.main import app

from fastapi.testclient import TestClient

headers = {"Accept": "application/json"}
client = TestClient(app, headers=headers)

def test_get_echo_reply():
    response = client.get('/')
    assert response.status_code == 200
    assert response.json() == {"message": "Hello, stranger"}

@pytest.mark.parametrize("test_input", [
    ({"name": "Vasiliy"}),
    ({"surname": "Petrov"}),
    ({})
])
def test_post_echo_reply(test_input):
    response = client.post('/',  json=test_input)
    assert response.status_code == 200
    assert json.loads(response.json()) == test_input

def test_get_request_parser():
    response = client.get('/request')
    assert response.status_code == 200
    assert response.json() == {
        "headers": dict(response.request.headers),
        "cookies": dict(client.cookies),
        "method": response.request.method,
        "body": {}
    }

@pytest.mark.parametrize("test_input", [
    ({"name": "Vasya", "surname": "Vasiliev"}),
    ({"color": "black", "something": 5}),
    ({})
])
def test_post_request_parser(test_input):
    response = client.post('/request', json=test_input)
    assert response.status_code == 200
    assert response.json() == {
        "headers": dict(response.request.headers),
        "cookies": dict(client.cookies),
        "method": response.request.method,
        "body": test_input
    }


from sklearn.metrics import accuracy_score, roc_auc_score, f1_score, precision_score, recall_score

from common import utils
import numpy as np
from discrimination.discrimination_utils import load_data_demencia, results_to_csv, roc_auc_score_multiclass
from discrimination.svm_utils import train_svm

config = utils.load_config('config/config_demencia16k-225B.yml')  # loading configuration
x_train, y_train = load_data_demencia(config=config)  # loading data
# out_results =

# train
# print("Checkpoint utilized:", config['pretrained_model_details']['checkpoint_path'])
list_c = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 0.1]
for c in list_c:
    array_preds, array_trues, array_probs = train_svm(svm_type='linear-loocv', C=c, X=x_train, y=np.ravel(y_train))

    acc = accuracy_score(array_trues, array_preds)
    auc = roc_auc_score(array_trues, array_probs, multi_class='ovo', labels=np.unique(y_train))
    # auc = roc_auc_score(array_trues, array_probs[:, 1])
    aucs = roc_auc_score_multiclass(actual_class=array_trues, pred_class=array_preds)

    array_preds[array_preds == 3] = 2
    array_trues[array_trues == 3] = 2
    f1 = f1_score(array_trues, array_preds)
    prec = precision_score(array_trues, array_preds)
    rec = recall_score(array_trues, array_preds)
    # results_to_csv(file_name=out_results,
    #                list_values=,
    #                list_columns=)
    print("with", c, "acc:", acc, " f1:", f1, " prec:", prec, " recall:", rec, 'AUC:', auc, 'auc-c0:', aucs[1],
          'auc-c1:', aucs[2])


from django.shortcuts import render, redirect, get_object_or_404
from django.urls import reverse
from notifications.models import Notification
from blog.models import BlogDetailPage
from visitor_record.utils import count_visits

def my_notifications(request):
    obj = BlogDetailPage.objects.all()[0]
    data = count_visits(request, obj)

    context = {}
    context['client_ip'] = data['client_ip']
    context['location'] = data['location']
    context['total_hits'] = data['total_hits']
    context['total_visitors'] =data['total_vistors']
    context['cookie'] = data['cookie']
    response = render(request, 'my_notifications/my_notifications.html', context)
    response.set_cookie(context['cookie'], 'true', max_age=300)
    return response

def my_notification(request, my_notification_pk):
    my_notification = get_object_or_404(Notification, pk=my_notification_pk)
    my_notification.unread = False
    my_notification.save()
    return redirect(my_notification.data['url'])

def delete_my_read_notifications(request):
    notifications = request.user.notifications.read()
    notifications.delete()
    return redirect(reverse('my_notifications'))


from model import GPTLanguageModel
import wandb
import time
import json
import torch

mode = 'big'

if mode == 'test':
    # TEST hparam config
    batch_size = 32 # how many independent sequences will we process in parallel?
    block_size = 8 # what is the maximum context length for predictions?
    max_iters = 5000
    eval_interval = 500
    learning_rate = 1e-3
    eval_iters = 200
    n_emb_d = 32 # number of embedding dimensions
    n_heads = 6
    n_layers = 6
    dropout = 0.2
elif mode == 'big':
    # BIG hparam config
    batch_size = 64 # how many independent sequences will we process in parallel?
    block_size = 256 # what is the maximum context length for predictions?
    max_iters = 5000
    eval_interval = 250
    learning_rate = 3e-4 # reduced as network is much bigge
    eval_iters = 200
    n_emb_d = 384 # number of embedding dimensions
    n_heads = 6
    n_layers = 6
    dropout = 0.2

device = 'mps' if torch.backends.mps.is_available() else 'cpu'
dataset_path = '../data/ansturm.txt'

torch.manual_seed(42)

# load the text
with open(dataset_path, 'r', encoding='utf-8') as f:
    text = f.read()

    punctuation = {
        u'\u2003': " ",
    }

    for key, value in punctuation.items():
        text = text.replace(key, value)

# all the unique characters that occur in the text
chars = sorted(list(set(text)))
vocab_size = len(chars)

print(chars)

# create a mapping from characters to integers
stoi = { ch:i for i,ch in enumerate(chars) }
itos = { i:ch for i,ch in enumerate(chars) }
encode = lambda s: [stoi[c] for c in s] # encoder: take a string, output a list of integers
decode = lambda l: ''.join([itos[i] for i in l]) # decoder: take a list of integers, output a string

# Train and test splits
data = torch.tensor(encode(text), dtype=torch.long)
n = int(0.9*len(data)) # first 90% will be train, rest val
train_data = data[:n]
val_data = data[n:]

# data loading
def get_batch(split):
    # generate a small batch of data of inputs x and targets y
    data = train_data if split == 'train' else val_data
    ix = torch.randint(len(data) - block_size, (batch_size,))
    x = torch.stack([data[i:i+block_size] for i in ix])
    y = torch.stack([data[i+1:i+block_size+1] for i in ix])
    x, y = x.to(device), y.to(device)
    return x, y

@torch.no_grad()
def estimate_loss():
    out = {}
    model.eval()
    for split in ['train', 'val']:
        losses = torch.zeros(eval_iters)
        for k in range(eval_iters):
            X, Y = get_batch(split)
            logits, loss = model(X, Y)
            losses[k] = loss.item()
        out[split] = losses.mean()
    model.train()
    return out

model = GPTLanguageModel(vocab_size, n_emb_d, n_heads, n_layers, dropout, block_size, device)
model = model.to(device)

# Print total & trainable parameters
pt_total_params = sum(p.numel() for p in model.parameters())
pt_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f'Total parameters: {pt_total_params}')
print(f'Trainable parameters: {pt_trainable_params}')

# WandB – Initialize a new run
wandb.login()
run = wandb.init(
    # Set the project where this run will be logged
    entity="yvokeller",
    project="frdm-GPT",
    # Track hyperparameters and run metadata
    config={
        "learning_rate": learning_rate,
        "batch_size": batch_size,
        "max_iters": max_iters,
        "eval_interval": eval_interval,
        "n_emb_d": n_emb_d,
        "n_heads": n_heads,
        "n_layers": n_layers,
        "dropout": dropout,
    })

# create a PyTorch optimizer
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)

for iter in range(max_iters):
    # every once in a while evaluate the loss on train and val sets
    if iter % eval_interval == 0 or iter == max_iters - 1:
        losses = estimate_loss()
        print(f"step {iter}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")
        wandb.log({"train_loss": losses['train'], "val_loss": losses['val']})

    # sample a batch of data
    xb, yb = get_batch('train')

    # evaluate the loss
    logits, loss = model(xb, yb)
    optimizer.zero_grad(set_to_none=True)
    loss.backward()
    optimizer.step()

# save the model & config.json to disk with timestampf format YYYY-MM-DD_HH-MM-SS
torch.save(model.state_dict(), f'../models/gpt.pt')

with open("../models/config.json", "w", encoding='utf-8') as f:
    f.write(json.dumps({
            'hparams': model.config,
            'chars': chars,
        }
    ))

# generate from the model
context = torch.zeros((1, 1), dtype=torch.long, device=device)
print(decode(model.generate(context, max_new_tokens=500)[0].tolist()))


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 20 12:06:22 2017

@author: Work
"""

import nltk
from nltk.book import *
from __future__ import print_function
from __future__ import division

def unusual_words(text):
    text_vocab = set(w.lower() for w in text if w.isalpha())
    english_vocab = set(w.lower() for w in nltk.corpus.words.words())
    unusual = text_vocab - english_vocab
    return sorted(unusual)

unusual_words(nltk.corpus.gutenberg.words('austen-sense.txt'))

unusual_words(nltk.corpus.nps_chat.words())

from nltk.corpus import stopwords
stopwords.words('english')
len(stopwords.words('english'))

def content_fraction(text):
    stopwords = nltk.corpus.stopwords.words('english')
    content = [w for w in text if w.lower() not in stopwords]
    #return content
    return len(content) / len(text)

content_fraction(nltk.corpus.reuters.words())

len(nltk.corpus.brown.words())
content_fraction(nltk.corpus.brown.words())

puzzle_letters = nltk.FreqDist('egivrvonl')
obligatory = 'r'
wordlist = nltk.corpus.words.words()
[w for w in wordlist if len(w) >= 6
                         and obligatory in w
                         and nltk.FreqDist(w) <= puzzle_letters]

names = nltk.corpus.names
names.fileids()
male_names = names.words('male.txt')
female_names = names.words('female.txt')
[w for w in male_names if w in female_names]

cfd = nltk.ConditionalFreqDist(
        (fileid, name[-1])
        for fileid in names.fileids()
        for name in names.words(fileid))
cfd.plot()

entries = nltk.corpus.cmudict.entries()
len(entries)

for entry in entries[42371:42379]:
    print(entry)

syllable = ['N', 'IHO', 'K', 'S']
[word for word, pron in entries if pron[-4:] == syllable]

[w for w, pron in entries if pron[-1] == 'M' and w[-1] == 'n']

from nltk.corpus import swadesh
swadesh.fileids()
swadesh.words('en')

fr2en = swadesh.entries(['fr', 'en'])
fr2en
translate = dict(fr2en)
translate['chien']
translate['jeter']

de2en = swadesh.entries(['de', 'en'])
es2en = swadesh.entries(['es', 'en'])
translate.update(dict(de2en))
translate.update(dict(es2en))
translate['Hund']
translate['perro']

languages = ['en', 'de', 'nl', 'es', 'fr', 'pt', 'la']
for i in [139, 140, 141, 142]:
    print(swadesh.entries(languages)[i])

from nltk.corpus import toolbox
toolbox.entries('rotokas.dic')

from nltk.corpus import wordnet as wn
wn.synsets('motorcar')
wn.synset('car.n.01').lemma_names()

wn.synset('car.n.01').definition()
wn.synset('car.n.01').examples()

wn.synset('car.n.01').lemmas()
wn.lemma('car.n.01.automobile')
wn.lemma('car.n.01.automobile').synset()
wn.lemma('car.n.01.automobile').name()

wn.synsets('car')
for synset in wn.synsets('car'):
    print(synset.lemma_names())
wn.lemmas('car')

motorcar = wn.synset('car.n.01')
types_of_motorcar = motorcar.hyponyms()
types_of_motorcar[0]
sorted(lemma.name() for synset in types_of_motorcar for lemma in synset.lemmas())

motorcar.hypernyms()
paths = motorcar.hypernym_paths()
len(paths)
[synset.name() for synset in paths[0]]
[synset.name() for synset in paths[1]]