import acme
import certbot
import certbot.account
import certbot.constants
import certbot.main
import certbot.plugins
import certbot.plugins.dns_common
import certbot.interfaces
import errno
import dns
import dns.name
import grp
import hashlib
import logging
import namedlist
import os
import OpenSSL
import pwd
import ipalib
import shutil
import stat
import sys
import zope
import zope.component
import zope.interface

__version__ = '0.0.6'

# Set up logging
logger = logging.getLogger(__name__)

class PermError(Exception):
    """ Raised if permissions don't match specifications/requirments or unsafe permissions are found """
    pass

class IntegrityError(Exception):
    """ Raised if the internal integrity and validity of the application is subject"""
    pass

def check_permission(perm_mode, flags=stat.S_IWOTH):
    """
    Check if a bit is is set in an integer, very useful for checking if
    a particular permission is set of a file by comparing os.stat.st.mode

    Multiple modes can be combined by using by using the bitwise OR operator
     e.g.
    check_permission(0o754, stat.S_IROTH | stat.S_IWGRP)
    -> True

    Valid modes from stat:

        S_ISUID = 04000
        S_ISGID = 02000
        S_ENFMT = S_ISGID
        S_ISVTX = 01000
        S_IREAD = 00400
        S_IWRITE = 00200
        S_IEXEC = 00100
        S_IRWXU = 00700
        S_IRUSR = 00400
        S_IWUSR = 00200
        S_IXUSR = 00100
        S_IRWXG = 00070
        S_IRGRP = 00040
        S_IWGRP = 00020
        S_IXGRP = 00010
        S_IRWXO = 00007
        S_IROTH = 00004
        S_IWOTH = 00002
        S_IXOTH = 00001
    """
    return bool(perm_mode & flags)

def check_dir_perms(path, dir_perm=stat.S_IWOTH, file_perm=stat.S_IWOTH, users=('root',), groups=('root',), recurse=True):
    """
    Check dir structure and verify only specified users/groups have access

    If any directories have the dir_perm bits set we'll raise an error and the
    same goes for files matching file_perm bits.

    See check_permission for more info on how permission bit checking works.
    """
    directories = ((path, (), ()),) if not recurse else os.walk(path)
    for dir_name, sub_dirs, files in directories:
        attrib = os.stat(dir_name)
        if attrib.st_uid not in [pwd.getpwnam(user).pw_uid for user in users]:
            err_msg = 'Directory: "{0}" is owned by {1} which is not in the list of allowed users: "{2!s}"'
            raise PermError(err_msg.format(dir_name, pwd.getpwuid(attrib.st_uid).pw_name, users))

        if attrib.st_gid not in [grp.getgrnam(group).gr_gid for group in groups]:
            err_msg = 'The group for directory: "{0}" is {1} which is not in the list of allowed groups: "{2!s}"'
            raise PermError(err_msg.format(dir_name, grp.getgrgid(attrib.st_gid).gr_name, groups))

        if check_permission(attrib.st_mode, dir_perm):
            # Could add strmode for python one day and make nice human errors
            err_msg = 'The permissions on directory: "{0}" are "{1!s}" and violate restriction "{2!s}"'
            raise PermError(err_msg.format(dir_name, oct(attrib.st_mode), oct(dir_perm)))

        for f in files:
            file_attrib = os.stat(os.path.join(dir_name, f))
            if check_permission(file_attrib.st_mode, file_perm):
                # Could add strmode for python one day and make nice human errors
                err_msg = 'The permissions on file: "{0}" are "{1!s}" and violate restriction "{2!s}"'
                raise PermError(err_msg.format(os.path.join(dir_name, f), oct(file_attrib.st_mode), oct(file_perm)))

def match_owner_group(dest_path, source_path):
    """ Matches owner/group from one filesystem object to another """
    source_stat = os.stat(source_path)
    return os.chown(dest_path, source_stat[stat.ST_UID], source_stat[stat.ST_GID])

def mkdirp(path, inherit_owner_group=False, permission_mode=None, strict=False):
    """ Recursive mkdir (mkdir -p) """
    try:
        if permission_mode:
            os.makedirs(path, permission_mode)
        else:
            os.makedirs(path)
        logger.debug('Created directory: {0}'.format(path))
        if inherit_owner_group is True:
            match_owner_group(path, os.path.abspath(os.path.join(path, '../')))
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            if permission_mode and strict:
                try:
                    check_dir_perms(path,
                                    dir_perm=permission_mode,
                                    recurse=False)
                except PermError:
                    logger.debug('Provided path {0} exists and has incorrect permissions'.format(path))
                    raise
            pass
        else:
            raise

class CertMongerAction(object):
    """ Represents an action requested by Certmonger """
    # Supported Exit Codes
    EXIT_ISSUED = 0
    EXIT_WAIT = 1
    EXIT_REJECTED = 2
    EXIT_UNREACHABLE = 3
    EXIT_UNCONFIGURED = 4
    EXIT_WAIT_WITH_DELAY = 5
    EXIT_OPERATION_NOT_SUPPORTED = 6
    EXIT_NEW_KEYPAIR_REQUIRED = 17

    PATHS = {'config_dir': '/etc/certmonger/letsencrypt',
             'work_dir': '/var/lib/certmonger/letsencrypt',
             'log_dir': '/var/log/letsencrypt'}

    defaults = certbot.constants.CLI_DEFAULTS

    def __init__(self,
                 paths=PATHS,
                 email=None,
                 key_size=4096,
                 verify_ssl=True,
                 user_agent='{0}/{1}'.format(__name__, __version__),
                 staging=True,
                 verbosity=logging.INFO,
                 propagation_seconds=10):

        formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s')
        logger.setLevel(verbosity)

        # Set up logging to use syslog
        syslog_handler = logging.handlers.SysLogHandler(address='/dev/log')
        syslog_handler.setFormatter(formatter)
        syslog_handler.setLevel(verbosity)
        logger.addHandler(syslog_handler)
        # Create console logger for stdout/stderr if we have a tty, full debug
        if sys.stdout.isatty():
            console_handler = logging.StreamHandler()
            console_handler.setLevel(logging.DEBUG)
            console_handler.setFormatter(formatter)
            logger.addHandler(console_handler)

        # Load and store relevant environment variables
        self.environment = self.load_environment_variables()

        # If no email is specified we try to get one from the request
        if not email:
            try:
                email = self.environment['CERTMONGER_REQ_EMAIL']
            except KeyError:
                email = None
                self.defaults['register_unsafely_without_email'] = True

        self.defaults['email'] = email

        # Set timeout for DNS propagation
        #settattr(self.defaults, 'cerlet:ipa_propagation_seconds', propagation_seconds)
        self.defaults['propagation_seconds'] = propagation_seconds

        # Set key size
        self.defaults['key_size'] = key_size

        # Configure SSL/TLS Server cert verification
        self.defaults['verify_ssl'] = verify_ssl

        # Set prod/staging server selector
        if staging:
            self.defaults['staging'] = staging

        # Find all plugins and set
        self.defaults['plugins'] = certbot.plugins.disco.PluginsRegistry.find_all()

        # Set User Agent
        self.defaults['user_agent'] = user_agent

        # Override account directory
        self.defaults['accounts_dir'] = os.path.join(paths['config_dir'],
                                                     certbot.constants.ACCOUNTS_DIR,
                                                     self.environment['CERTMONGER_REQ_HOSTNAME'])

        # Create any directories which don't exist with correct
        # permisisons/owner/group and set in config
        for key, path in paths.iteritems():
            mkdirp(path, permission_mode=0o700)
            self.defaults[key] = path

        # Create config object from defaults in certbot and assign defaults
        Config = namedlist.namedlist('Config', ' '.join(self.defaults.keys()))
        self.namespace = Config(**self.defaults)
        self.config = certbot.configuration.NamespaceConfig(namespace=self.namespace)
        zope.component.provideUtility(self.config)

        # Configure displayer depending on if we have a tty or not
        if sys.stdout.isatty():
            self.displayer = certbot.display.util.NoninteractiveDisplay(sys.stdout)
        else:
            self.displayer = certbot.display.util.NoninteractiveDisplay(open(os.devnull, "w"))

        zope.component.provideUtility(self.displayer)

        # Set up Certbot Account Storage, at some point this should be moved
        # and stored by IPA in LDAP. For now we create separate accounts per
        # host and store details on the filesystem.
        account_storage = certbot.account.AccountFileStorage(self.config)
        accounts = account_storage.find_all()

        # Assume there will only be one account per host and that it will never
        # need to be updated with a new email address ...
        if len(accounts) == 0:
            # Register account with Let's Encrypt Server if needed, we always agree
            # to the TOS terms (see lambda).
            account, acme = certbot.client.register(config=self.config,
                                                    account_storage=account_storage,
                                                    tos_cb=lambda *_, **__: True)

            account_storage.save_regr(account, acme)
        else:
            acme = None
            account = accounts[0]

        # Instantiate authenticator using DNS/FreeIPA
        self.plugin = self.defaults['plugins']['cerlet:ipa'].init(config=self.config)
        # Instantiate the installer using Certmonger
        #self.installer = certbot.plugins.disco.PluginsRegistry.find_all()['cerlet:ipa']

        # Instantiate client
        self.client = certbot.client.Client(config=self.config,
                                            account_=account,
                                            auth=self.plugin,
                                            installer=self.plugin,
                                            acme=acme)

    @staticmethod
    def _raise_not_implemented(exception=NotImplementedError):
        """ Raises the provided exception """
        raise exception

    @staticmethod
    def load_environment_variables(pattern='CERTMONGER'):
        """ Loads environment variables matching a pattern """
        matches = {}
        for key, value in os.environ.items():
            if pattern in key:
                matches[key] = value

        return matches

    @classmethod
    def operation_factory(cls, operation):
        """
        Returns the appopriate method for an operation as they are
        specified by CertMonger in environment variables.
        """
        logger.debug('Certmonger operation detected, evaluating environment variables and performing actions as requested')
        valid_operations = {'SUBMIT': cls.submit,
                            'POLL': cls.poll,
                            'IDENTIFY': cls.identify,
                            'GET-NEW-REQUEST-REQUIREMENTS': cls.requirements,
                            'GET-RENEW-REQUEST-REQUIREMENTS': cls.renew_requirements,
                            'GET-SUPPORTED-TEMPLATES': cls.templates,
                            'GET-DEFAULT-TEMPLATE': cls.default_template,
                            'FETCH-SCEP-CA-CAPS': cls._raise_not_implemented,
                            'FETCH-SCEP-CA-CERTS': cls._raise_not_implemented,
                            'FETCH-ROOTS': cls.get_ca_root_certs}
        try:

            return valid_operations[operation](cls())
        except NotImplementedError:
            logger.warning('Operation {0} is not supported in this version of {1} ({2})'.format(operation, __name__, __version__))
            return self.EXIT_OPERATION_NOT_SUPPORTED
        except KeyError:
            logger.error('Unknown Operation: {0}'.format(operation))
            return self.EXIT_OPERATION_NOT_SUPPORTED

    def register(self, config=None):
        certbot.main.register(config or self.config, unused_plugins=None)

    def submit(self, csr=None, domains=None, ca_profile=None, ca_nickname=None, ca_issuer=None):
        """
        Accepts a single Certificate Signing Request, PKCS#10/PEM encoded

        First time enrollment requested by Certmonger

        Keyword arguments:

        csr -- Enrollment request in PEM form. PKCS#10 format, PEM encoded
        ca_profile -- Name of enrollment profile/template/certtype
        ca_nickname --  Name by which the CA is known, and would have been specified to the -c
            option to the "getcert" command. For example "prod" or "test"
        ca_issuer --  Requested issuer for enrollment

        Issues certificates will be returned in PEM encoded x.509 format
        and an exit status of 0 will be returned.

        If the certificates can't be issued immediately a "cookie" value/token
        will be returned and an exit status of 1 returned. The requestor is
        free to try again at their convenience presenting the cookie/token.

        If the request is rejected an error will returned along with an exit
        status of 2.

        If there was a connection or networking exception the error will be
        returned along with an exit status of 3.

        If additional data is required the specific error will be returned
        along with an exit status of 4.

        * If the client should wait for a specific period of time (for example, if
            the CA has told it when to try again), output a delay size in seconds, a
            newline, and a "cookie" value, and exit with status 5.  The daemon will try
            again after the specified amount of time has passed.

        * If the CA indicates that the client needs to try again using a different
            key pair in the signing request (for example, if its policy limits the
            number of times a particular key pair can be enrolled, or the length of
            time one can be in service), exit with status 17.  The daemon will generate
            a new key pair and try again.

        * If the helper does not understand what is being asked of it, exit with
            status 6.  You should never return this value for "SUBMIT" or "POLL", but
            it is mentioned here so that we can refer to this list later.
        """
        logger.debug('Submitting certificate signing request')
        CSR = namedlist.namedlist('CSR', 'data form')
        csr = CSR(data = csr or self.environment['CERTMONGER_CSR'], form='pem')
        domains = domains or [self.environment['CERTMONGER_REQ_SUBJECT']]

        try:
            cert, chain = self.client.obtain_certificate_from_csr(domains, csr)
            sys.stdout.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert.body))
            return self.EXIT_ISSUED

        except acme.messages.Error as e:
            logger.error(e)
            sys.stdout.write(str(e))
            return self.EXIT_REJECTED

        except acme.error.ClientError as e:
            logger.error(e)
            sys.stdout.write(str(e))
            return self.EXIT_UNREACHABLE

        except Exception as e: # Catch all for exceptions
            logger.exception(e)
            sys.stdout.write(str(e))
            return self.EXIT_REJECTED

    def poll(self, cookie=None):
        """
        Poll status of previously submitted request

        Keyword arguments:

        cookie -- A cookie/token to identify a previously submitted request

        If the submit method previously returned with status 1 or 5, this can
        be called to retry.

        Returned certificates and exit statuses are the same as the submit
        method.
        """
        logger.debug('Polling for certificate signing request status')
        cookie = cookie or self.environment['CERTMONGER_CA_COOKIE']

    @staticmethod
    def identify():
        """
        Outputs the version of the helper and returns an exit status of 0
        """
        logger.debug('Returning version of helper/plugin')
        return __version__

    def requirements(self, renew=False):
        """
        Returns a list of required arguments/environment variables for the
        poll and submit methods, if renew is true it lists those required when
        renewing a certificate rather than requesting a new one.
        """
        submit_requirements = 'CERTMONGER_CSR,CERTMONGER_REQ_SUBJECT,CERTMONGER_OPERATION,CERTMONGER_REQ_HOSTNAME,CERTMONGER_REQ_EMAIL'
        logger.debug('Returning list of required attributes/arguments:')
        logger.debug(submit_requirements)
        sys.stdout.write(submit_requirements)

    def renew_requirements(self):
        return self.requirements(renew=True)

    def templates(default_only=False):
        """
        Returns list of all profiles/templates/cert-types, by default all
        supported templates are returned.
        """
        logger.debug('Returning templates/profiles')

    def default_template(self):
        return self.templates(default_only=True)

    def get_ca_root_certs(self):
        """
        Return a dictionary of nickname/cert with the cert in PEM format.
        """
        logger.debug('Dumping CA Root certificates')
        cert_path = os.path.join(os.path.split(__file__)[0], 'certs')
        root_certs = {'dstrootx3.pem': b'06:87:26:03:31:A7:24:03:D9:09:F1:05:E6:9B:CF:0D:32:E1:BD:24:93:FF:C6:D9:20:6D:11:BC:D6:77:07:39',
                      'isrgrootx1.pem': '96:BC:EC:06:26:49:76:F3:74:60:77:9A:CF:28:C5:A7:CF:E8:A3:C0:AA:E1:1A:8F:FC:EE:05:C0:BD:DF:08:C6',
                      'letsencryptauthorityx3.pem': b'73:1D:3D:9C:FA:A0:61:48:7A:1D:71:44:5A:42:F6:7D:F0:AF:CA:2A:6C:2D:2F:98:FF:7B:3C:E1:12:B1:F5:68',
                      'lets-encrypt-x3-cross-signed.pem': b'25:84:7D:66:8E:B4:F0:4F:DD:40:B1:2B:6B:07:40:C5:67:DA:7D:02:43:08:EB:6C:2C:96:FE:41:D9:DE:21:8D'}

        for filename, sha256sum in root_certs.iteritems():
            cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, open(os.path.join(cert_path, filename)).read())
            if cert.digest('sha256') == sha256sum:  # Verify integrity
                nickname = ' '.join(x[1] for x in cert.get_subject().get_components())
                if cert.get_issuer() == cert.get_subject():  # Root CA's can be identified by always being self signed
                    sys.stdout.write('\n' + nickname + '\n')
                    sys.stdout.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert))
                    sys.stdout.write('\n')
                else:
                    sys.stdout.write(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert))
                    sys.stdout.write('\n')
            else:
                logger.error('Unable to verify the integrity of Root CA {0}'.format(filename))
                logger.error('SHA256 mismatch {0} - {1}'.format(sha256sum, cert.digest('sha256')))
                raise IntegrityError

@zope.interface.implementer(certbot.interfaces.IAuthenticator)
@zope.interface.provider(certbot.interfaces.IPluginFactory)
class Authenticator(certbot.plugins.dns_common.DNSAuthenticator):
    """ FreeIPA / Red Hat Enterprise Linux IdM authentication using DNS challenges """

    description = __doc__.strip().split("\n", 1)[0]

    def __init__(self, *args, **kwargs):
        # Maps provided domains to IPA zones
        self.zone_map = {}
        super(Authenticator, self).__init__(*args, **kwargs)

    def more_info(self):
        return self.description

    def _setup_credentials(self):
        # Set up IPA API connection
        logger.info('Setting up IPA Connection using kerberos credentials')
        try:
            ipalib.api.bootstrap_with_global_options(context=__name__)
            ipalib.api.finalize()
        except ipalib.errors.KerberosError:
            logger.exception('Exception occurred communicating and authenticating to the IPA server, make sure you have valid kerberos credentials')
            raise

        if ipalib.api.env.in_server:
            ipalib.api.Backend.ldap2.connect()
        else:
            ipalib.api.Backend.rpcclient.connect()

    def _perform(self, domain, validation_domain_name, validation):  # pragma: no cover
        logger.debug('Adding DNS Records {0} to zone: {1} with value: {2}'.format(validation_domain_name, domain, validation))
        dns_zone_candidate = dns.name.from_text(domain)
        while True:
            try:
                ipalib.api.Command.dnszone_show(idnsname=unicode(dns_zone_candidate.to_text()))
                record = dns.name.from_text(validation_domain_name).relativize(dns_zone_candidate)
                try:
                    ipalib.api.Command.dnsrecord_add(dnszoneidnsname=unicode(dns_zone_candidate.to_text()), idnsname=unicode(record), txtrecord=validation)
                except ipalib.errors.EmptyModlist:
                    logger.warning('DNS entry already exists: {0} with value: {1}'.format(record, validation))
                self.zone_map[validation_domain_name] = (dns_zone_candidate, record)
                break
            except (ipalib.errors.NotFound, ipalib.errors.ConversionError):
                logger.debug('Unable to find Zone: {0}, checking for parent'.format(dns_zone_candidate.to_text(omit_final_dot=True)))
                try:
                    dns_zone_candidate = dns_zone_candidate.parent()
                except dns.name.NoParent:
                    message = 'Unable to find DNS Zone on IPA server'
                    logger.exception(message)
                    raise ipalib.errors.NotFound(format=None, message=message.decode())

    def _cleanup(self, domain, validation_domain_name, validation):
        logger.debug('Removing DNS entry: {0} to zone: {1} with value: {2}'.format(validation_domain_name, domain, validation))
        ipalib.api.Command.dnsrecord_del(dnszoneidnsname=unicode(self.zone_map[validation_domain_name][0].to_text()), idnsname=unicode(self.zone_map[validation_domain_name][1]), txtrecord=validation)

    @classmethod
    def add_parser_arguments(cls, add, default_propagation_seconds=10):
        super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds)
        add('-h', '--host_name', '--host', dest='ipa_host', help='Hostname or IP Address of the IPA (Satellite) server')
        add('-d', '--ipa_domain_name', '--ipa_domain', dest='ipa_domain', help='Domain in IPA (Satellite) to register under')
        add('-H', '--xml_rpc_url', '--xmlrpc-url', dest='ipa_xml_rpc_url', help='XML RPC service location')
        add('-C', '--ca_path', '--capath', dest='ca_path', help='Path do a directory containing PEM encoded CA files')
        add('-c', '--ca_file', '--cafile', dest='ca_file', help='Path do a file containing PEM encoded CA certificates')
        add('-t', '--keytab_name', '--keytab-name', dest='keytab_file', help='Path to a keytab file containing credentials for IPA server authentication')
        add('-k', '--submitter_principal', '--submitter-principal', dest='submitter_principal', help='Kerberos principal for IPA server authentication')
        add('-K', '--use_ccache_creds', '--use-ccache-creds', dest='use_ccache_creds', default=False, action='store_true', help='Use default ccache for authorization instead of authenticating')
        add('-P', '--request_principal', '--principal-of-request', dest='request_principal', help='Principal(s) (FQDN) used in signing request, comma separated')
        add('-T', '--request_profile', '--profile', dest='request_profile', help='Use a specific profile when requesting enrollment')

@zope.interface.implementer(certbot.interfaces.IInstaller)
@zope.interface.provider(certbot.interfaces.IPluginFactory)
class Installer(certbot.plugins.common.Installer):
    """Certmonger Installer, outputs the cert to stdout"""
    description = __doc__.strip().split("\n", 1)[0]

    # pylint: disable=missing-docstring,no-self-use

    def prepare(self):
        pass  # pragma: no cover

    def more_info(self):
        return self.description

    def get_all_names(self):
        return []

    def deploy_cert(self, domain, cert_path, key_path,
                    chain_path=None, fullchain_path=None):
        with open(cert_path, "r") as f:
            shutil.copyfileobj(f, sys.stdout)

    def enhance(self, domain, enhancement, options=None):
        pass  # pragma: no cover

    def supported_enhancements(self):
        return []

    def save(self, title=None, temporary=False):
        pass  # pragma: no cover

    def rollback_checkpoints(self, rollback=1):
        pass  # pragma: no cover

    def recovery_routine(self):
        pass  # pragma: no cover

    def view_config_changes(self):
        pass  # pragma: no cover

    def config_test(self):
        pass  # pragma: no cover

    def restart(self):
        pass  # pragma: no cover

def main():
    """ Entry point when run directly """
    env = CertMongerAction.load_environment_variables()
    if env:
        sys.exit(CertMongerAction.operation_factory(os.getenv('CERTMONGER_OPERATION')))
    from pkg_resources import load_entry_point
    sys.argv += ['--authenticator', 'cerlet:ipa']
    sys.exit(load_entry_point('certbot', 'console_scripts', 'certbot')())


# -*- coding: utf-8 -*-
from __future__ import absolute_import

import os
import click

from sources.python_weekly import PythonWeekly
from utils.logging import get_logger

logger = get_logger('main.py')

# The hashtable of sources.
source_map = {
    'python_weekly': PythonWeekly,
}

# Registry the group command.
@click.group()
@click.pass_context
def pip_cli(ctx, **kwargs):
    ctx.obj = kwargs

@pip_cli.command(name='run')
@click.option(
    '-s', '--source',
    required=True,
    type=str,
    help='The source of newsletter.',
)
@click.option(
    '-i', '--issue',
    required=False,
    type=int,
    help='The issue number.',
)
@click.option(
    '-t', '--to-issue',
    required=False,
    type=int,
    help='Get from issue 1 up until the given issue.',
)
def run(source: str, issue: int, to_issue):
    """
    Get source with a given issue.
    """

    if source in source_map:
        class_source = source_map[source]

        if not to_issue:
            # Getting 1 issue only.
            class_source(issue_number=issue).run()
        else:
            # Getting the whole issues from 1 to n.
            for i in range(1, to_issue):
                class_source(issue_number=i).run()

    logger.info(f'Getting source: {source} with issue: {issue}')

def main():
    pip_cli()

if __name__ == '__main__':
    pip_cli()


#!/usr/bin/env python3
import re, copy, argparse, json

class Node(dict):
    def __init__(self, *args, **kwargs):
        super(Node, self).__init__(*args, **kwargs)

    def add_child(self, node):
        if not 'children' in self:
            self['children'] = list()
        self['children'].append(node)

    def get_nodes(self, nodes=None):
        if nodes is None:
            nodes = []
        nodes.append(self)
        if 'children' in self:
            for i in self['children']:
                i.get_nodes(nodes)
        return nodes

def from_tsv(name, input_path, output_path):

    root = Node(entry=name, name=name, layer='root')

    with open(input_path, 'rU') as mapping_file:
        levels = mapping_file.readline().strip().split('\t')
        levels.insert(0, 'root')
        nodes_layer = {key: {} for key in levels}
        nodes_layer['root'] = {'root': root}

        for line in mapping_file:
            entries = line.strip().split('\t')
            # prefix empty cells by parent id
            for index, entry in enumerate(entries):
                if entry == ["", "-"]:
                    unique_entry = levels[index + 1]
                    if index == 0:
                        unique_entry = "root_" + unique_entry
                    else:
                        unique_entry = entries[index - 1] + "_" + unique_entry
                    entries[index] = unique_entry

            for index, entry in enumerate(entries):
                layer = levels[index + 1]
                if entry not in nodes_layer[layer]:
                    parent_layer = levels[index]
                    node = Node(entry=entry, name=entry, layer=layer)
                    if index > 0:
                        nodes_layer[parent_layer][entries[index - 1]].add_child(node)
                    else:
                        nodes_layer[parent_layer]['root'].add_child(node)
                    nodes_layer[layer][entry] = node
    with open(output_path, 'w') as output:
        output.write(json.dumps(root, sort_keys=True, indent=4))

if __name__ == '__main__':
    name = 'Enteropathway Functional Hierarchies'
    #/home/omixer/Downloads/FOAM-onto_rel1.tsv
    ipath ='/home/omixer/Downloads/epathway_output.tsv'
    #/home/omixer/Downloads/FOAM-onto_rel1.json
    opath = '/home/omixer/Downloads/enteropathway.json'
    from_tsv(name, ipath, opath)


# two classes (class1, class2)
# only replace with a directory of yours
# finally .npy files saved in your directory with names train.npy, #test.npy, train_labels.npy, test_labels.npy
import cv2
import glob
import numpy as np
#Train data
train = []
train_labels = []

NO_OF_CLASSES = 30
IMG_SIZE = 128

filepaths = ["../../datasets/texture30/*.jpg"]
files = glob.glob("../../datasets/texture30/*.jpg")

for i in range(len(files)):
    j = int(files[i].split('/')[4][1:4])
    print(str(files[i].split('/')[4])," , ",str(j))
    train_label = np.zeros(NO_OF_CLASSES,'int8')
    train_label[j-1] = 1
        #prvchgint(myFile)
    image = cv2.imread(files[i], 0)
    train.append(cv2.resize(image, (IMG_SIZE, IMG_SIZE)))
    train_labels.append(train_label)
    #print(train_label)

train_labels = np.array(train_labels,dtype='int8') #as mnist
train = np.array(train)
print(train.shape)
# convert (number of images x height x width x number of channels) to (number of images x (height * width *3))
# for example (120 * 40 * 40 * 3)-> (120 * 4800)

train = np.reshape(train,[train.shape[0],train.shape[1]*train.shape[2]])
print(train.shape)

# # save numpy array as .npy formats
np.save('tex45Img',train)
np.save('tex45Imglabels',train_labels)

# #Test data
# test = []
# test_labels = []
# files = glob.glob ("/data/test/class1/*.JPG")
# for myFile in files:
#     image = cv2.imread (myFile)
#     test.append (image)
#     test_labels.append([1., 0.]) # class1
# files = glob.glob ("/data/test/class2/*.JPG")
# for myFile in files:
#     image = cv2.imread (myFile)
#     test.append (image)
#     test_labels.append([0., 1.]) # class2

# test = np.array(test,dtype='float32') #as mnist example
# test_labels = np.array(test_labels,dtype='float64') #as mnist
# test = np.reshape(test,[test.shape[0],test.shape[1]*test.shape[2]*test.shape[3]])

# # save numpy array as .npy formats
# np.save('test',test) # saves test.npy
# np.save('test_labels',test_labels)


import json
from watson_developer_cloud import ToneAnalyzerV3
import os

api_key = os.environ.get('WATSON_TONE_ANALYZER')

class ToneAnalyzer:

    def __init__(self, tone_analyzer = ToneAnalyzerV3(
        version='2017-09-21',
        iam_apikey=api_key,
        url= 'https://gateway-lon.watsonplatform.net/tone-analyzer/api/v3/tone?version=2017-09-21&sentences=false'
    )):
        self.tone_analyzer = tone_analyzer

    def send_for_analysis(self, text):
        tone_analysis = self.tone_analyzer.tone(
            { 'text': text },
            'application/json',
        ).get_result()
        return json.dumps(tone_analysis)


import torch.nn as nn
import torch
import torch.nn.functional as F
from tqdm import tqdm

from attribution_bottleneck.bottleneck.estimator import ReluEstimator
from attribution_bottleneck.attribution.base import AttributionMethod
from attribution_bottleneck.bottleneck.per_sample_bottleneck import PerSampleBottleneck
from attribution_bottleneck.bottleneck.estimator import Estimator
from attribution_bottleneck.utils.misc import resize, replace_layer, to_np

class PerSampleBottleneckReader(AttributionMethod):
    def __init__(self, model, estim: Estimator, beta=10, steps=10, lr=1, batch_size=10,
                 sigma=1, progbar=False):
        self.model = model
        self.original_layer = estim.get_layer()
        self.shape = estim.shape()
        self.beta = beta
        self.batch_size = batch_size
        self.progbar = progbar
        self.device = list(model.parameters())[0].device
        self.lr = lr
        self.train_steps = steps
        self.bottleneck = PerSampleBottleneck(estim.mean(), estim.std(), device=self.device,
                                              sigma=sigma, relu=isinstance(estim, ReluEstimator))
        self.sequential = nn.Sequential(self.original_layer, self.bottleneck)

    def heatmap(self, input_t, target):
        target_t = torch.tensor([target]) if not isinstance(target, torch.Tensor) else target
        self._run_training(input_t, target_t)
        return self._current_heatmap(shape=input_t.shape[2:])

    def _run_training(self, input_t, target_t):
        # Attach layer and train the bottleneck
        replace_layer(self.model, self.original_layer, self.sequential)
        self._train_bottleneck(input_t, target_t)
        replace_layer(self.model, self.sequential, self.original_layer)

    def _current_heatmap(self, shape=None):
        # Read bottleneck
        heatmap = self.bottleneck.buffer_capacity
        heatmap = to_np(heatmap[0])
        heatmap = heatmap.sum(axis=0)  # Sum over channel dim
        heatmap = heatmap - heatmap.min()  # min=0
        heatmap = heatmap / heatmap.max()  # max=0

        if shape is not None:
            heatmap = resize(heatmap, shape)

        return heatmap

    def _train_bottleneck(self, input_t: torch.Tensor, target_t: torch.Tensor):

        assert input_t.shape[0] == 1, "We can only fit on one sample"
        assert target_t.shape[0] == 1, "We can only fit on one label"

        batch = input_t.expand(self.batch_size, -1, -1, -1), target_t.expand(self.batch_size)
        optimizer = torch.optim.Adam(lr=self.lr, params=self.bottleneck.parameters())

        # Reset from previous run or modifications
        self.bottleneck.reset_alpha()

        # Train
        self.model.eval()
        for _ in tqdm(range(self.train_steps), desc="Training Bottleneck",
                      disable=not self.progbar):
            optimizer.zero_grad()
            out = self.model(batch[0])
            loss_t = self.calc_loss(outputs=out, labels=batch[1])
            loss_t.backward()
            optimizer.step(closure=None)

    def calc_loss(self, outputs, labels):
        """ Calculate the combined loss expression for optimization of lambda """
        information_loss = self.bottleneck.buffer_capacity.mean()  # Taking the mean is equivalent of scaling with 1/K
        cross_entropy = F.cross_entropy(outputs, target=labels)
        total = cross_entropy + self.beta * information_loss
        return total


from bs4 import BeautifulSoup
import requests
import requests.exceptions
from urllib.parse import urlsplit
from collections import deque
import re
from lxml import etree
import pandas as pd
import time

years=range(2002,2020,2)
resDF=pd.DataFrame()
#resDF=pd.read_csv("hiphop_data.csv")
for year in years:
    for year in range(year,year+2):
        print(year)
        baseUrl = "https://www.billboard.com/charts/year-end/"+str(year)+"/hot-r-and-and-b-hip-hop-songs"
        strhtml = requests.get(baseUrl).content

        soup = BeautifulSoup(strhtml, 'lxml')

        songnames = []
        artistnames = []

        data=soup.select("div[class='ye-chart-item__title']")
        for item in data:
            songnames.append(item.get_text().strip())

        data=soup.select("div[class='ye-chart-item__artist']")
        for item in data:
            artistnames.append(item.get_text().strip())

        track = zip(songnames, artistnames, [year] * 100, ["hiphop"] * 100)
        tmp_df = pd.DataFrame(track)
        resDF = pd.concat([resDF, tmp_df])
        time.sleep(5)

resDF.to_csv("hiphop_data.csv",index=False)

resDF=pd.read_csv("hiphop_data.csv")


import os
import sys
import datetime
import numpy as np
import xarray as xr
import dask
import dask.diagnostics
import cartowik.profiletools as cpf
import hyoga.open

# processed runs
PROC_RUNS = ['alpcyc4.2km.grip.0820', 'alpcyc4.2km.grip.1040.pp',
             'alpcyc4.2km.epica.0970', 'alpcyc4.2km.epica.1220.pp',
             'alpcyc4.2km.md012444.0800', 'alpcyc4.2km.md012444.1060.pp',
             'alpcyc4.1km.epica.1220.pp']

# global attributes
GLOB_ATTRS = dict(
    title='Alpine ice sheet glacial cycle erosion aggregated variables',
    author='Julien Seguinot',
    institution='',
    command='{user}@{host} {time}: {cmdl}\n'.format(
        user=os.getlogin(), host=os.uname()[1], cmdl=' '.join(sys.argv),
        time=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')),
    comment="""Aggregated dataset contents:
* Spatial variables aggregated in time over the entire simulation lenght from
120,000 years before present to the present.
* Time-series variables aggregated in space over the entire glaciated area,
defined by a 1-metre ice thickness threshold.
""")

def postprocess_extra(run_path):
    """Postprocess extra dataset for one run."""

    # output file and subtitle
    _, res, rec, *other = os.path.basename(run_path).split('.')
    out_file = 'processed/alpero.{}.{}.{}.agg.nc'.format(
        res, rec[:4], 'pp' if 'pp' in other else 'cp')

    # load output data (in the future combine='by_coords' will be the default)
    print("postprocessing " + out_file + "...")
    boot = hyoga.open.dataset(
        '~/pism/input/boot/'+'alps.srtm.hus12.gou11simi.{}.nc'.format(res))
    print("* reading multi-file extra dataset...")
    ex = hyoga.open.mfdataset(run_path+'/ex.???????.nc', parallel=True)

    # init postprocessed dataset with global attributes
    pp = xr.Dataset(attrs=ex.attrs, coords=dict(lon=ex.lon, lat=ex.lat))
    pp.attrs.update(
        subtitle='{} {} simulation {} precipitation reductions'.format(
            res, rec.upper(), 'with' if 'pp' in other else 'without'),
        **GLOB_ATTRS)
    pp.attrs.update(history=pp.command+pp.history)

    # register intermediate variables (I assumed the K_g numbers given in
    # supplement of Koppes et al., 2015 are for erosion rates in mm/a. An
    # alternative formula with two outliers removed gives 5.3e−12*v_b*2.62
    # (m/a). Now for some reason, these erosion laws seem to not exactly
    # correspond to the blue and pink lines on the summary figure by Cook et
    # al., 2020, which appear to be a factor two or three lower.)
    ex['icy'] = (ex.thk >= 1.0)
    ex['bedlift'] = ex.topg - boot.topg.where(boot.topg > 0, 0)
    ex['sliding'] = ex.velbase_mag.where(ex.icy)
    ex['warmbed'] = ex.icy*(ex.temppabase >= -1e-3)
    ex['coo2020'] = (1.665e-4*ex.sliding**0.6459).assign_attrs(
        ref='Cook et al. (2020)', units='m a-1')
    ex['her2015'] = (2.7e-7*ex.sliding**2.02).assign_attrs(
        ref='Herman et al. (2015)', units='m a-1')
    ex['hum1994'] = (1e-4*ex.sliding).assign_attrs(
        ref='Humphrey and Raymond (1994)', units='m a-1')
    ex['kop2015'] = (5.2e-11*ex.sliding**2.34).assign_attrs(
        ref='Koppes et al. (2015)', units='m a-1')

    # compute grid size
    dt = ex.age[0] - ex.age[1]
    dt = dt*1e3  # convert ka to a
    dx = ex.x[1] - ex.x[0]
    dy = ex.y[1] - ex.y[0]

    # compute glacial cycle integrated variables
    pp['cumu_sliding'] = (dt*ex.sliding.sum(axis=0, min_count=1)).assign_attrs(
        long_name='cumulative basal motion', units='m')
    pp['glacier_time'] = (dt*ex.icy.sum(axis=0)).assign_attrs(
        long_name='total ice cover duration', units='years')
    pp['warmbed_time'] = (dt*ex.warmbed.sum(axis=0)).assign_attrs(
        long_name='temperate-based ice cover duration', units='years')

    # trigger computation to avoid memory errors (18m, 70%, 8GiB)
    # NOTE: time and total mem for single-thread scheduler on altair, with
    # nothing else running. Trigger compute() more often to further reduce mem
    # consumption (maybe at the cost of repeating some operations).
    print("* computing time-integrated variables...")
    with dask.diagnostics.ProgressBar():
        pp = pp.compute()

    # compute timeseries
    pp['glacier_area'] = (dx*dy*ex.icy.sum(axis=(1, 2))).assign_attrs(
        long_name='glacierized area', units='m2')
    pp['volumic_lift'] = (dx*dy*ex.bedlift.sum(axis=(1, 2))).assign_attrs(
        long_name='volumic bedrock uplift', units='m3')
    pp['warmbed_area'] = (dx*dy*ex.warmbed.sum(axis=(1, 2))).assign_attrs(
        long_name='temperate-based ice cover area', units='m2')

    # trigger computation to avoid memory errors (24m, 35%, 4GiB)
    print("* computing space-integrated variables...")
    with dask.diagnostics.ProgressBar():
        pp = pp.compute()

    # compute erosion time-aggregated variables
    for law in ['coo2020', 'her2015', 'hum1994', 'kop2015']:
        pp[law+'_cumu'] = (dt*ex[law].sum(axis=0, min_count=1)).assign_attrs(
            long_name=ex[law].ref+' cumulative glacial erosion potential',
            units='m')

    # trigger computation to avoid memory errors (12m, 35%, 4GiB)
    print("* computing cumulative erosion potential...")
    with dask.diagnostics.ProgressBar():
        pp = pp.compute()

    # compute erosion space-aggregated variables
    # there's still a divide warning here, maybe check what that is
    # NOTE: for the 1-km-run hypsogram on 10-m bands, dask-2021.3.0 splits the
    # chunks even when asked not to. The progress bar only appears after two
    # hours (I nearly gave up), but the actual computations takes only 15 min!
    # In total the script took 194 min on altair.
    x, y = cpf.read_shp_coords(
        '../data/native/profile_rhine.shp', interval=1000)
    for law in ['coo2020', 'her2015', 'hum1994', 'kop2015']:
        pp[law+'_rate'] = (dx*dy*ex[law].sum(axis=(1, 2))).assign_attrs(
            long_name=ex[law].ref+' domain total volumic erosion rate',
            units='m3 year-1')
        pp[law+'_hyps'] = np.exp(np.log(ex[law]).groupby_bins(
            boot.topg, bins=range(0, 4501, 100)).mean(
                dim='stacked_y_x')).assign_attrs(
            long_name=ex[law].ref+' erosion rate geometric mean in band',
            units='m year-1')
        pp[law+'_rhin'] = ex[law].where(ex.icy).interp(
                x=x, y=y, method='linear').assign_attrs(
            long_name=ex[law].ref+' rhine transect erosion rate',
            units='m year-1')

    # trigger computation to avoid memory errors (12m, 40%, 5GiB)
    print("* computing erosion rate variables...")
    with dask.diagnostics.ProgressBar():
        pp = pp.compute()

    # compute glacier cover hypsogram
    pp['glacier_hyps'] = (
        dx*dy*ex.icy.groupby_bins(boot.topg, bins=range(0, 4501, 100)).sum(
            dim='stacked_y_x')).assign_attrs(
        long_name='glacierized area within elevation band', units='m2')

    # trigger computation to avoid memory errors (6m, 40%, 5GiB)
    print("* computing glacier cover hypsogram...")
    with dask.diagnostics.ProgressBar():
        pp = pp.compute()

    # replace intervals (xarray issue #2847)
    pp['topg_bins'] = [b.mid for b in pp.topg_bins.values]
    pp = pp.rename({'topg_bins': 'z'})
    pp['z'] = pp.z.assign_attrs(
        long_name='elevation band midpoints', units='m')
    pp['d'] = pp.d.assign_attrs(
        long_name='distance along transect', units='m')

    # copy grid mapping and pism config
    pp['mapping'] = ex.mapping
    pp['pism_config'] = ex.pism_config

    # export to netcdf
    print("* writing postprocessed file...")
    delayed = pp.to_netcdf(
        out_file, mode='w', compute=False, encoding={var: dict(
            zlib=True, shuffle=True, complevel=1) for var in pp.variables})
    with dask.diagnostics.ProgressBar():
        delayed.compute()

    # close datasets
    boot.close()
    ex.close()
    pp.close()

def main():
    """Main program called during execution."""

    # create directory if missing
    if not os.path.exists('processed'):
        os.makedirs('processed')

    # to use the dask distributed client
    # client = dask.distributed.Client()
    # print(client.dashboard_link)

    # ask dask to not split large chunks (but it still does)
    dask.config.set(**{'array.slicing.split_large_chunks': False})

    # postprocess selected runs
    for run in PROC_RUNS:
        postprocess_extra(os.environ['HOME'] + '/pism/output/e9d2d1f/' + run)

if __name__ == '__main__':
    main()


import sqlite3, time
from os import path

# log database SQL
CREATE_LOG   = 'CREATE TABLE conn_log   (id INTEGER PRIMARY KEY AUTOINCREMENT, user INTEGER, proto INTEGER, binary TEXT, dport INTEGER, dest STRING)'
CREATE_STAMP = 'CREATE TABLE conn_stamp (id INTEGER PRIMARY KEY AUTOINCREMENT, log_id INTEGER, outcome INTEGER, time DATE)'
INSERT_LOG   = 'INSERT INTO conn_log (user, proto, binary, dport, dest) VALUES (?, ?, ?, ?, ?)'
INSERT_STAMP = 'INSERT INTO conn_stamp (log_id, outcome, time) VALUES (?, ?, ?)'
EXISTS_LOG   = 'SELECT count(id) FROM conn_log WHERE user=? and proto=? and binary=? and dport=? and dest=?'
LOG_ID       = 'SELECT id FROM conn_log WHERE user=? and proto=? and binary=? and dport=? and dest=?'

# violatoin database SQL
CREATE_VIOLATION = 'CREATE TABLE conn_violation (id INTEGER PRIMARY KEY AUTOINCREMENT, user INTEGER, proto INTEGER, binary TEXT, dport INTEGER, dest STRING)'
CREATE_HANDLED  = 'CREATE TABLE conn_handled   (id INTEGER PRIMARY KEY AUTOINCREMENT, user INTEGER, proto INTEGER, binary TEXT, dport INTEGER, dest STRING)'
EXISTS_VIOLATION = 'SELECT count(id) FROM conn_violation WHERE user = ? AND proto = ? AND binary = ? AND dport = ? AND dest = ?'
EXISTS_HANDLED   = 'SELECT count(id) FROM conn_handled   WHERE user = ? AND proto = ? AND binary = ? AND dport = ? AND dest = ?'
INSERT_VIOLATION = 'INSERT INTO conn_violation (user, proto, binary, dport, dest) VALUES (?, ?, ?, ?, ?)'

class _Logger:
    ''' Log utility class
    '''
    def __init__(self, suffix, log_dir, tables, use_date=True):
        self.suffix   = suffix
        self.log_dir  = log_dir
        self.tables   = tables
        self.use_date = use_date
        self.conn     = self.open_log()

    def open_log(self):
        ''' Create and/or connect to database
        '''
        log_name = '%s/%s.%s' % (self.log_dir, self._next_log(), self.suffix)
        if not self.use_date:
            log_name = '%s/%s.%s' % (self.log_dir, 'log', self.suffix)

        exists = path.exists(log_name)
        conn   = sqlite3.connect(log_name)

        # create the table structure if the log did not exist
        if not exists:
            for table in self.tables:
                conn.execute(table)
                conn.commit()

        return conn

    def _next_log(self):
        self.date = time.localtime()
        return "%s.%s.%s" % (self.date.tm_year, self.date.tm_mon, self.date.tm_mday)

    def _check_rotate(self):
        return time.localtime() != self.date.tm_mday

    def rotate_log(self):
        ''' Rotate the database (based on day of month)
        '''
        if not self.use_date:
            return
        # close the open connection and open a new database
        self.cleanup()
        self.conn = self.open_log()

    def cleanup(self):
        ''' Cleanup database connections on shutdown
        '''
        self.conn.close()

class ConnectionLogger(_Logger):
    ''' Log utility for connection information gathered
        return - rowid for the timestamp entry
    '''
    def __init__(self, log_dir):
        _Logger.__init__(self, 'log', log_dir, ( CREATE_LOG, CREATE_STAMP ))

    def log_conn(self, uid, proto, app, dport, dest, outcome):
        ''' Log a new outgoing connection
        '''
        # hand log rotation
        if self._check_rotate():
            self.rotate_log()

        last_curr = None
        # another instance of an already seen connection
        if self.conn.execute(EXISTS_LOG, (uid, proto, app, dport, dest)).next()[0] > 0:
            last_curr = self.conn.execute(INSERT_STAMP, (self.conn.execute(LOG_ID, (uid, proto, app, dport, dest)).next()[0], outcome, time.asctime()))
        # a never seen connection
        else:
            curr = self.conn.execute(INSERT_LOG, (uid, proto, app, dport, dest))
            last_curr = self.conn.execute(INSERT_STAMP, (curr.lastrowid, outcome, time.asctime()))
        self.conn.commit()

        return last_curr.lastrowid

# SQL REQUIRED
# HANDLED LOGGER REQUIRED
class ViolationLogger(_Logger):
    ''' Log utility for policy violations
    '''
    def __init__(self, log_dir):
        _Logger.__init__(self, 'violation', log_dir, ( CREATE_VIOLATION, ))
        self.handled = HandledLogger(log_dir)

    def exists(self, uid, proto, app, dport, dest):
        ''' Check if a violation was already logged
        '''
        return self.conn.execute(EXISTS_VIOLATION, (uid, proto, app, dport, dest)).next()[0] > 0

    def log_violation(self, uid, proto, app, dport, dest):
        ''' Log a new violation
        '''
        # hand log rotation
        if self._check_rotate():
            self.rotate_log()

        # add violations that have not yet been recorded
        if not self.handled.exists(uid, proto, app, dport, dest) and not self.exists(uid, proto, app, dport, dest):
            self.conn.execute(INSERT_VIOLATION, (uid, proto, app, dport, dest))
        self.conn.commit()

class HandledLogger(_Logger):
    ''' Log utility for handled policy violations
    '''
    def __init__(self, log_dir):
        _Logger.__init__(self, 'handled', log_dir, ( CREATE_HANDLED, ), use_date=False)

    def exists(self, uid, proto, app, dport, dest):
        ''' Check if a violation was already handled
        '''
        return self.conn.execute(EXISTS_HANDLED, (uid, proto, app, dport, dest)).next()[0] > 0


#!/usr/bin/env python
import keras, tensorflow as tf, numpy as np, gym, sys, copy, argparse
import random
from algs.DQN import create_dqn
from algs.a2c import A2C
from algs.ddpg import DDPG
import time
import logging
import os.path

logger = logging.getLogger("10703")
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')

logs_dir = "logs"
curr_model_dir = None
models_dir = "models"

a2c_model = "algs/a2c.py"
dqn_model = "algs/DQN.py"

def parse_arguments():
	# Ex. python main.py --env CartPole-v0 --replay-batch 32 --model-name cartpole_dqn_w_mem --deepness deep
    parser = argparse.ArgumentParser(description='Experience Replay Argument Parser')
    parser.add_argument('--env-name',dest='env_name',type=str, default="LunarLander-v2")
    parser.add_argument('--alg', type=str, default="a2c", help="a2c or dqn")
    parser.add_argument('--model-name',dest='model_name',type=str,required=True)

    parser.add_argument('--seed', dest='seed', type=int)
    parser.add_argument('--num-episodes', dest='num_episodes', type=int,
                        default=50000, help="Number of episodes to train on.")
    parser.add_argument('--gamma', type=float, default=0.99)

    parser.add_argument('--memory-size', type=int, default=50000, help="ER buffer initial max size")

    # Utils
    parser.add_argument('--render', dest='render', action="store_true")
    parser.add_argument('--record-video-only',dest='record_video_only',action="store_true")
    parser.add_argument('--test-only', action="store_true")
    parser.add_argument('--train-mod', type=int, default=100, help="Print every x number of episodes")
    parser.add_argument('--test-mod', type=int, default=500, help="Test every x number of episodes")

    # A2C
    parser.add_argument('--actor-model-path', type=str, default=None,
                        help="Path to the actor model file.")
    parser.add_argument('--actor-lr', type=float,
                        default=0.01, help="The actor's learning rate.")
    parser.add_argument('--critic-model-path', type=str, default=None)
    parser.add_argument('--critic-lr', type=float,
                        default=0.01, help="The critic's learning rate.")

    parser.add_argument('--N', dest='N', type=int,
                        default=50, help="The number of steps in N-step A2C.")

    # DQN
    parser.add_argument('--epsilon', dest='epsilon', type=float, default=0.5)
    parser.add_argument('--deepness',dest='deepness',type=str,default=False)
    parser.add_argument('--lr',dest='lr',type=float,default=0.0001)
    parser.add_argument('--alt-learn', dest='alt_learn', action="store_true")

    #DDPG
    parser.add_argument('--tau', type=float, default=0.001)

    # ER
    parser.add_argument('--replay-batch',dest='replay_batch',type=int, default=32)
    parser.add_argument('--hindsight-replay', dest='hindsight_replay',action="store_true")
    parser.add_argument('--combined-replay', dest='combined_replay', action="store_true")
    parser.add_argument('--priority-replay', dest='priority_replay', action="store_true")
    # TODO: add diffeq ultimate replay evolution adaptation

    return parser.parse_args()

def main(args):

    global curr_model_dir

    args = parse_arguments()

    if not os.path.exists(models_dir):
        os.makedirs(models_dir)

    curr_model_dir = f"{models_dir}/{args.model_name}/"
    if not os.path.exists(curr_model_dir):
        os.makedirs(curr_model_dir)

    if not os.path.exists(logs_dir):
        os.makedirs(logs_dir)

    #start logging settings
    os.environ['TZ'] = 'EST+05EDT,M4.1.0,M10.5.0'
    time.tzset()

    time_str = time.strftime('%Y_%m_%d_%H_%M_%S')
    log_file_name = f'{logs_dir}/{args.model_name}_{time_str}.log'
    hdlr = logging.FileHandler(log_file_name)
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.DEBUG)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    #end logging settings

    # Setting the session to allow growth, so it doesn't allocate all GPU memory.
    gpu_ops = tf.GPUOptions(allow_growth=True)
    config = tf.ConfigProto(gpu_options=gpu_ops)
    sess = tf.Session(config=config)

    # Setting this as the default tensorflow session.
    keras.backend.tensorflow_backend.set_session(sess)

    #assert(args.env in ["CartPole-v0", "MountainCar-v0", "LunarLander-v2"])
    assert(args.alg in ["a2c", "dqn", "ddpg"])

    logger.info(f"Command line args: {args}")
    logger.info(f"Log saving to {log_file_name}")
    logger.info(f"Alg: {args.alg}")

    if args.hindsight_replay:
        if args.env_name == "MountainCar-v0" or args.env_name == "MountainCarContinuous-v0":
            default_goal = np.array([[0.5]]) # flag at x = 0.5
        elif args.env_name == "LunarLander-v2" or args.env_name == "LunarLanderContinuous-v2":
            default_goal = np.array([[0.0, 0.0]]) # landing pad at x,y = (0,0)
        elif args.env_name == "Pendulum-v0":
            default_goal = np.array([[1.0, 0.0]]) # cos(theta), sin(theta), theta dot
                                                    # theta is normalized between pi and -pi
        else:
            raise ValueError("Hindsight not enabled for this env")

    else:
        default_goal = None

    if args.seed != None:
        time_seed = args.seed
    else:
        #set consistent seed based on time
        time_seed = int(''.join(time_str.split('_'))) % (2 ** 32)

    np.random.seed(time_seed)
    logger.info(f"Numpy random seed {time_seed}")

    env = gym.make(args.env_name)
    num_inputs = env.observation_space.shape[0]

    num_outputs = None
    try:
        num_outputs = env.action_space.n
    except AttributeError:
        pass
    print(f"Num env inputs (state space): {num_inputs}")
    print(f"Num env outputs (actions): {num_outputs}")

    if args.alg == "a2c":
        agent = A2C(env, args.model_name, args.actor_model_path, args.actor_lr,
            args.critic_model_path, args.critic_lr, N=args.N, logger=logger)
    elif args.alg == "dqn":
        agent, use_episodes, num_train_episodes, num_train_steps = create_dqn(logger, args, env, default_goal, curr_model_dir, time_seed)
    elif args.alg == "ddpg":
        agent = DDPG(env, args.critic_lr, args.actor_lr, args.gamma, tau=args.tau,
            batch_size=args.replay_batch, default_goal=default_goal)

    if args.record_video_only:
        agent.test(record_video=True)
        return

    if args.test_only:
        agent.test()
    else:
        if args.alg == "a2c":
            assert not args.priority_replay, "NYI"
            assert not args.combined_replay, "NYI"
            assert not args.hindsight_replay, "NYI"
            agent.train(args.num_episodes, gamma=args.gamma,
                report_interval=args.train_mod, test_interval=args.test_mod, render=args.render)
        elif args.alg == "dqn":
            agent.train(use_episodes, num_train_episodes, num_train_steps,
            rep_batch_size = False if args.replay_batch == 0 else args.replay_batch,
            print_episode_mod=args.train_mod, test_episode_mod=args.test_mod,
            replay_mem_size=args.memory_size, default_goal=default_goal)
        elif args.alg == "ddpg":
            agent.train(env, args.num_episodes, default_goal=default_goal, hindsight_replay=args.hindsight_replay,
                priority_replay=args.priority_replay, combined_replay=args.combined_replay,
                train_mod=args.train_mod, test_mod=args.test_mod)

    logger.info(f"Log saved to {log_file_name}")

if __name__ == '__main__':
	main(sys.argv)


#dirt.py
import pygame, sys

class Dirt(pygame.sprite.Sprite):
    def __init__(self, player, x, y, width, height):
        pygame.sprite.Sprite.__init__(self)
        self.dirting = False
        self.currentSprite = 0
        self.player = player
        self.x = x
        self.y = y
        self.width = width
        self.height = height
        self.dirt_0 = pygame.image.load("assets/dirt_0.png")
        self.dirt_1_left = pygame.image.load("assets/dirt_1_left.png")
        self.dirt_2_left = pygame.image.load("assets/dirt_2_left.png")
        self.dirt_1_right = pygame.image.load("assets/dirt_1_right.png")
        self.dirt_2_right = pygame.image.load("assets/dirt_2_right.png")
        self.image = self.dirt_0
        self.rect = self.image.get_rect()
        self.dirt_0 = pygame.transform.scale(self.dirt_0, (self.rect.width*4, self.rect.height*4))
        self.dirt_1_left = pygame.transform.scale(self.dirt_1_left, (self.rect.width*4, self.rect.height*4))
        self.dirt_2_left = pygame.transform.scale(self.dirt_2_left, (self.rect.width*4, self.rect.height*4))
        self.dirt_1_right = pygame.transform.scale(self.dirt_1_right, (self.rect.width*4, self.rect.height*4))
        self.dirt_2_right = pygame.transform.scale(self.dirt_2_right, (self.rect.width*4, self.rect.height*4))
        self.image = self.dirt_0
        self.rect = self.image.get_rect()
        self.rect.y = self.player.rect.y

    def update(self):
        if(self.player.currentSprite < 3 or self.player.currentSprite > 4):
            self.dirt()
            self.rect.x = self.player.rect.x + 5 * (4 - self.player.currentSprite)
            if(self.player.currentSprite < 3):
                if(self.currentSprite == 0):
                    self.image = self.dirt_1_left
                else:
                    self.image = self.dirt_2_left
            else:
                if(self.currentSprite == 0):
                    self.image = self.dirt_1_right
                else:
                    self.image = self.dirt_2_right
        else:
            self.image = self.dirt_0
    def dirt(self):
        if(self.currentSprite == 0):
            self.currentSprite = 1
        else:
            self.currentSprite = 0


# -*-coding:utf-8-*-
"""
# File       : test_meter_profile.py
# Time       ：2021/12/21 14:18
# Author     ：cao jiann
# version    ：python 3.7
"""

import datetime

from common.HESRequest import *
from common.marker import *

class Test_Meter_Relay_Status:

    @smokeTest
    def test_read_relay_status_sync(self, requestMessage, caseData, device):
        """
        使用同步读取的方式去对电表进行读取闸状态
         """
        data = caseData('testData/RelayControlTask/read_RelayControlStatus.json')
        requestData = data['ReadRelayControlStatusSync']['request']
        requestData['payload'][0]['deviceNo'] = device['device_number']
        transactionId = str(device['device_number']) + '_' + time.strftime('%y%m%d%H%M%S', time.localtime())
        requestData['payload'][0]['transactionId'] = transactionId
        response, elapsed = HESRequest().post(url=requestMessage, params=requestData)
        print('Request ---  ', response)
        assert '636F6E6E6563746564' in str(response) or '646973636F6E6E6563746564' in str(response)

    @asyncTest
    def test_read_relay_status_async(self, device, requestMessage, dbConnect, caseData):
        """
        使用异步读取的方式去对电表进行读取闸状态
         """
        count = 0
        print("Step 1 : 生成异步操作读取任务，hes-api异步执行，生成running表")
        data = caseData('testData/RelayControlTask/read_RelayControlStatus.json')
        requestData = data['ReadRelayControlStatusAsync']['request']
        # 设定三分钟异步任务，三分钟后失效
        currentTime = datetime.datetime.now().strftime('%y%m%d%H%M%S')
        endTime = (datetime.datetime.now() + datetime.timedelta(minutes=3)).strftime('%y%m%d%H%M%S')

        requestData['payload'][0]['startTime'] = currentTime
        requestData['payload'][0]['endTime'] = endTime
        requestData['payload'][0]['deviceNo'] = device['device_number']
        transactionId = str(device['device_number']) + '_' + time.strftime('%y%m%d%H%M%S', time.localtime())
        requestData['payload'][0]['transactionId'] = transactionId
        response = requests.post(url=requestMessage, json=requestData)
        assert response.status_code == 200

        print("Step 2 : 生成异步任务后，任务正常执行完毕进入his，进入his表，则认为任务结束")
        # 过期时间到，也会进入his表，这里暂不考虑
        time.sleep(3)
        # 查询生成Core执行的任务的 auto_run_id
        sql_running = "select auto_run_id from H_TASK_RUNNING where NODE_NO='{}' and JOB_TYPE='GET_COMMON_PARAM'".format(
            device['device_number'])
        db_queue = dbConnect.fetchall_dict(sql_running)
        while len(db_queue) == 0 and count < 2:
            time.sleep(3)
            db_queue = dbConnect.fetchall_dict(sql_running)
            print(db_queue)
            print('Waiting for read relay_control Tasks to Create...')
            count = count + 1

        print("Step 3 : 验证读任务执行结束，task正常移入his表")
        # 查询生成Core执行结束后，his表任务状态
        sql_his = "select task_state from H_TASK_RUN_HIS where auto_run_id='{}'".format(db_queue[0]['auto_run_id'])
        db_queue = dbConnect.fetchall_dict(sql_his)
        while len(db_queue) == 0 and count < 50:
            time.sleep(5)
            db_queue = dbConnect.fetchall_dict(sql_his)
            print(db_queue)
            print('Waiting for read relay_control Tasks to finish...')
            count = count + 1
        assert db_queue[0]['task_state'] == 3


import logging
import tempfile

import pytest

from torchsim.core.graph import Topology
from torchsim.core.persistence.loader import Loader
from torchsim.core.persistence.saver import Saver
from torchsim.research.research_topics.rt_1_1_1_one_expert_sp.topologies.mnist_sp_topology import MnistSpTopology
from tests.core.graph.test_save_load import TAState
from tests.testing_utils import discover_main_topology_classes

logger = logging.getLogger(__name__)

skip_topologies = ['BaselinesReinforcementLearningTopology', 'SeToyArchDebugTopology',
                   'SEDatasetSampleCollectionTopology', 'DebugAgentTopology', 'NNetTopology',
                   'LrfObjectDetectionTopology']

@pytest.mark.parametrize('topology_class', discover_main_topology_classes())
def test_topologies_can_be_initialized(topology_class):
    topology_class()

@pytest.mark.slow
@pytest.mark.parametrize('topology_class', discover_main_topology_classes(skip_topologies))
def test_topologies_can_run_step(topology_class):
    """Just try that topologies can be initialized and run for one step."""
    topology_class().step()

@pytest.mark.parametrize('topology_class', [MnistSpTopology])#, Task0TaSeTopology, L3ConvTopology, L3SpConvTopology, GradualLearningBasicTopology])
def test_save_load(topology_class):
    """Unit test for saving and loading of topologies.

    1. Initialize a topology
    2. Take a snapshot of its state ("before state")
    3. Save the topology
    4. Change it
    5. Verify that it changed from the before state
    6. Load the topology
    7. Verify that it is now equal to the before state
    """
    topology = topology_class()
    topology.step()
    state_before = TAState(topology)

    with tempfile.TemporaryDirectory() as directory:
        saver = Saver(directory)
        topology.save(saver)
        saver.save()

        change_topology(topology)
        state_after_change = TAState(topology)
        assert state_after_change != state_before

        loader = Loader(directory)
        topology.load(loader)

    state_after_load = TAState(topology)
    assert state_before == state_after_load

@pytest.mark.skip(reason="Specialized test")
@pytest.mark.parametrize('topology_class, n_steps', [(MnistSpTopology, 1000)])
def test_save_load_many_steps(topology_class, n_steps):
    """A variation of the save/load test, for multi-step state changes.

    There can be parts of a node's state (e.g. a random number that was generated) that is not stored immediately in
    memory blocks but manifests themselves in the memory blocks after a number of steps.

    This is test specialized in detecting the failure to save/load those parts of the state.
    """
    topology = topology_class()
    topology.step()
    state_before = TAState(topology)

    with tempfile.TemporaryDirectory() as directory:
        saver = Saver(directory)
        topology.save(saver)
        saver.save()

        for _ in range(n_steps):
            topology.step()
        state_after_steps = TAState(topology)
        assert state_after_steps != state_before

        loader = Loader(directory)
        topology.load(loader)

    state_after_load = TAState(topology)
    assert state_after_load == state_before

    for _ in range(n_steps):
        topology.step()
    state_after_load_and_steps = TAState(topology)
    assert state_after_load_and_steps == state_after_steps

def change_topology(topology: Topology):
    """Changes the state of the topology.

    Just running a step is not guaranteed to effect a change, so we explicitly alter the memory blocks.
    """
    for ta_node in TAState.get_ta_nodes(topology):
        for block in ta_node.memory_blocks:
            if block is not None and block.tensor is not None:
                block.tensor.add_(1)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import Flask
from validator import MyRequestValidator
from core import db, oauth
from views import yoloapi

def create_app(settings_override=None):
    """ Method for creating and initializing application.

        :param settings_override: Dictionary of settings to override.
    """
    app = Flask(__name__)

    # Update configuration.
    app.config.from_object('settings')
    app.config.from_pyfile('settings.cfg', silent=True)
    app.config.from_object(settings_override)

    # Initialize extensions on the application.
    db.init_app(app)
    oauth.init_app(app)
    oauth._validator = MyRequestValidator()

    # Register views on the application.
    app.register_blueprint(yoloapi)

    return app

if __name__ == '__main__':

    # Enable Flask-OAuthlib logging for this application.
    import logging
    logger = logging.getLogger('flask_oauthlib')
    logger.addHandler(logging.StreamHandler())
    logger.setLevel(logging.DEBUG)

    # Create app and SQL schemas in database, then run the application.
    app = create_app()
    db.create_all(app=app)
    app.run(ssl_context='adhoc')


import urllib.request
from bs4 import BeautifulSoup
import json
import PostScrape

extractedRecords = []

def crawlPagesUpTo(pages):
    if int(pages) < 1:
        print("Invalid integer.")
        return
    headers = {'user-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.3'}
    print('Extracting from a total of %d links.'%(int(pages) * 25))
    for page in range(int(pages)):
        curr = "https://old.reddit.com/top/"
        if page != 0:
            curr += "?count=" + str(page * 25) + "&after=t3_" + str(getRecords()[-1]['tag'])
        request = urllib.request.Request(curr, headers=headers)
        html = urllib.request.urlopen(request).read()
        soup = BeautifulSoup(html, 'html.parser')
        siteTable = soup.find("div", attrs={'id':'siteTable'})
        links = siteTable.find_all("a", class_="title")
        commentLinks = siteTable.find_all("a", class_="bylink comments may-blank")
        for index in range(25):
            PostScrape.resetDetails()
            title = links[index].text
            url = links[index]['href']
            comments = commentLinks[index].text
            commenturl = commentLinks[index]['href']
            tag = commenturl.split('/')[6]
            ad = False
            print('%d. Extracting data from %s.'%((index + 1), commenturl))
            PostScrape.scrapeDetails(commenturl)
            postDetails = PostScrape.getDetails()
            if not url.startswith('http'):
                url = "https://old.reddit.com" + url
            if not commenturl.startswith('http'):
                commenturl = "https://old.reddit.com" + commenturl
            if url.startswith('https://alb.reddit.com') or url.startswith('http://alb.reddit.com'):
                ad = True
            record = {
                'title':title,
                'url':url,
                'comments':comments,
                'commenturl':commenturl,
                'tag':tag,
                'ad':ad,
                'postDetails':postDetails
            }
            extractedRecords.append(record)
        print('Finished extracting from a total of %d links.'%(int(pages) * 25))

def resetRecords():
    extractedRecords.clear()

def dumpRecords():
    with open('dumpCrawl.json', 'w') as outfile:
        json.dump(extractedRecords, outfile)

def getRecords():
    return extractedRecords

if __name__ == "__main__":
    crawlPagesUpTo(input("How many pages would you like to crawl? "))
    dumpRecords()


from collections import deque

def bfs():
    while q1:
        x, y, cnt = q1.popleft()
        visited[x][y]=cnt

        for i in range(4):
            nx=x+dx[i]
            ny=y+dy[i]

            if 0<=nx<r and 0<=ny<c:
                if graph[nx][ny] == "X": continue  # 돌이면
                elif visited[nx][ny]==-1 and graph[nx][ny]!="D":
                    visited[nx][ny]=visited[x][y]+1
                    graph[nx][ny]="*"
                    q1.append((nx,ny,cnt+1))

def bfs2(x,y):
    global result
    visited[x][y] = 0

    while q2:
        x, y = q2.popleft()

        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]

            if 0<=nx<r and 0<=ny<c:
                if graph[nx][ny] == "X": continue
                elif graph[nx][ny]=="D": #비버 굴 도착
                    result=visited[x][y]+1
                    return #리턴 해주기
                else:
                    # "." , "*" 일때
                    if visited[nx][ny] != -1 and visited[nx][ny] <= visited[x][y] + 1:
                        continue
                    visited[nx][ny]=visited[x][y]+1
                    q2.append((nx,ny))

if __name__ == '__main__':
    r,c=map(int,input().split())
    graph=[]
    dx=[-1,1,0,0]
    dy=[0,0,-1,1]

    for i in range(r):
        graph.append(list(input().rstrip()))

    visited=[[-1 for _ in range(c)] for _ in range(r)]
    water=[]

    q1=deque()
    q2=deque()
    for i in range(r):
        for j in range(c):
            if graph[i][j]=="*":
                cnt=0
                q1.append((i,j,cnt))
            elif graph[i][j]=="S":
                cnt2=0
                p,q=i,j
                q2.append((i,j))

    result=-1
    bfs()
    bfs2(p,q)

    if result==-1:
        print("KAKTUS")
    else:
        print(result)


from flask import Flask, request, jsonify
from flask_cors import CORS, cross_origin
import base64
import replicate
import io
import openai
import os

app = Flask(__name__)
cors = CORS(app)

openai.api_key = os.getenv('OPENAIKEY')

@app.route("/process", methods=['POST'])
@cross_origin()
def process():
    picture_data = request.json['picture']
    picture_bytes = base64.b64decode(picture_data.split(',')[1])
    image_file = io.BytesIO(picture_bytes)
    output = replicate.run(
                "salesforce/blip:2e1dddc8621f72155f24cf2e0adbde548458d3cab9f00c0139eea840d0ac4746",
                input={
                    "image": image_file,
                    "task": "visual_question_answering",
                    "question": "What item is this?"
                    }
            )

    remove_caption = output.split(':')[1]
    prompt = create_prompt(remove_caption)
    answer = get_response(prompt)
    return jsonify({"answer": answer})

def create_prompt(message):
    return ("Please provide a a super short, succinct, useful explaination while avoiding generalizations and summaries. How do I recycle: "
                + message
            )

def get_response(message):
    response = openai.ChatCompletion.create(
        model = 'gpt-3.5-turbo',
        temperature = 1,
        messages = [
                {"role": "user", "content": message}
                ]
    )
    rv = response.choices[0]["message"]["content"]
    return rv

if __name__ == "__main__":
    app.run(debug=True)


from elasticsearch import Elasticsearch, helpers
import json
import argparse

parser = argparse.ArgumentParser()
#required args

parser.add_argument('--api_key', dest='api_key', required=True)
parser.add_argument('--cloud_id', dest='cloud_id', required=True)

args = parser.parse_args()

print("Init elasticsearch connection")
es = Elasticsearch(
    cloud_id=args.cloud_id,
    api_key=args.api_key,
    request_timeout=30
)
print("Elasticsearch connection establish")
moviesFile = open("movies.json")
moviesJson = json.load(moviesFile)
print("Indexing data to Elasticsearch")
status = helpers.parallel_bulk(es, moviesJson, index='search-movies')
# look through each result for status
for i in status:
    if i[0] == False:
        print("There was an error importing a record.  Error: ", i[1])
# if not success:
#     print(info)
print("Finish")


from typing import List

class Solution:
    def findMaxLength(self, nums: List[int]) -> int:
        zero_count = 0
        one_count = 0

        # current count for each number
        counts = [(0, 0)]
        # difference of one_count - zero_count
        diff_count_map = {0: 0}
        for i, num in enumerate(nums):
            if num == 0:
                zero_count += 1
            elif num == 1:
                one_count += 1

            counts.append((zero_count, one_count))
            if one_count - zero_count not in diff_count_map:
                diff_count_map[one_count - zero_count] = i + 1

        # for all count, find a complementary one, and update the max length
        max_length = 0
        for j, cnt in enumerate(counts):
            if cnt[1] - cnt[0] in diff_count_map:
                max_length = max(
                    max_length, j - diff_count_map[cnt[1] - cnt[0]])

        return max_length


from hashlib import md5
from itertools import count
from typing import Iterable

def mining(prefix) -> Iterable[tuple[int, str]]:
    for i in count():
        yield i, md5((prefix + str(i)).encode('ascii')).hexdigest()

line = open('04_input.txt').readline().strip()

for i, hash in mining(line):
    if hash.startswith('00000'):
        print('Star 1:', i)
        break
for i, hash in mining(line):
    if hash.startswith('000000'):
        print('Star 2:', i)
        break


from models.RainbowModel import RainbowModel
import tensorflow as tf
import tensorflow.keras as keras
import utils.settings as settings

class ResNetModel(RainbowModel):

    def _create_model(self):
        n_feature_maps = 64

        window_size = settings.data_dimension_dict["window_size"]
        n_features = settings.data_dimension_dict["n_features"]
        n_classes = settings.data_dimension_dict["n_classes"]

        input_layer = keras.layers.Input((window_size, n_features))
        x = self._preprocessing_layer(input_layer)

        # BLOCK 1

        conv_x = keras.layers.Conv1D(
            filters=n_feature_maps, kernel_size=8, padding="same"
        )(x)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = keras.layers.Activation("relu")(conv_x)

        conv_y = keras.layers.Conv1D(
            filters=n_feature_maps, kernel_size=5, padding="same"
        )(conv_x)
        conv_y = keras.layers.BatchNormalization()(conv_y)
        conv_y = keras.layers.Activation("relu")(conv_y)

        conv_z = keras.layers.Conv1D(
            filters=n_feature_maps, kernel_size=3, padding="same"
        )(conv_y)
        conv_z = keras.layers.BatchNormalization()(conv_z)

        # expand channels for the sum
        shortcut_y = keras.layers.Conv1D(
            filters=n_feature_maps, kernel_size=1, padding="same"
        )(x)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)

        output_block_1 = keras.layers.add([shortcut_y, conv_z])
        output_block_1 = keras.layers.Activation("relu")(output_block_1)

        # BLOCK 2

        conv_x = keras.layers.Conv1D(
            filters=n_feature_maps * 2, kernel_size=8, padding="same"
        )(output_block_1)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = keras.layers.Activation("relu")(conv_x)

        conv_y = keras.layers.Conv1D(
            filters=n_feature_maps * 2, kernel_size=5, padding="same"
        )(conv_x)
        conv_y = keras.layers.BatchNormalization()(conv_y)
        conv_y = keras.layers.Activation("relu")(conv_y)

        conv_z = keras.layers.Conv1D(
            filters=n_feature_maps * 2, kernel_size=3, padding="same"
        )(conv_y)
        conv_z = keras.layers.BatchNormalization()(conv_z)

        # expand channels for the sum
        shortcut_y = keras.layers.Conv1D(
            filters=n_feature_maps * 2, kernel_size=1, padding="same"
        )(output_block_1)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)

        output_block_2 = keras.layers.add([shortcut_y, conv_z])
        output_block_2 = keras.layers.Activation("relu")(output_block_2)

        # BLOCK 3

        conv_x = keras.layers.Conv1D(
            filters=n_feature_maps * 2, kernel_size=8, padding="same"
        )(output_block_2)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = keras.layers.Activation("relu")(conv_x)

        conv_y = keras.layers.Conv1D(
            filters=n_feature_maps * 2, kernel_size=5, padding="same"
        )(conv_x)
        conv_y = keras.layers.BatchNormalization()(conv_y)
        conv_y = keras.layers.Activation("relu")(conv_y)

        conv_z = keras.layers.Conv1D(
            filters=n_feature_maps * 2, kernel_size=3, padding="same"
        )(conv_y)
        conv_z = keras.layers.BatchNormalization()(conv_z)

        # no need to expand channels because they are equal
        shortcut_y = keras.layers.BatchNormalization()(output_block_2)

        output_block_3 = keras.layers.add([shortcut_y, conv_z])
        output_block_3 = keras.layers.Activation("relu")(output_block_3)

        # FINAL

        gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3)

        output_layer = keras.layers.Dense(n_classes, activation="softmax")(
            gap_layer
        )

        model = keras.models.Model(inputs=input_layer, outputs=output_layer)

        model.compile(
            loss=tf.keras.losses.CategoricalCrossentropy(),
            optimizer=keras.optimizers.Adam(),
            metrics=["accuracy"]
            )

        self.model = model
        return model


import configparser
from openpyxl import load_workbook,Workbook
from wordcloud import WordCloud
import matplotlib.pyplot as plt

def find_dimensions(wb,sheets,header_rows = 3,header_columns = 2): # common
    i = 0
    j = 0
    dimensions = {}
    for ws in sheets :
        i = 0
        j = 0
        while wb[ws].cell(row=header_rows+i, column=header_columns-1).value != None :
            i += 1
        while wb[ws].cell(row=header_rows-1, column=j+header_columns).value != None :
            j += 1
        dimensions[ws] = [i,j]
    return dimensions

def read_questions() :
    config = configparser.ConfigParser()
    config.read('config.ini')

    header_rows = int(config['excel_sheet_params']['header_rows'])
    header_columns = int(config['excel_sheet_params']['header_columns'])
    input_file = config['excel_sheet_params']['filename']
    output_file_name = config['excel_sheet_params']['output_xlsx_file']
    questions = []
    wb = load_workbook(filename = input_file)
    sheets = wb.get_sheet_names()
    dimensions = find_dimensions(wb,sheets,header_rows,header_columns)
    max_questions = max(dimensions.keys(), key = lambda k: dimensions[k][1])
    for i in range(header_columns,dimensions[max_questions][1]+header_columns):
        questions.append([])
        for ws in sheets:
            if dimensions[ws][1] > i-header_columns :
                for j in range(header_rows,dimensions[ws][0]+header_rows):
                    questions[i-header_columns].append(wb[ws].cell(row=j, column=i).value)

    return questions,dimensions[max_questions][1],dimensions,sheets

def generate_word_cloud(sol,filename,thread_message,eligible_qts = []) :
    if eligible_qts == [] :
        eligible_qts = [i for i in range(len(sol))]
    for i in range(len(sol)):
        wordcloud = WordCloud().generate(sol[i])
        plt.imshow(wordcloud, interpolation='bilinear')
        plt.axis("off")
        plt.title('Word Cloud of Question '+ str(i+1), fontsize=14, fontweight='bold')
        plt.savefig(filename + str(eligible_qts[i]+1) +".png")
        plt.gcf().clear()


import glob
import logging
import os

from cros.factory.probe.lib import probe_function
from cros.factory.utils.arg_utils import Arg

DEFAULT_FILE_KEY = 'file_raw'

def ReadFile(path, binary_mode=False, skip=0, size=-1):
  logging.debug('Read file: %s', path)
  if not os.path.isfile(path):
    return None
  mode = 'rb' if binary_mode else 'r'
  with open(path, mode) as f:
    f.seek(skip)
    data = f.read(size)
  if not binary_mode:
    ret = data.strip()
  else:
    binary_data = ['0x%02x' % char for char in data]
    ret = ' '.join(binary_data)
  if not ret:
    return None
  return ret

class FileFunction(probe_function.ProbeFunction):
  """Read the content of a file.

  Description
  -----------
  The content of the file is stripped and the empty content is filtered. If the
  ``split_lines`` argument is set, then the content will be splitted by line.
  The file path is allowed unix style to match multiple files.

  Examples
  --------
  Let's say if the file tree looks like:

  - ``/tmp/aaa/x`` contains::

      Hello, Google
      Hello, ChromiumOS

  - ``/tmp/aaa/y`` contains::

      Bye, Everyone

  And the probe statement is::

    {
      "eval": "file:/tmp/aaa/x"
    }

  Then the probed results will be::

    [
      {
        "file_row": "Hello, Google\\nHello, ChromiumOS"
      }
    ]

  If the probe statement is ::

    {
      "eval": {
        "file": {
          "file_path": "/tmp/aaa/*",
          "split_line": true,
          "key": "my_awesome_key"
        }
      }
    }

  , then the probed results will be::

    [
      {
        "my_awesome_key": "Hello, Google"
      },
      {
        "my_awesome_key": "Hello, ChromiumOS"
      },
      {
        "my_awesome_key": "Bye, Everyone"
      }
    ]

  In above example we use ``"split_line": true`` to let this function treat
  each line of the content of a file as different results.  And instead of
  just specifying a real path, we have ``/tmp/aaa/*`` to match both
  ``/tmp/aaa/x`` and ``/tmp/aaa/y``.
  """
  ARGS = [
      Arg('file_path', str, 'The file path of target file.'),
      Arg('key', str, 'The key of the result.',
          default=DEFAULT_FILE_KEY),
      Arg('split_line', bool, 'True to split lines to multiple results.',
          default=False),
  ]

  def Probe(self):
    ret = []
    for path in glob.glob(self.args.file_path):
      if os.path.isfile(path):
        data = ReadFile(path)
        if data is None:
          continue
        contents = data.splitlines() if self.args.split_line else [data]
        ret += [{self.args.key: content.strip()}
                for content in contents if content.strip()]
    return ret


import sys

import Qt.QtWidgets as QtWidgets
import pytest, unittest.mock as mock

import gazu
from tests.fixtures import fixtures
import kitsupublisher.ui_data.table_headers as headers
from kitsupublisher.views.TasksTab import TasksTab
from kitsupublisher.views.MainWindow import MainWindow

def mock_table_functions():
    gazu.user.all_tasks_to_do = mock.MagicMock(return_value=fixtures.tasks)
    gazu.task.all_task_statuses = mock.MagicMock(
        return_value=fixtures.status_names
    )
    gazu.task.all_done_tasks_for_person = mock.MagicMock(
        return_value=fixtures.done_tasks
    )
    gazu.client.get_current_user = mock.MagicMock(return_value=fixtures.user)
    gazu.person.get_organisation = mock.MagicMock(
        return_value=fixtures.organisation
    )
    headers.tab_columns = fixtures.tab_columns

def mock_panel_functions():
    gazu.files.get_all_preview_files_for_task = mock.MagicMock(
        return_value=fixtures.all_preview_files_for_task
    )
    gazu.task.all_comments_for_task = mock.MagicMock(
        return_value=fixtures.all_comments_for_task
    )
    gazu.task.get_last_comment_for_task = mock.MagicMock(
        return_value=fixtures.all_comments_for_task[0]
    )

@pytest.fixture(scope="module", autouse=True)
def before_each_test():
    mock_table_functions()
    app = QtWidgets.QApplication(sys.argv)
    window = MainWindow(app, real_time=False)
    return app, window

def test_creation(before_each_test):
    """
    Test the creation of the table
    """
    _, window = before_each_test
    tasks_table = TasksTab(window, headers.tab_columns)
    header_col_count = tasks_table.columnCount()
    header_row_count = tasks_table.rowCount()
    for row in range(0, header_row_count):
        for col in range(0, header_col_count - 1):
            assert isinstance(
                tasks_table.item(row, col), QtWidgets.QTableWidgetItem
            )

def test_wrong_attribute(before_each_test):
    """
    Test the behaviour of the class when asked for an inexisting task attribute
    """
    _, window = before_each_test
    with pytest.raises(AssertionError):
        tab_columns = {
            "task_attribute_that_does_not_exist": "random_column_name"
        }
        TasksTab(window, tab_columns)

def test_sort(before_each_test):
    """
    Test that the table is well sorted
    """
    _, window = before_each_test
    tab_columns = {
        "project_name": "Project Name",
        "task_type_name": "Task type name",
        "entity_name": "Name",
    }
    tasks_table = TasksTab(window, tab_columns)

    header_row_count = tasks_table.rowCount()
    for row in range(1, header_row_count):
        for col in range(len(tab_columns)):
            cell1 = tasks_table.item(row - 1, col)
            cell2 = tasks_table.item(row, col)
            if cell1 != cell2:
                assert cell1 < cell2

def test_panel_initialization(before_each_test):
    """
    Test the panel initialization
    """
    app, window = before_each_test
    mock_panel_functions()
    tab_columns = {
        "project_name": "Project Name",
        "task_type_name": "Task type name",
        "entity_name": "Name",
    }
    tasks_table = TasksTab(window, tab_columns)
    item = tasks_table.item(0, 0)
    tasks_table.setCurrentItem(item)
    tasks_table.on_click()
    assert hasattr(window, "task_panel")
    assert window.task_panel.task == item.task
    assert window.task_panel.task_panel_vertical_layout.count() == 4


from django.shortcuts import render, redirect
from profiles.forms import ContactForm

def Contact(request):

    user = request.user

    if request.method == 'POST':
        form = ContactForm(request.POST)
        confirm = False

        if form.is_valid():
            instance = form.save(commit = False)
            instance.user = user
            instance.save()
            confirm = True

            return redirect('contact')

        data = {
            'form': form,
            'confirm': confirm
        }

        return render(request, "contact.html", data)

    else:
        form = ContactForm()
        return render(request, 'contact.html', {'form': form})


"""
py311
hdpoorna
"""

# import packages
import os
import imageio
import cv2
from datetime import datetime
from pongWrapper import *
import numpy as np
import tensorflow as tf
from helpers import config
from helpers.dqn_helper import *

# make the env
env = PongWrapper(render_mode="rgb_array", points_per_episode=config.POINTS_PER_EPISODE)

# set constants
config.NUM_ACTIONS = env._ACTION_SPACE_SIZE

# load the model
MODEL_ID = "pong-conv-greedy-2023-07-20-16-53-20"
MODEL_EPISODE = "73"
model_dir_path = os.path.join("saved_models", f"{MODEL_ID}/model-{MODEL_EPISODE}")
policy_model = tf.saved_model.load(model_dir_path)

# initial state
current_state = env.reset()[0]

terminated = False      # goal achieved
truncated = False       # timed out

frames_orig = [env.render()]
frames_mod = [env.render_cv(current_state)]

while not (terminated or truncated):
    qs = policy_model(inputs=[scale_states(current_state)], training=False)
    # print(qs)
    action = tf.argmax(qs[0])
    # print(action)
    obs, reward, terminated, truncated, _ = env.step(action.numpy())

    frames_orig.append(env.render())
    frame_rgb = env.render_cv(obs)
    frames_mod.append(frame_rgb)

    current_state = obs

results_dir = f"results/{MODEL_ID}"
make_dir(results_dir)

now_utc = datetime.utcnow()
now_str = now_utc.strftime("%Y-%m-%d-%H-%M-%S")
orig_path = "{}/{}-orig.gif".format(results_dir, now_str)
mod_path = "{}/{}-mod.gif".format(results_dir, now_str)

imageio.mimsave(orig_path, frames_orig, duration=1000/env.MAX_FPS, loop=0)
imageio.mimsave(mod_path, frames_mod, duration=1000/env.MAX_FPS, loop=0)
print(f"gifs saved to {results_dir}")

cv2.destroyAllWindows()
env.close()