import time
import click

import requests
from sqlalchemy.exc import IntegrityError

from db import sql_engine, Session
from db.models import EtfCategory, IsinCategory, Etf
from db.table_manager import create_table
from scraping.items import EtfItem, string_to_date

class Extraetf():
    """
    This class extracts ETF data from extraetf.com via an undocumented API. As such an API is always subject to change
    we must expect this implementation to work only for a limited amount of time.
    """

    def __init__(self):
        # the category names on extraetf.com
        self.category_types = {'sector_name': 'Sektor', 'land_name': 'Land', 'region_name': 'Region',
                               'asset_class_name': "Asset Klasse", 'strategy_name': 'Strategie',
                               'fund_currency_id': 'Währung',
                               'bond_type_name': 'Anlageart', 'commodity_class_name': 'Rohstoffklasse',
                               'bond_maturity_name': 'Laufzeit',
                               'bond_rating_name': 'Rating'}
        create_table(sql_engine)
        self.session = Session()
        self.cgry_cache = dict()

    def collect_data(self):
        offset = 0
        limit = 200

        while True:
            params = {'offset': offset, 'limit': limit, 'ordering': '-assets_under_management', 'leverage_from': 1,
                      'leverage_to': 1}
            response = requests.get('https://de.extraetf.com/api-v2/search/full/', params=params)
            time.sleep(2)

            data = response.json()
            page = int(offset / limit + 1)
            results = data['results']
            click.echo(f"Extracted etfs from page {page}!")
            self.__parse_page(results)

            if data['next'] is None:
                break

            offset += limit

        self.session.close()

    def __parse_page(self, results):
        """
        Parses and stores a page from extraetf.com and extracts the data from each detail page displayed on that page.
        """

        for result in results:
            detail_params = {'isin': result['isin']}
            detail_response = requests.get('https://de.extraetf.com/api-v2/detail/', detail_params)
            detail_result = detail_response.json()['results'][0]

            item = self.__parse_item(result, detail_result)
            item = self.__process_item(item)
            self.__save_item(item)

            self.save_item_categories(result, detail_result)

            time.sleep(0.25)  # slow down to avoid getting banned :)

    def __parse_item(self, result, detail_result):
        """
        Parses the data of one detail page, i.e. of one ETF
        """
        item = EtfItem()
        for field in item.fields:
            item.setdefault(field, None)

        item['name'] = result['fondname']
        item['isin'] = result['isin']
        item['wkn'] = result['wkn']
        item['ter'] = result['ter']
        click.echo(f"Parsing ETF '{result['isin']}'")

        replication = detail_result['replication_methodology_first_level']
        repl_detail = detail_result['replication_methodology_second_level']
        if repl_detail is not None:
            replication = repl_detail if replication is None else '(' + repl_detail + ')'

        item['replication'] = replication
        item['distribution_policy'] = result['distribution_policy']
        item['fund_size'] = result['assets_under_management']
        item['inception'] = result['launch_date']
        item['fund_currency'] = result['fund_currency_id']
        item['fund_domicile'] = result['fund_domicile']

        item['fund_provider'] = detail_result['provider_name']
        item['legal_structure'] = result['legal_structure']
        item['fund_structure'] = detail_result['fund_structure']
        item['ucits_compliance'] = detail_result['ucits_konform']
        item['administrator'] = detail_result['administrator']
        item['revision_company'] = detail_result['auditor_company']
        item['custodian_bank'] = detail_result['depotbank']
        item['tax_germany'] = detail_result['tax_status_de']
        item['tax_switzerland'] = detail_result['tax_status_ch']
        item['tax_austria'] = detail_result['tax_status_au']
        item['tax_uk'] = detail_result['tax_status_uk']
        item['distribution_frequency'] = detail_result['distribution_interval']

        item['volatility_one_year'] = detail_result['volatility_1_year']
        item['fiscal_year_end_month'] = detail_result['fiscal_year_end_month']
        item['indextype'] = detail_result['index_type']
        item['swap_counterparty'] = detail_result['swap_counterparty']
        item['collateral_manager'] = detail_result['collateral_manager']
        item['securities_lending'] = detail_result['has_securities_lending']
        item['securities_lending_counterparty'] = detail_result['securities_lending_party']

        item['net_assets_currency'] = detail_result['net_assets_currency']
        item['is_accumulating'] = detail_result['is_accumulating']
        item['is_derivative_based'] = detail_result['is_derivative_based']
        item['is_distributing'] = detail_result['is_distributing']
        item['is_etc'] = detail_result['is_etc']
        item['is_etf'] = detail_result['is_etf']
        item['is_hedged'] = detail_result['is_hedged']
        item['hedged_currency'] = detail_result['is_hedged_currency']
        item['is_index_fund'] = detail_result['is_index_fund']
        item['is_leveraged'] = detail_result['is_leveraged']
        item['is_physical_full'] = detail_result['is_physical_full']
        item['is_short'] = detail_result['is_short']
        item['is_socially_responsible_fund'] = detail_result['is_socially_responsible_fund']
        item['is_structured'] = detail_result['is_structured']
        item['is_swap_based_etf'] = detail_result['is_swap_based_etf']
        item['is_synthetic_replication'] = detail_result['is_synthetic_replication']

        return item

    def __process_item(self, item):
        """
        Converts the data of an item into appropriate formats if neccesary
        """
        item['inception'] = string_to_date(item['inception'])

        return item

    def __save_item(self, item: EtfItem):
        etf = item.to_etfitemdb()
        try:
            current_data: Etf = self.session.query(Etf).filter_by(isin=etf.isin).first()
            if current_data is not None:
                click.echo(f"Updating existing ETF {current_data.isin}")
                self.__update_item(current_data, etf)

            else:
                self.session.add(etf)
                self.session.commit()
        except:
            click.echo(f"Could not save data for {etf.isin}!")
            self.session.rollback()
            raise

    def __update_item(self, current_data, etf):
        """
        Overwrites certain ETF data from justetf if a particular ETF is included in both databases
        """

        # we only update attributes that do not exist at justetf.com or are of higher precision at extraetf.com
        # higher precision
        current_data.fund_size = etf.fund_size
        current_data.ter = etf.ter
        # new
        current_data.tax_germany = etf.tax_germany
        current_data.net_assets_currency = etf.net_assets_currency
        current_data.is_accumulating = etf.is_accumulating
        current_data.is_derivative_based = etf.is_derivative_based
        current_data.is_distributing = etf.is_distributing
        current_data.is_etc = etf.is_etc
        current_data.is_etf = etf.is_etf
        current_data.is_hedged = etf.is_hedged
        current_data.hedged_currency = etf.hedged_currency
        current_data.is_index_fund = etf.is_index_fund
        current_data.is_leveraged = etf.is_leveraged
        current_data.is_physical_full = etf.is_physical_full
        current_data.is_short = etf.is_short
        current_data.is_socially_responsible_fund = etf.is_socially_responsible_fund
        current_data.is_structured = etf.is_structured
        current_data.is_swap_based_etf = etf.is_swap_based_etf
        current_data.is_synthetic_replication = etf.is_synthetic_replication
        self.session.commit()

    def save_item_categories(self, result, detail_result):
        """
        Saves the item category for each item
        """
        isin = result['isin']
        categories = [(detail_result[cat_key], cat_type) for cat_key, cat_type in self.category_types.items()]

        try:
            for cat_name, cat_type in categories:
                self.__save_item_category(cat_name, cat_type, isin)

        except:
            click.echo(f"Could not save data for {result['isin']}!")
            self.session.rollback()
            raise

    def __save_item_category(self, cat_name, cat_type, isin):
        """
        Saves the category if it is a previously unknown category and associates a particular ISIN with that category in the database.
        """

        if cat_name is not None and cat_name:
            if cat_name in self.cgry_cache:
                # if category has already been added to db we can use the cache to retrieve its id
                cat_id = self.cgry_cache[cat_name]
            else:
                cat_id = self.__add_category(cat_name, cat_type)
            if cat_id:
                ic = IsinCategory()
                ic.etf_isin = isin
                ic.category_id = cat_id

                try:
                    self.session.add(ic)
                    self.session.commit()
                except IntegrityError:
                    click.echo(f"Could not insert category data for ETF with ISIN {isin} and category {cat_name} "
                                    f"as it already exists!")
                    self.session.rollback()

            else:
                click.echo(f"No ID was found for item with category {cat_name}")

    def __add_category(self, cat_name, cat_type):
        """
        Stores a new category in database
        """
        etf_category = EtfCategory()
        etf_category.name = cat_name
        etf_category.type = cat_type
        self.session.add(etf_category)
        self.session.commit()
        self.session.refresh(etf_category)  # refresh to get auto-incremented cat_id
        cat_id = etf_category.id
        self.cgry_cache[cat_name] = cat_id
        return cat_id


# -*- coding: UTF-8 -*-
from django.conf import settings

import pusher
import json
import logging
logger = logging.getLogger('lawpal.services')

PUSHER_APP_ID = getattr(settings, 'PUSHER_APP_ID', None)
PUSHER_KEY = getattr(settings, 'PUSHER_KEY', None)
PUSHER_SECRET = getattr(settings, 'PUSHER_SECRET', None)

if PUSHER_APP_ID is None:
    raise Exception("You must specify a PUSHER_APP_ID in your local_settings.py")
if PUSHER_KEY is None:
    raise Exception("You must specify a PUSHER_KEY in your local_settings.py")
if PUSHER_SECRET is None:
    raise Exception("You must specify a PUSHER_SECRET in your local_settings.py")

class PusherPublisherService(object):
    """ Service to push data out to channels on pusher.com
    so the js lib can pick them up """
    channels = None
    event = None
    data = {}

    def __init__(self, channel, event, **kwargs):
        # is we pass in a tuple or list (iterable)
        # otherwise make it a list
        self.channels =  channel if hasattr(channel, '__iter__') else [channel]
        self.event = event

        # append our core data
        self.data.update(kwargs)

        logger.debug('Initialized PusherPublisherService with {data}'.format(data=json.dumps(self.data)))

        if not settings.IS_TESTING:
            self.pusher = pusher.Pusher(app_id=PUSHER_APP_ID, key=PUSHER_KEY, secret=PUSHER_SECRET)

    def process(self, **kwargs):
        if not settings.IS_TESTING:

            if 'event' not in kwargs:
                kwargs.update({'event': self.event})

            self.data.update(kwargs)

            for channel in self.channels:
                logger.debug('Sending pusher event on #{channel}'.format(channel=channel))

                self.data.update({
                    'channel': channel,
                })

                self.pusher[channel].trigger(self.event, self.data)


import json
import logging
import os.path
import re
import shutil
import ssl
import subprocess
import time
import urllib2

import settings

_BOARD = open('/etc/board').read().strip()
_REPO = ('ccrisan', 'motionPie')
_DOWNLOAD_URL = 'https://github.com/{owner}/{repo}/releases/download/%(version)s/motionpie-%(board)s-%(version)s.img.gz'.format(
        owner=_REPO[0], repo=_REPO[1])
_LIST_VERSIONS_URL = 'https://api.github.com/repos/{owner}/{repo}/releases'.format(
        owner=_REPO[0], repo=_REPO[1])
_DOWNLOAD_DIR = '/data/.firmware_update'
_DOWNLOAD_FILE_NAME = os.path.join(_DOWNLOAD_DIR, 'firmware.gz')

# versions

def get_version():
    import motioneye

    return motioneye.VERSION

def get_all_versions():
    url = _LIST_VERSIONS_URL
    url += '?_=' + str(int(time.time())) # prevents caching

    try:
        logging.debug('board is %s' % _BOARD)
        logging.debug('fetching %s...' % url)

        context = ssl._create_unverified_context()

        response = urllib2.urlopen(url, timeout=settings.REMOTE_REQUEST_TIMEOUT, context=context)
        releases = json.load(response)

        versions = []
        for release in releases:
            if release.get('prerelease') and not os.path.exists('/data/etc/prereleases'):
                continue

            for asset in release.get('assets', []):
                if not re.match('^motionpie-%s-\d{8}\.img.gz$' % _BOARD, asset['name']):
                    continue

            versions.append(release['name'])

        logging.debug('available versions: %(versions)s' % {'versions': ', '.join(versions)})

        return sorted(versions)

    except Exception as e:
        logging.error('could not get versions: %s' % e, exc_info=True)

    return []

def compare_versions(version1, version2):
    version1 = [int(n) for n in version1.split('.')]
    version2 = [int(n) for n in version2.split('.')]

    len1 = len(version1)
    len2 = len(version2)
    length = min(len1, len2)
    for i in xrange(length):
        p1 = version1[i]
        p2 = version2[i]

        if p1 < p2:
            return -1

        elif p1 > p2:
            return 1

    if len1 < len2:
        return -1

    elif len1 > len2:
        return 1

    else:
        return 0

# updating

def download(version):
    url = _DOWNLOAD_URL % {'version': version, 'board': _BOARD}

    try:
        logging.info('downloading %s...' % url)

        shutil.rmtree(_DOWNLOAD_DIR, ignore_errors=True)
        os.makedirs(_DOWNLOAD_DIR)
        subprocess.check_call(['/usr/bin/wget', url, '--no-check-certificate', '-O', _DOWNLOAD_FILE_NAME])

    except Exception as e:
        logging.error('could not download update: %s' % e)

        raise

    try:
        logging.info('decompressing %s...' % _DOWNLOAD_FILE_NAME)

        subprocess.check_call(['/bin/gunzip', _DOWNLOAD_FILE_NAME])

    except Exception as e:
        logging.error('could not decompress archive: %s' % e)

        raise

    extracted_file_name = _DOWNLOAD_FILE_NAME.replace('.gz', '')

    try:
        logging.info('reading partiton table...')

        output = subprocess.check_output(['/sbin/fdisk', '-l', extracted_file_name])
        lines = [l.strip().replace('*', ' ') for l in output.split('\n') if l.startswith(extracted_file_name)]
        boot_info = lines[0].split()
        root_info = lines[1].split()

        boot_start, boot_end = int(boot_info[1]), int(boot_info[2])
        root_start, root_end = int(root_info[1]), int(root_info[2])

    except Exception as e:
        logging.error('could not read partition table: %s' % e)

        raise

    try:
        logging.info('extracting boot.img...')

        subprocess.check_call(['/bin/dd', 'if=' + extracted_file_name, 'of=' + os.path.join(_DOWNLOAD_DIR, 'boot.img'),
                'bs=2048', 'skip=' + str(boot_start / 4), 'count=' + str((boot_end - boot_start + 1) / 4)])

    except Exception as e:
        logging.error('could not extract boot.img: %s' % e)

        raise

    try:
        logging.info('extracting root.img...')

        subprocess.check_call(['/bin/dd', 'if=' + extracted_file_name, 'of=' + os.path.join(_DOWNLOAD_DIR, 'root.img'),
                'bs=2048', 'skip=' + str(root_start / 4), 'count=' + str((root_end - root_start + 1) / 4)])

    except Exception as e:
        logging.error('could not extract root.img: %s' % e)

        raise

def perform_update(version):
    logging.info('updating to version %(version)s...' % {'version': version})

    logging.info('killing motioneye init script...')
    os.system('kill $(pidof S95motioneye)')

    logging.info('stopping netwatch init script...')
    os.system('/etc/init.d/S41netwatch stop')

    download(version)

    logging.info('backing up /boot/config.txt')
    if os.system('/bin/cp /boot/config.txt /tmp/config.txt'):
        logging.error('failed to backup /boot/config.txt')

        raise Exception('failed to backup /boot/config.txt')

    logging.info('unmounting boot partition...')
    if os.system('/bin/umount /boot'):
        logging.error('failed to unmount boot partition')

        raise Exception('failed to unmount boot partition')

    try:
        logging.info('installing boot image...')
        boot_img = os.path.join(_DOWNLOAD_DIR, 'boot.img')

        subprocess.check_call(['/bin/dd', 'if=' + boot_img, 'of=/dev/mmcblk0p1', 'bs=1M'])

    except Exception as e:
        logging.error('could not install boot image: %s' % e)

        raise

    logging.info('mounting boot partition read-write...')
    if os.system('/bin/mount -o rw /dev/mmcblk0p1 /boot'):
        logging.error('failed to mount boot partition')

        raise Exception('failed to mount boot partition')

    logging.info('restoring up /boot/config.txt')
    if os.system('/bin/cp /tmp/config.txt /boot/config.txt'):
        logging.error('failed to restore /boot/config.txt')

        raise Exception('failed to restore /boot/config.txt')

    logging.info('preparing to boot in fwupdate mode...')
    try:
        config_lines = [c.strip() for c in open('/boot/config.txt', 'r').readlines() if c.strip()]

    except Exception as e:
        logging.error('failed to read /boot/config.txt: %s' % e, exc_info=True)

        raise

    config_lines.append('initramfs fwupdater.gz')

    try:
        with open('/boot/config.txt', 'w') as f:
            for line in config_lines:
                f.write(line + '\n')

    except Exception as e:
        logging.error('failed to write /boot/config.txt: %s' % e, exc_info=True)

        raise

    logging.info('rebooting...')

    if os.system('/sbin/reboot'):
        logging.error('failed to reboot')
        logging.info('hard rebooting...')
        open('/proc/sysrq-trigger', 'w').write('b') # reboot


import sys, pygame

pygame.init()
#muestro ventana
size =800, 600
screen= pygame.display.set_mode(size)

pygame.display.set_caption("Juego Ball")
#Variables
width, height = 800, 600
speed = [1, 1]
white = 255, 255, 255

#creA UNA IMAGEN Y OBTENGO SU RECTANGULO
ball = pygame.image.load("ball1.png")
ballrect = ball.get_rect()
#CREO UN OBJETO IMAGEN DE BATE Y OBTENGO SU RECTANGULO
bate = pygame.image.load("bate2.jpg")
baterect = bate.get_rect()
#bate en centro de pantalla
baterect.move_ip(400, 260)
#comenzamos bucle de juego
run=True
while run:
    pygame.time.delay(0)
    #capturamos eventos que se han producido
    for event in pygame.event.get():
        #si el vento es saalir de la ventana, terminamos
        if event.type == pygame.QUIT: run= False

#Compruebo si se ha pulsado alguna tecla
    keys = pygame.key.get_pressed()
    if keys[pygame.K_UP]:
        baterect=baterect.move(0, -1)
    if keys[pygame.K_DOWN]:
        baterect=baterect.move(0, 1)

 #Compryebo si hay colision
    if baterect.colliderect(ballrect):
         speed[0] = - speed[0]

        #MUEVO  LA PELOTA
    ballrect = ballrect.move(speed)
    #compreubo si la pelota llega a los limites de la ventana
    if ballrect.left < 0 or ballrect.right > width:
        speed[0] = -speed[0]
    if ballrect.top < 0 or ballrect.bottom > height:
        speed[1] = -speed[1]
        #pinto el fondo de blanco, dibujo pelota y actualizo
    screen.fill(white)
    screen.blit(ball, ballrect)
    pygame.display.flip()

    #salgo de pygame
pygame.quit()


from __future__ import print_function

import json
import os
import boto3
import collections

print('Loading function')
ASGInstanceCount = collections.namedtuple('ASGInstanceCount', ['min', 'desired', 'max', 'launched'])

def lambda_handler(event, context):
    # print("Received event: " + json.dumps(event, indent=2))
    message = json.loads(event['Records'][0]['Sns']['Message'])
    # print("From SNS: " + event['Records'][0]['Sns']['Message'])
    # print('AWS_STACK_ID: ' + os.environ['AWS_STACK_ID'])
    if message['Event']:
        print('EVENT: ', message['Event'])
        return eval(get_handler(message['Event']))(message)
    else:
        return do_nothing(message)

    return message

def get_handler(Event):
    return {
        'autoscaling:EC2_INSTANCE_LAUNCH': 'on_instance_launch',
        'autoscaling:EC2_INSTANCE_LAUNCH_ERROR': 'on_instance_launch_error',
        'autoscaling:EC2_INSTANCE_TERMINATE': 'on_instance_terminate',
        'autoscaling:EC2_INSTANCE_TERMINATE_ERROR': 'on_instance_terminate_error',
        'autoscaling:TEST_NOTIFICATION' : 'do_nothing'
    }[Event]

def do_nothing(message):
    print('do_nothing')
    print("Unknown Event. Received message: " + json.dumps(message, indent=2))
    return

def send_asg_success(status, asg, asg_instance_counts):
     sqs_url = os.environ['AWS_DL_MASTER_SQS_URL']
     print("sqs_url: ", sqs_url)
     sqs_con = boto3.client('sqs')
     msg_dict = asg_instance_counts._asdict()
     msg_dict['status'] = status.lower()
     msg_dict['asg'] = asg
     msg_dict['event'] = 'asg-setup'

     print('sending message to sqs:', json.dumps(msg_dict))
     sqs_con.send_message(QueueUrl=sqs_url, MessageBody=json.dumps(msg_dict))
     return

'''
    get various instance counts associated with the asg
'''
def get_instance_count(autoscaling_group_name):
    print('get_instance_count')

    autoscale_con = boto3.client('autoscaling')

    asg = autoscale_con.describe_auto_scaling_groups(AutoScalingGroupNames=[autoscaling_group_name])['AutoScalingGroups'][0]
    num_instances_healthy = 0

#   TODO: check if pagination needs to be handled for asg.instances
    for each_instance in asg['Instances']:
        '''
        we will only consider instances that are inService or are in Pending state
        http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html
        since this lambda function is expected to run during stack creation, we'll ignore the
        case where the User could go and Stop the Instance and can move the instance state to 'Pending'
        '''
        if each_instance['LifecycleState'] == 'InService' and each_instance['HealthStatus'] == 'Healthy':
            num_instances_healthy += 1
        elif each_instance['LifecycleState'] == 'Pending' and each_instance['HealthStatus'] == 'Healthy':
            num_instances_healthy += 1
        else:
            continue

    asg_instance_counts = ASGInstanceCount(min=asg['MinSize'], max=asg['MaxSize'], desired=asg['DesiredCapacity'], launched=num_instances_healthy)
    print(asg_instance_counts._asdict())
    return asg_instance_counts

def on_instance_launch(message):
    print('on_instance_launch')

    autoscaling_group_name = message['AutoScalingGroupName']
    availability_zone = message['Details']['Availability Zone']
    start_time = message['StartTime']
    instance_id = message['EC2InstanceId']
    request_id = message['RequestId']

    if autoscaling_group_name and 'WorkerAutoScalingGroup' in autoscaling_group_name:
        autoscaling_group = 'WorkerAutoScalingGroup'
    elif autoscaling_group_name and 'MasterAutoScalingGroup' in autoscaling_group_name:
        autoscaling_group = 'MasterAutoScalingGroup'
    else:
        print('Unknown AutoScaling group,message :',message)
        return

    print('AutoScalingGroupName: ', autoscaling_group_name, ', EC2InstanceId: ', instance_id, \
    ', Availability Zone: ', availability_zone, ', Instance StartTime: ', start_time, ', RequestId: ',request_id)

    logical_resource_id = None
    asg_instance_counts = get_instance_count(autoscaling_group_name)

    if asg_instance_counts.launched == asg_instance_counts.desired:
        print('Launched desired number of instances:', asg_instance_counts.launched)
        send_asg_success('SUCCESS', autoscaling_group_name, asg_instance_counts)

        if autoscaling_group is 'MasterAutoScalingGroup':
            cfn_con = boto3.client('cloudformation')
            print('Sending cfn-signal SUCCESS to:', autoscaling_group_name, 'with instance Id: ', instance_id)
            try:
                cfn_con.signal_resource(StackName=os.environ['AWS_DL_STACK_ID'], LogicalResourceId=autoscaling_group, \
                    UniqueId=instance_id,Status='SUCCESS')
            except Exception as e:
                print('exception sending cfn-signal: ', e.message)
        else:
            autoscale_con = boto3.client('autoscaling')
            print('Suspending ReplaceUnhealthy processes for the asg: ', autoscaling_group_name)
            autoscale_con.suspend_processes(AutoScalingGroupName=autoscaling_group_name, ScalingProcesses=['ReplaceUnhealthy'])

    return

'''
suspend autoscaling policy
change desired capacity
send success message to sqs

'''
def on_instance_launch_error(message):
    print('on_instance_launch_error')

    autoscaling_group_name = message['AutoScalingGroupName']
    availability_zone = message['Details']['Availability Zone']
    start_time = message['StartTime']
    instance_id = message['EC2InstanceId']
    request_id = message['RequestId']

    print('AutoScalingGroupName: ', autoscaling_group_name, ', EC2InstanceId: ', instance_id, \
    ', Availability Zone: ', availability_zone, ', Instance StartTime: ', start_time, ', RequestId: ',request_id)
    print('StatusCode: ', message['StatusCode'],  'StatusMessage: ', message['StatusMessage'])

    autoscale_con = boto3.client('autoscaling')
    asg_instance_counts = get_instance_count(autoscaling_group_name)

    '''
    change desired capacity and suspend processes only if we have atleast the min_size requested
    '''
    if asg_instance_counts.launched >= asg_instance_counts.min:
        print('setting desired capacity of asg: ', autoscaling_group_name, ' to number of Healthy instances: ', asg_instance_counts.launched)
        autoscale_con.set_desired_capacity(AutoScalingGroupName=autoscaling_group_name, DesiredCapacity=asg_instance_counts.launched)
        print('Suspending ReplaceUnhealthy processes for the asg: ', autoscaling_group_name)
        autoscale_con.suspend_processes(AutoScalingGroupName=autoscaling_group_name, ScalingProcesses=['ReplaceUnhealthy'])
        print('sending worker asg setup message complete to sqs')
        send_asg_success('SUCCESS', autoscaling_group_name, asg_instance_counts)

    return

'''
'''
def on_instance_terminate(message):
    print('on_instance_terminate')

    autoscaling_group_name = message['AutoScalingGroupName']
    availability_zone = message['Details']['Availability Zone']
    start_time = message['StartTime']
    instance_id = message['EC2InstanceId']
    request_id = message['RequestId']

    print('AutoScalingGroupName: ', autoscaling_group_name, ', EC2InstanceId: ', instance_id, \
    ', Availability Zone: ', availability_zone, ', Instance StartTime: ', start_time, ', RequestId: ',request_id)

    return

def on_instance_terminate_error():
    print('on_instance_terminate_error')

    autoscaling_group_name = message['AutoScalingGroupName']
    availability_zone = message['Details']['Availability Zone']
    start_time = message['StartTime']
    instance_id = message['EC2InstanceId']
    request_id = message['RequestId']

    print('AutoScalingGroupName: ', autoscaling_group_name, ', EC2InstanceId: ', instance_id, \
    ', Availability Zone: ', availability_zone, ', Instance StartTime: ', start_time, ', RequestId: ',request_id)

    return


import pytest
import os
import glob

from camoco.Config import cf
from camoco import Tools as tools

import camoco as co
import pandas as pd

''' -------------------------------------------------------------------------
            'Test' Fixtures
'''

@pytest.fixture(scope='module')
def testRefGen(Zm5bFGS):
    # This was a mistake
    return Zm5bFGS

@pytest.fixture(scope='module')
def testGWAS(ZmWallace):
    return ZmWallace

@pytest.fixture(scope='module')
def testCOB(ZmRNASeqTissueAtlas):
    return ZmRNASeqTissueAtlas

''' -------------------------------------------------------------------------
            RefGen Fixtures
'''

@pytest.fixture(scope="module")
def Zm5bFGS():
    if cf.test.force.RefGen:
        tools.del_dataset('RefGen', 'Zm5bFGS', force=True)
    if not tools.available_datasets('RefGen', 'Zm5bFGS'):
        # We have to build it
        gff = os.path.expanduser(
            os.path.join(
                cf.options.testdir,
                'raw', 'RefGen', 'ZmB73_5b_FGS.gff.gz'
            )
        )
        # This is stupid and necessary because pytables wont let me open
        # more than one table
        co.RefGen.from_gff(
            gff, 'Zm5bFGS', 'Maize 5b Filtered Gene Set'
        )
    return co.RefGen('Zm5bFGS')

@pytest.fixture(scope="module")
def Zm5bFGS_duplicate(Zm5bFGS):
    # Build a refgen over an already built refgen
    # We have to build it
    gff = os.path.expanduser(
        os.path.join(
            cf.options.testdir,
            'raw', 'RefGen', 'ZmB73_5b_FGS.gff.gz'
        )
    )
    # This is stupid and necessary because pytables wont let me open
    # more than one table
    co.RefGen.from_gff(
        gff, 'Zm5bFGS', 'Maize 5b Filtered Gene Set'
    )
    return co.RefGen('Zm5bFGS')

''' -------------------------------------------------------------------------
            COB Fixtures
'''

@pytest.fixture(scope="module")
def ZmRNASeqTissueAtlas(Zm5bFGS):
    if cf.test.force.COB:
        print('Rebuilding ZmRNASeqTissueAtlas')
        tools.del_dataset('COB', 'ZmRNASeqTissueAtlas', force=True)
        tools.del_dataset('Expr', 'ZmRNASeqTissueAtlas', force=True)
    if not tools.available_datasets('Expr', 'ZmRNASeqTissueAtlas'):
        # Build it
        return co.COB.from_table(
            os.path.join(cf.options.testdir,
                'raw', 'Expr', 'RNASEQ',
                'MaizeRNASeqTissue.tsv.bz2',
            ),
            'ZmRNASeqTissueAtlas',
            'Maize RNASeq Tissue Atlas Network, Sekhon 2013, PLoS ONE',
            Zm5bFGS,
            rawtype='RNASEQ',
            max_gene_missing_data=0.3,
            max_accession_missing_data=0.08,
            min_single_sample_expr=1,
            min_expr=0.001,
            quantile=False,
            max_val=300,
            dry_run=False
        )
    else:
        return co.COB('ZmRNASeqTissueAtlas')

@pytest.fixture(scope="module")
def ZmRoot(Zm5bFGS):
    if cf.test.force.COB:
        tools.del_dataset('Expr','ZmRoot',force=True)
    if not tools.available_datasets('Expr','ZmRoot'):
        return co.COB.from_table(
            os.path.join(
                cf.options.testdir,
                'raw','Expr',
                'RNASEQ','ROOTFPKM.tsv.gz'
            ),
            'ZmRoot',
            'Maize Root Network',
            Zm5bFGS,
            rawtype='RNASEQ',
            max_gene_missing_data=0.3,
            max_accession_missing_data=0.08,
            min_single_sample_expr=1,
            min_expr=0.001,
            quantile=False,
            max_val=300
        )
    else:
        return co.COB('ZmRoot')

@pytest.fixture(scope="module")
def ZmSAM(Zm5bFGS):
    if cf.test.force.COB:
        tools.del_dataset('Expr','ZmSAM',force=True)
    if not tools.available_datasets('Expr','ZmSAM'):
        return co.COB.from_table(
            os.path.join(
                cf.options.testdir,
                'raw','Expr','RNASEQ',
                'TranscriptomeProfiling_B73_Atlas_SAM_FGS_LiLin_20140316.txt.gz'
            ),
            'ZmSAM',
            'Maize Root Network',
            Zm5bFGS,
            rawtype='RNASEQ',
            max_gene_missing_data=0.4,
            min_expr=0.1,
            quantile=False,
            dry_run=False,
            max_val=250
        )
    else:
        return co.COB('ZmSAM')

@pytest.fixture(scope="module")
def ZmSAM2(Zm5bFGS):
    if cf.test.force.COB:
        tools.del_dataset('Expr','ZmSAM2',force=True)
    if not tools.available_datasets('Expr','ZmSAM2'):
        return co.COB.from_table(
            os.path.join(
                cf.options.testdir,
                'raw','Expr','RNASEQ',
                'TranscriptomeProfiling_B73_Atlas_SAM_FGS_LiLin_20140316.txt.gz'
            ),
            'ZmSAM2',
            'Maize Root Network, but loose',
            Zm5bFGS,
            rawtype='RNASEQ',
            max_gene_missing_data=0.4,
            min_single_sample_expr=1,
            min_expr=0.01,
            quantile=False,
            dry_run=False,
            max_val=250
        )
    else:
        return co.COB('ZmSAM2')

@pytest.fixture(scope="module")
def ZmPAN2(Zm5bFGS):
    if cf.test.force.COB:
        tools.del_dataset('Expr','ZmPAN2',force=True)
    if not tools.available_datasets('Expr','ZmPAN2'):
        return co.COB.from_table(
            os.path.join(
                cf.options.testdir,
                'raw','Expr','RNASEQ',
                'PANGenomeFPKM.txt.gz'
            ),
            'ZmPAN2',
            'Maize Root Network but extra loose',
            Zm5bFGS,
            rawtype='RNASEQ',
            max_gene_missing_data=0.4,
            min_single_sample_expr=1,
            min_expr=0.01,
            quantile=False,
            dry_run=False,
            sep=',',
            max_val=300
        )
    else:
        return co.COB('ZmPAN2')

@pytest.fixture(scope="module")
def ZmPAN(Zm5bFGS):
    if cf.test.force.COB:
        tools.del_dataset('Expr','ZmPAN',force=True)
    if not tools.available_datasets('Expr','ZmPAN'):
        return co.COB.from_table(
            os.path.join(
                cf.options.testdir,
                'raw','Expr','RNASEQ',
                'PANGenomeFPKM.txt.gz'
            ),
            'ZmPAN',
            'Maize Root Network',
            Zm5bFGS,
            rawtype='RNASEQ',
            max_gene_missing_data=0.4,
            min_expr=1,
            quantile=False,
            dry_run=False,
            sep=',',
            max_val=300
        )
    else:
        return co.COB('ZmPAN')

''' -------------------------------------------------------------------------
            GWAS Fixtures
'''
@pytest.fixture(scope='module')
def testGWAS(testRefGen):
    if cf.test.force.Ontology:
        tools.del_dataset('GWAS','testGWAS',force=True)
    df = pd.DataFrame({
        'Trait' : ['a','a','b','b'],
        'CHR' : ['chr1','chr2','chr3','chr4'],
        'POS' : [100,200,300,400],
        'Start' : [100,200,300,400],
        'End' : [1000,20000,3000,4000],
        'id' : ['snp1','snp2','snp3','snp4'],
        'pval' : [0.05,0.05,0.01,0.01]
    })
    gwas = co.GWAS.from_DataFrame(
        df,
        'testGWAS',
        'Test GWAS Dataset',
        testRefGen,
        chr_col='CHR',
        pos_col='POS',
        id_col='id',
        term_col='Trait'
    )
    return gwas

@pytest.fixture(scope='module')
def ZmWallace(Zm5bFGS):
    if cf.test.force.Ontology:
        tools.del_dataset('GWAS','ZmWallace',force=True)
    if not tools.available_datasets('GWAS','ZmWallace'):
        # Grab path the csv
        csv = os.path.join(
            cf.options.testdir,
            'raw','GWAS','WallacePLoSGenet',
            'Wallace_etal_2014_PLoSGenet_GWAS_hits-150112.txt.gz'
        )
        # Define our reference geneome
        df = pd.DataFrame.from_csv(csv,index_col=None,sep='\t')
        # Import class from dataframe
        gwas  = co.GWAS.from_DataFrame(
            df, 'ZmWallace', 'Wallace PLoS ONE Dataset.',
            Zm5bFGS,
            term_col='trait', chr_col='chr', pos_col='pos'
        )
        return gwas
    else:
        return co.GWAS('ZmWallace')

@pytest.fixture(scope="module")
def ZmIonome(Zm5bFGS):
        # Delete the old dataset
    if cf.test.force.Ontology:
        tools.del_dataset('GWAS','ZmIonome',force=True)
    if not tools.available_datasets('GWAS','ZmIonome'):
        # Grab path the csv
        csv = os.path.join(
            cf.options.testdir,
            'raw','GWAS','Ionome',
            'sigGWASsnpsCombinedIterations.longhorn.allLoc.csv.gz'
        )
        # Define our reference geneome
        df = pd.DataFrame.from_csv(csv,index_col=None)
        # Import class from dataframe
        IONS  = co.GWAS.from_DataFrame(
            df,'ZmIonome','Maize Ionome',
            Zm5bFGS,
            term_col='el',chr_col='chr',pos_col='pos'
        )
        # Get rid of pesky Cobalt
        IONS.del_term('Co59')
        # I guess we need a test in here too
        return IONS
    else:
        return co.GWAS('ZmIonome')

'''----------------------------------------------------------------------------
    GOnt Fixtures
----------------------------------------------------------------------------'''

@pytest.fixture(scope='module')
def testGO(Zm5bFGS):
    if cf.test.force.Ontology:
        tools.del_dataset('GOnt','TestGO',force=True)
    if not tools.available_datasets('GOnt','TestGO'):
        obo = os.path.join(
            cf.options.testdir,
            'raw','GOnt','go.test.obo'
        )
        gene_map_file = os.path.join(
            cf.options.testdir,
            'raw','GOnt','go.test.tsv'
        )
        return co.GOnt.from_obo(
           obo, gene_map_file, 'TestGO',
           'Test GO', Zm5bFGS
        )
    else:
        return co.GOnt('testGO')

@pytest.fixture(scope="module")
def testZmGO(Zm5bFGS):
    if cf.test.force.Ontology:
        tools.del_dataset('GOnt','ZmGO',force=True)
    if not tools.available_datasets('GOnt','ZmGO'):
        obo = os.path.join(
            cf.options.testdir,
            'raw','GOnt','go.obo.gz'
        )
        gene_map_file = os.path.join(
            cf.options.testdir,
            'raw','GOnt','zm_go.tsv.gz'
        )
        return co.GOnt.from_obo(
           obo, gene_map_file, 'ZmGO',
           'Maize Gene Ontology', Zm5bFGS
        )
    else:
        return co.GOnt('ZmGO')


from __future__ import absolute_import, division, print_function, unicode_literals
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from IPython.display import clear_output
from six.moves import urllib
import tensorflow as tf
import tensorflow.compat.v2.feature_column as fc

#Some basic matplot functions
""" x = [1, 2, 2.5, 3, 4]
y = [1, 4, 7, 9, 15]
plt.plot(x, y, 'ro')
plt.axis([0, 6, 0, 20])
plt.plot(np.unique(x), np.poly1d(np.polyfit(x, y, 1))(np.unique(x)))
plt.xlabel('Years')
plt.ylabel('Price')
plt.show()
 """

dftrain = pd.read_csv('/Users/youssefhemimy/Development /Python /TF-CoreAlgos/train.csv') # training data
dfeval = pd.read_csv('/Users/youssefhemimy/Development /Python /TF-CoreAlgos/eval.csv') # testing data
print(dftrain.head())
y_train = dftrain.pop('survived')
y_eval = dfeval.pop('survived')

dftrain.describe() #statistic details about the data

print("data shape is", dftrain.shape)  # shows the shape(rows, columns)

#dftrain.age.hist(bins=20) # plot a histogram

#dftrain.sex.value_counts().plot(kind = 'barh')

print("evaldata shape is" , dfeval.shape)

#Create categorical columns and numerical columns

CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',
                        'embark_town', 'alone']

NUMERICAL_COLUMNS = ['age','fare']

feature_columns = []

for feature_name in CATEGORICAL_COLUMNS:
    #gets a list of all unique values from give feature column
    vocalbulary = dftrain[feature_name].unique()
    feature_columns.append(tf.feature_column.categorical_column_with_vocabulary_list(feature_name, vocalbulary))

for feature_name in NUMERICAL_COLUMNS:
    feature_columns.append(tf.feature_column.numeric_column(feature_name, dtype=tf.float32))

print(feature_columns)

#https://www.tensorflow.org/tutorials/estimator/linear  DOC
#create input function to convert our current pandas dataframe into tf.data.Dataset object
def make_input_fn(data_df, label_df, num_epochs=10, shuffle=True, batch_size=32):
  def input_function():  # inner function, this will be returned
    ds = tf.data.Dataset.from_tensor_slices((dict(data_df), label_df))  # create tf.data.Dataset object with data and its label
    if shuffle:
      ds = ds.shuffle(1000)  # randomize order of data
    ds = ds.batch(batch_size).repeat(num_epochs)  # split dataset into batches of 32 and repeat process for number of epochs
    return ds  # return a batch of the dataset
  return input_function  # return a function object for use

train_input_fn = make_input_fn(dftrain, y_train)  # here we will call the input_function that was returned to us to get a dataset object we can feed to the model
eval_input_fn = make_input_fn(dfeval, y_eval, num_epochs=1, shuffle=False)

#Creating the model
linear_est = tf.estimator.LinearClassifier(feature_columns=feature_columns)

linear_est.train(train_input_fn) #train
#result = linear_est.evaluate(eval_input_fn) #get model metrics/stats by testing on testing data
result = list(linear_est.predict(eval_input_fn))
clear_output()
#print("Accuracy is", result[0]) # the result variable is stats about the model


from PyQt4 import QtGui, QtCore
import xml.etree.ElementTree as ET
import pandas
import sys
import os
import random

class MirGui(QtGui.QWidget):
    def __init__(self, parent=None):
        super(MirGui, self).__init__(parent)
        self.mz_file = None
        self.tolerance = None
        self.output = None
        self.output_filename = None
        self.init_gui()

    def init_gui(self):
        self.mz_file = QtGui.QPushButton('Select m/z list .csv file...')
        self.tolerance = QtGui.QLineEdit('Tolerance (+/- Da)')
        self.output = QtGui.QPushButton('Select output directory...')
        self.output_filename = QtGui.QLineEdit('Output Filename')
        generate = QtGui.QPushButton('Generate Filter List')

        grid = QtGui.QGridLayout()
        grid.setSpacing(10)
        grid.addWidget(self.mz_file, 1, 0)
        grid.addWidget(self.tolerance, 2, 0)
        grid.addWidget(self.output, 3, 0)
        grid.addWidget(self.output_filename, 4, 0)
        grid.addWidget(generate, 5, 0)
        self.setLayout(grid)

        self.center()
        self.setWindowTitle('Bruker flexImaging Filter List Generator')
        self.show()

        QtCore.QObject.connect(self.mz_file, QtCore.SIGNAL('clicked()'), self.get_csv_file)
        QtCore.QObject.connect(self.output, QtCore.SIGNAL('clicked()'), self.get_output_directory)
        QtCore.QObject.connect(generate, QtCore.SIGNAL('clicked()'), self.generate_filter_list)

    def center(self):
        qr = self.frameGeometry()
        cp = QtGui.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def get_csv_file(self):
        self.mz_file.setText(str(QtGui.QFileDialog.getOpenFileName(self, 'Open File', 'C:\\')).replace('/', '\\'))

    def get_output_directory(self):
        self.output.setText(str(QtGui.QFileDialog.getExistingDirectory(self, 'Select Directory',
                                                                       'C:\\')).replace('/', '\\'))

    def write_mir(self, mz_list, tolerance, output):
        mir_object = ET.Element('ImagingResults', {'flexImagingVersion': '4.1.116.257', 'last_modified': ''})
        for mz in mz_list:
            attrib_dict = {'Type': 'PkFilter',
                           'Name': str(mz),
                           'Color': '#' + '%06x' % random.randint(0, 0xFFFFFF),
                           'Show': '0',
                           'MinIntensity': '0',
                           'IntensityThreshold': '100',
                           'AbsIntens': '0',
                           'LogScale': '0',
                           'MinMass': str(mz - tolerance),
                           'MaxMass': str(mz + tolerance),
                           'Integrate': '0',
                           'UsePercentile': '0',
                           'IntensityPercentile': '0.95',
                           'FindMass': '0',
                           'RelMass': '0'}
            result_object = ET.SubElement(mir_object, 'Result', attrib_dict)
            mir_object.append(result_object)
        mir_tree = ET.ElementTree(mir_object)
        mir_tree.write(os.path.join(output, str(self.output_filename.text())) + '.mir', method='xml')

    def generate_filter_list(self):
        mz_list = pandas.read_csv(str(self.mz_file.text()), usecols=['mz'])['mz'].values.tolist()
        self.write_mir(mz_list, float(self.tolerance.text()), str(self.output.text()))

def main():
    app = QtGui.QApplication(sys.argv)
    gui = MirGui()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()


"""

echelle1d.py

A generic set of code to process echelle 1d spectra that are stored as a
collection of 1d spectra.  This code is used by some of the ESI and NIRES
code that are in the keckcode repository

"""

from numpy import arange
from matplotlib import pyplot as plt
from astropy.io import fits as pf
from astropy.table import Table
from specim import specfuncs as ss

# ---------------------------------------------------------------------------

class Ech1d(list):
    """
    A generic set of code to process echelle 1d spectra that are stored as a
    collection of 1d spectra.  This code is used by some of the ESI and NIRES
    code that are in the keckcode repository
    """

    def __init__(self, inspec, informat='text', ordinfo=None, summary=True,
                 verbose=True):
        """

        Load the data.  This can be accomplished in one of three ways:
          1. by providing a list of Spec1d instances
          2. by providing a single filename, where the file contains the
             full set of single 1d spectra that are associated with a
             single echelle exposure for which the extraction has been done
          3. by providing a filelist, where each file in the list contains
             a single 1d spectrum in one of the approved formats
             (see Spec1d for possible informat values)

        """

        """ Initialize some variables """
        self.ordinfo = ordinfo

        """
        Load the spectra in a manner reflecting how the input spectra
         are provided
        """
        if isinstance(inspec, list):
            """
            Input spectra are a list of either Spec1d objects or filenames
            """
            for i, spec in enumerate(inspec):
                if isinstance(spec, ss.Spec1d):
                    self.append(spec)
                elif isinstance(spec, str):
                    if self.ordinfo is not None:
                        info = self.ordinfo[i]
                        tsec = [info['pixmin'], info['pixmax']]
                    else:
                        tsec = None
                    spec1d = ss.Spec1d(spec, informat=informat, trimsec=tsec,
                                       verbose=False)
                    self.append(spec1d)
                else:
                    errstr = '\n\nERROR: Input list must contain either '
                    errstr+ 'filenames or Spec1d instances\n\n'
                    raise TypeError(errstr)

        elif isinstance(inspec, str):
            """ Input spectra are contained in a single input file """
            if verbose:
                print('Opening echelle spectrum file: %s' % inspec)
                print('')
            try:
                hdu = pf.open(inspec)
            except IOError:
                raise IOError
            norder = len(hdu) - 1
            for i in range(1, len(hdu)):
                spec = ss.Spec1d(hdu[i], verbose=False)
                self.append(spec)

        else:
            print('')
            print('ERROR: First input must be either a filename or a list')
            print('  of either filenames or Spec1d instances')
            print('')
            raise TypeError

        """ Summarize the input if requested """
        if summary:
            self.input_summary()

    # -----------------------------------------------------------------------

    def input_summary(self):
        """

        Prints a summary of the data in the input file

        """

        print('Ord N_pix lam_start lam_end  dispave')
        print('--- ----- --------- -------- -------')

        """ Get information on the order labels, if it exists """
        if self.ordinfo is not None:
            orders = self.ordinfo['order']
        else:
            orders = arange(len(self))

        """ Print out the information for each order """
        for spec, order in zip(self, orders):
            print(' %2d %5d  %8.2f %8.2f  %5.2f' %
                  (order, len(spec), spec['wav'][0], spec['wav'][-1],
                   spec.dispave))

    # -----------------------------------------------------------------------

    def plot_all(self, plotmode='single', mode='input', title=None, z=None,
                 showz=True, linetype='strongem', smo=None, xlim=None,
                 pltylim=None, fig=None, verbose=False, debug=False,
                 **kwargs):
        """

        Plots all the spectra in the echelle data set in a single plot.

        """

        """ Set starting values """
        wmin = 1.e12
        wmax = 0.

        """ Set up the figure for plotting """
        if fig is not None:
            plotfig = plt.gcf()
            ax = plt.gca()
        else:
            plotfig, ax = plt.subplots(1, 1)

        """ Plot the spectra """
        if verbose:
            print(len(self))
        for count, spec in enumerate(self):

            """ Set plot labeling """
            if count == (len(self) - 1):
                if title is None:
                    title = 'Extracted Spectrum'
                showz = True
                if debug:
                    print('Bang')
            else:
                tmptitle = title
                title = None
                showz = False

            if verbose:
                if title is not None:
                    print(count, title)
                else:
                    print(count, 'None')

            """ Plot the spectrum """
            if smo is not None:
                spec.smooth(smo, mode=mode, title=title, verbose=verbose,
                            fig=plotfig, ax=ax, **kwargs)
            else:
                spec.plot(mode=mode, title=title, fig=plotfig, ax=ax,
                          **kwargs)
            title = tmptitle

            """ Adjust the plot limits """
            if spec['wav'].min() < wmin:
                wmin = spec['wav'].min()
            if spec['wav'].max() > wmax:
                wmax = spec['wav'].max()

        """ Set final plot limits """
        dw = wmax - wmin
        if xlim is not None:
            plt.xlim(xlim)
        else:
            plt.xlim(wmin-0.05*dw, wmax+0.05*dw)
        if pltylim is not None:
            plt.ylim(pltylim)

        """
        Mark spectral lines if desired
        Do this in a separate loop since the tickmark length depends
         on the y-axis range of the display
        """
        for spec in self:
            if z is not None:
                if smo is not None:
                    usesmooth = True
                else:
                    usesmooth = False
                try:
                    spec.mark_lines(linetype, z, showz=showz,
                                    usesmooth=usesmooth)
                except NameError:
                    spec.mark_lines('strongem', z, showz=showz,
                                    usesmooth=usesmooth)

    # ------------------------------------------------------------------------

    def save_multi(self, outroot, mode='input', outformat='multitab',
                   verbose=True):
        """

        Saves the echelle spectra, contained in separate Spec1d instances
         within this class, to one or more output files.
        The output options are:
         1. 'multitab' - Writes a single output file, which is a
                         multiextension fits file.  Each extension contains
                         a fits table ('fitstab' format in the Spec1d lingo)
         2. 'text'     - Produces several output files, one for each order.
                         The files are ascii text files, with columns for,
                         at minimum, wavelength and flux, and possibly also
                         variance and sky

        """

        if outformat == 'multitab':

            """
            Set up the required, essentially empty, primary HDU and the
            HDUList
            """
            phdu = pf.PrimaryHDU()
            hdulist = pf.HDUList(phdu)

            """
            Make a separate HDU for each echelle order and then create a
            HDUList
            """
            # lll = [pf.table_to_hdu(Table(spec)) for spec in self]
            # hdulist = pf.HDUList([phdu] + lll)
            for spec in self:
                savespec = spec.select_mode(mode)
                hdulist.append(pf.table_to_hdu(Table(savespec)))

            """ Save the output """
            outfile = '%s.fits' % outroot
            hdulist.writeto(outfile, overwrite=True)

        else:
            print('')
            print('Not yet implemented yet')
            print('')
            return

        if verbose:
            print('Spectrum saved in mode %s to file %s' %
                  (outformat, outfile))


import os
import json
from flask import Flask, request

app = Flask(__name__)
registration_file = None

@app.route('/')
def adminHome():
    return 'Welcome to admin Page!!!'

@app.route('/users')
def getAllUsers():
    users = getUserDetails()
    if len(users) <= 0:
        return "No user registered yet!!!"
    else:
        return {'Available Users': [user['uname'] for user in users]}

@app.route('/user/<uname>')
def getUserData(uname):
    users = getUserDetails()
    print('uname:',uname)
    for user in users:
        print(user)
        if user['uname'] == uname:
            return user
    return "{} not found!!!".format(uname)

def getUserDetails():
    feeds = []
    if not os.path.isfile(registration_file):
        return feeds

    with open(registration_file) as feedsjson:
        feeds = json.load(feedsjson)

    return feeds

if __name__ == '__main__':
    registration_file = os.environ['REG_FILE']
    app.run(host=os.environ['FLASKHOSTNAME'],
            port=os.environ['FLASKPORT'],
            debug=True)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('mfa', '0008_user_keys_last_used'),
    ]

    operations = [
        migrations.AddField(
            model_name='user_keys',
            name='owned_by_enterprise',
            field=models.NullBooleanField(default=None),
        ),
        migrations.RunSQL("update mfa_user_keys set owned_by_enterprise = %s where key_type='FIDO2'"%(True if getattr(settings,"MFA_OWNED_BY_ENTERPRISE",False) else False ))
    ]


"""Client - the main organisational entity in bika.
"""

# ~~~~~~~~~~ Irrelevant code for Odoo ~~~~~~~~~~~
# from dependencies.dependency import ClassSecurityInfo
# from dependencies.dependency import schemata
# from dependencies import atapi
# from dependencies.dependency import permissions
# from lims import interfaces
# from lims.config import *
# from lims.content.organisation import Organisation
# from lims.interfaces import IClient
# from lims.utils import isActive
# from dependencies.dependency import getUtility
# from dependencies.dependency import implements
# from dependencies.dependency import alsoProvides

import logging

from openerp import fields, models,osv

_logger = logging.getLogger(__name__)

from models.base_olims_model import BaseOLiMSModel
from fields.string_field import StringField
from fields.reference_field import ReferenceField
from fields.boolean_field import BooleanField
from fields.text_field import TextField
from fields.widget.widget import  TextAreaWidget, ReferenceWidget, StringWidget, BooleanWidget
from lims import bikaMessageFactory as _
from dependencies.dependency import DisplayList
from dependencies.dependency import getToolByName
from dependencies.dependency import safe_unicode
from lims import PMF, bikaMessageFactory as _
from lims.workflow import getCurrentState, StateFlow, InactiveState

import json
import sys

#schema = Organisation.schema.copy() + atapi.Schema((

schema = (

    StringField('Name',
        required = 1,
        searchable = True,
        validators = ('uniquefieldvalidator',),
        widget = StringWidget(
            label=_("Name"),
        ),
    ),
    StringField('TaxNumber',
        widget = StringWidget(
            label=_("VAT number"),
        ),
    ),
    StringField('Phone',
        widget = StringWidget(
            label=_("Phone"),
        ),
    ),
    StringField('Fax',
        widget = StringWidget(
            label=_("Fax"),
        ),
    ),
    StringField('EmailAddress',
        schemata = 'Address',
        widget = StringWidget(
            label=_("Email Address"),
        ),
        validators = ('isEmail',)
    ),
          # # ~~~~~~~~~~ PhysicalAddress behavior in Odoo is as selection field ~~~~~~~~~~~
        fields.Many2one(comodel_name='olims.country',string='physical_country'),
        fields.Many2one(comodel_name='olims.state',string='physical_state', domain="[('Country', '=', physical_country)]"),
        fields.Many2one(comodel_name='olims.district',string='physical_district', domain="[('State', '=', physical_state)]"),
        fields.Char(string='physical_city'),
        fields.Char(string='physical_postalcode'),
        fields.Char(string='physical_address'),

          # # ~~~~~~~~~~ PostalAddress behavior in Odoo is as selection field ~~~~~~~~~~~
          fields.Many2one(comodel_name='olims.country',string='postal_country'),
          fields.Many2one(comodel_name='olims.state',string='postal_state', domain="[('Country', '=', postal_country)]"),
          fields.Many2one(comodel_name='olims.district',string='postal_district', domain="[('State', '=', postal_state)]"),
          fields.Char(string='postal_city'),
          fields.Char(string='postal_postalcode'),
          fields.Char(string='postal_address'),

       # # ~~~~~~~~~~ BillingAddress behavior in Odoo is as selection field ~~~~~~~~~~~
          fields.Many2one(comodel_name='olims.country',string='billing_country'),
          fields.Many2one(comodel_name='olims.state',string='billing_state', domain="[('Country', '=', billing_country)]"),
          fields.Many2one(comodel_name='olims.district',string='billing_district', domain="[('State', '=', billing_state)]"),
          fields.Char(string='billing_city'),
          fields.Char(string='billing_postalcode'),
          fields.Char(string='billing_address'),

    StringField('AccountType',
        schemata = 'Bank details',
        widget = StringWidget(
            label=_("Account Type"),
        ),
    ),
    StringField('AccountName',
        schemata = 'Bank details',
        widget = StringWidget(
            label=_("Account Name"),
        ),
    ),
    StringField('AccountNumber',
        schemata = 'Bank details',
        widget = StringWidget(
            label=_("Account Number"),
        ),
    ),
    StringField('BankName',
        schemata = 'Bank details',
        widget = StringWidget(
            label=_("Bank name"),
        ),
    ),
    StringField('BankBranch',
        schemata = 'Bank details',
        widget = StringWidget(
            label=_("Bank branch"),
        ),
    ),

    StringField('ClientID',
        required=1,
        searchable = True,
        #validators = ('uniquefieldvalidator', 'standard_id_validator'),
        widget=StringWidget(
            label=_('Client ID'),
            description=_('ClientID is required.'),
        ),
    ),

    BooleanField('BulkDiscount',
        default = False,
        #write_permission = ManageClients,
        widget = BooleanWidget(
            label=_("Bulk discount applies"),
        ),
    ),

    BooleanField('MemberDiscountApplies',
        default = False,
        #write_permission = ManageClients,
        widget = BooleanWidget(
            label=_("Member discount applies"),
        ),
    ),

    # ~~~~~~~ To be implemented ~~~~~~~
    # atapi.LinesField('EmailSubject',
    #     schemata = 'Preferences',
    #     default = ['ar', ],
    #     vocabulary = EMAIL_SUBJECT_OPTIONS,
    #     widget = atapi.MultiSelectionWidget(
    #         description=_("Items to be included in email subject lines"),
    #         label=_("Email subject line"),
    #     ),
    # ),
    #

    fields.Many2many(string='DefaultCategories',
                   comodel_name='olims.analysis_category',
#                    schemata="Method",
                   required=False,
                   help="Always expand the selected categories in client views",
#     schemata = 'Preferences',
    #     required = 0,
    #     multiValued = 1,
    #     vocabulary = 'getAnalysisCategories',
    #     vocabulary_display_path_bound = sys.maxint,
    #     allowed_types = ('AnalysisCategory',),
    #     relationship = 'ClientDefaultCategories',
    #     widget = atapi.ReferenceWidget(
    #         checkbox_bound = 0,
    #         label=_("Default categories"),
    #         description=_("Always expand the selected categories in client views"),
    #     ),
    ),

    #
    # ReferenceField(string='RestrictedCategories',
    #            selection=[('olims.analysis_category', 'AnalysisCategory')],
    #            required=0,
    #             widget = ReferenceWidget(
    #             checkbox_bound = 0,
    #             #label=_("Default categories"),
    #             help=_("Always expand the selected categories in client views"),
    #         ),
    #     ),

        fields.Many2many(string='RestrictedCategories',
                   comodel_name='olims.analysis_category',
#                    schemata="Method",
                   required=False,
                   help="Show only selected categories in client views",
    #  schemata = 'Preferences',
    #     required = 0,
    #     multiValued = 1,
    #     vocabulary = 'getAnalysisCategories',
    #     validators = ('restrictedcategoriesvalidator',),
    #     vocabulary_display_path_bound = sys.maxint,
    #     allowed_types = ('AnalysisCategory',),
    #     relationship = 'ClientRestrictedCategories',
    #     widget = atapi.ReferenceWidget(
    #         checkbox_bound = 0,
    #         label=_("Restrict categories"),
    #         description=_("Show only selected categories in client views"),
    #     ),
    ),

    ## ~~~~~~~ To be implemented ~~~~~~~ multiValued
    # atapi.ReferenceField('RestrictedCategories',
    #     schemata = 'Preferences',
    #     required = 0,
    #     multiValued = 1,
    #     vocabulary = 'getAnalysisCategories',
    #     validators = ('restrictedcategoriesvalidator',),
    #     vocabulary_display_path_bound = sys.maxint,
    #     allowed_types = ('AnalysisCategory',),
    #     relationship = 'ClientRestrictedCategories',
    #     widget = atapi.ReferenceWidget(
    #         checkbox_bound = 0,
    #         label=_("Restrict categories"),
    #         description=_("Show only selected categories in client views"),
    #     ),
    # ),

    fields.Selection(string='DefaultARSpecs',
         selection=( ('ar_specs', 'Analysis Request Specifications'),
                                 ('lab_sampletype_specs', 'Sample Type Specifications (Lab)'),
                                 ('client_sampletype_specs', 'Sample Type Specifications (Client)')),
         help="DefaultARSpecs_description",
    #     schemata = "Preferences",
         default = 'ar_specs',
    #     vocabulary =  DEFAULT_AR_SPECS,
    #     widget = atapi.SelectionWidget(
    #         label=_("Default AR Specifications"),
    #         description=_("DefaultARSpecs_description"),
    #         format='select',
    #     )
     ),

    BooleanField('DefaultDecimalMark',
        default = True,
        #write_permission = ManageClients,
         widget = BooleanWidget(
            label=_("Default decimal mark"),
            description=_("The decimal mark selected in Bika Setup will be used."),
        ),
    ),

    fields.Selection(string='DecimalMark',
         selection=(
                    ('.', _('Dot (.)')),
                    (',', _('Comma (,)')),
                                    ),
         help="Decimal mark to use in the reports from this Client.",
    #     schemata = "Preferences",
         default = '.',
    #     vocabulary =  DECIMAL_MARKS,
    #     widget = atapi.SelectionWidget(
    #         label=_("Custom decimal mark"),
    #         description=_("Decimal mark to use in the reports from this Client."),
    #         format = 'select',
    #     )
     ),

)

# ~~~~~~~~~~ Irrelevant code for Odoo ~~~~~~~~~~~
# schema['AccountNumber'].write_permission = ManageClients
# schema['title'].widget.visible = False
# schema['description'].widget.visible = False
# schema['EmailAddress'].schemata = 'default'
#
# schema.moveField('ClientID', after='Name')

class Client(models.Model, BaseOLiMSModel):#(Organisation):
    _name='olims.client'
    # implements(IClient)
    # security = ClassSecurityInfo()
    # displayContentsTab = False
    # schema = schema

    _at_rename_after_creation = True
    def _renameAfterCreation(self, check_auto_id=False):
        from lims.idserver import renameAfterCreation
        renameAfterCreation(self)

    def Title(self):
        """ Return the Organisation's Name as its title """
        return safe_unicode(self.getField('Name').get(self)).encode('utf-8')

    #security.declarePublic('getContactFromUsername')
    def getContactFromUsername(self, username):
        for contact in self.objectValues('Contact'):
            if contact.getUsername() == username:
                return contact.UID()

    #security.declarePublic('getContactUIDForUser')
    def getContactUIDForUser(self):
        """ get the UID of the user associated with the authenticated user
        """
        membership_tool = getToolByName(self, 'portal_membership')
        member = membership_tool.getAuthenticatedMember()
        username = member.getUserName()
        r = self.portal_catalog(
            portal_type = 'Contact',
            getUsername = username
        )
        if len(r) == 1:
            return r[0].UID

    #security.declarePublic('getARImportOptions')
    def getARImportOptions(self):
        return ARIMPORT_OPTIONS

    #security.declarePublic('getAnalysisCategories')
    def getAnalysisCategories(self):
        """ return all available analysis categories """
        bsc = getToolByName(self, 'bika_setup_catalog')
        cats = []
        for st in bsc(portal_type = 'AnalysisCategory',
                      inactive_state = 'active',
                      sort_on = 'sortable_title'):
            cats.append((st.UID, st.Title))
        return DisplayList(cats)

    def getContacts(self, only_active=True):
        """ Return an array containing the contacts from this Client
        """
        contacts = []
        if only_active:
            contacts = [c for c in self.objectValues('Contact') if
                        getCurrentState(c, StateFlow.inactive) == InactiveState.active]
        else:
            contacts = self.objectValues('Contact')
        return contacts;

    def getDecimalMark(self):
        """ Return the decimal mark to be used on reports for this
            client. If the client has DefaultDecimalMark selected, the
            Default value from Bika Setup will be returned. Otherwise,
            will return the value of DecimalMark.
        """
        if self.getDefaultDecimalMark() == False:
            return self.Schema()['DecimalMark'].get(self)
        return self.bika_setup.getDecimalMark()

# schemata.finalizeATCTSchema(schema, folderish = True, moveDiscussion = False)
#
# atapi.registerType(Client, PROJECTNAME)

Client.initialze(schema)


#Utilize SpaCy, load the best-model and calculate top entries for selected POS tags

#Import all needed tools
import os
import spacy
import sys
import re
import glob
from collections import Counter
nlp = spacy.load('./sac_ner/model-best')
nlp.max_length = 1500000
#open directory
os.chdir('Alpine_Journal')

#Read the collection of file (alpine full collection):
#Enter the readfile here and read as string:

persons=[]
gpes=[]
dates= []
mountains = []
waters = []
rocks= []
for file in os.listdir():
    if file.endswith('.txt'):
        with open(file, 'r', encoding='utf-8') as f:
            fullpack = f.read()
            fullpack = fullpack.replace('\n', '')
            #print(fullpack)

            #Enter the string text here in spacy nlp processor
            doc = nlp(fullpack)
            for ent in doc.ents:
                if ent.label_ == "PERSON":
                    persons.append(ent.text)
                if ent.label_ == "GPE":
                    gpes.append(ent.text)
                if ent.label_ == "DATE":
                    dates.append(ent.text)
                if ent.label_ == "MNTN":
                    mountains.append(ent.text)
                if ent.label_ == "WATER":
                    waters.append(ent.text)
                if ent.label_ == "ROCK":
                    rocks.append(ent.text)

            # top 10 most common person tokens
person_freq = Counter(persons)
common_ten_persons = person_freq.most_common(10)
print(f'PERSON: {common_ten_persons}') # added

            # top 10 most common gpe tokens
gpe_freq = Counter(gpes)
common_ten_gpe = gpe_freq.most_common(10)
print(f'GPE: {common_ten_gpe}')

            # top 10 most common date tokens
date_freq = Counter(dates)
common_ten_date = date_freq.most_common(10)
print(f'DATES: {common_ten_date}')

            # top 10 most common mountain tokens
mountain_freq = Counter(mountains)
common_ten_mountain = mountain_freq.most_common(10)
print(f'Mountains: {common_ten_mountain}')

            # top 10 most common water tokens
water_freq = Counter(waters)
common_ten_water = water_freq.most_common(10)
print(f'WATER: {common_ten_water}')

            # top 10 most common rock tokens
rock_freq = Counter(rocks)
common_ten_rock = rock_freq.most_common(10)
print(f'ROCKS: {common_ten_rock}')


from pydantic import BaseModel,HttpUrl,constr
from pydantic.typing import Optional, Literal, List
from decouple import config

#serializer
class IngredientBase(BaseModel):
    id: Optional[int] = None
    name: str
    brand: str
    measure_unit: int

class Ingredient(IngredientBase):
    name: str
    brand: str
    measure_unit: int

    class Config:
        orm_mode = True


import numpy as np
import os
import cv2
import socket
import time
import struct

def ip2int(addr):
    return struct.unpack("!I", socket.inet_aton(addr))[0]

def get_ip_address():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    return s.getsockname()[0]

TCP_IP = '192.168.0.21'
TCP_PORT = 4965

encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90]

cap = cv2.VideoCapture(0)

sock = socket.socket()

while True:
    sock.connect((TCP_IP, TCP_PORT))
    myip = ip2int(get_ip_address())
    sock.send(struct.pack('I', myip))

    sock.recv(1)

    startT = time.time()
    while(cap.isOpened()):
        cap.grab()
        ret, frame = cap.retrieve()
        result, encoded = cv2.imencode('.jpg', frame, encode_param)
        data = np.array(encoded)
        imageByte = data.tobytes()
        #print(len(imageByte))
        TS = time.time()-startT
        #print(TS);
        sock.send(struct.pack('d', TS))
        sock.send(str(len(imageByte)).ljust(16).encode())
        sock.send(imageByte)

cap.release()
sock.close()


"""
Helper script to download raw recipe data from a URL and dump it to disk.
The resulting files can be used as test input data.
"""

import sys, json, pprint
import requests
import extruct
from scrape_schema_recipe import scrape_url
from w3lib.html import get_base_url

for url in sys.argv[1:]:
    try:
        data = scrape_url(url)[0]
        slug = list(filter(None, url.split("/")))[-1]
        filename = f"{slug}.json"
        with open(filename, "w") as f:
            json.dump(data, f, indent=4, default=str)
        print(f"Saved {filename}")
    except Exception as e:
        print(f"Error for {url}: {e}")
        print("Trying extruct instead")
        pp = pprint.PrettyPrinter(indent=2)
        r = requests.get(url)
        base_url = get_base_url(r.text, r.url)
        data = extruct.extract(r.text, base_url=base_url)
        pp.pprint(data)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import itertools
import sys
from collections import defaultdict

def read_input(filename):
    table = defaultdict(dict)
    for line in open(filename):
        tokens = line.strip().rstrip('.').split(' ')
        table[tokens[0]][tokens[-1]] = int(tokens[3]) * (-1 if tokens[2] == "lose" else 1)
    return table

def score(table, p):
    s = 0
    for i in range(len(p)):
        pers  = p[i]
        left  = p[(i - 1) % len(p)]
        right = p[(i + 1) % len(p)]
        s += table[pers][left] + table[pers][right]
    return s

def part1(table):
    nodes = list(table.keys())
    best = 0
    for p in itertools.permutations(nodes):
        best = max(best, score(table, p))
    return best

def part2(table):
    for k in list(table.keys()):
        table[k]["me"] = 0
        table["me"][k] = 0
    return part1(table)

def main(filename):
    table = read_input(filename)
    print(part1(table))
    print(part2(table))

if __name__ == '__main__':
    main('input' if len(sys.argv) == 1 else sys.argv[1])


import sys, os, time, math
from fastapi import FastAPI
import torch
import time
from typing import Optional
from fastapi.middleware.cors import CORSMiddleware
import torch
from diffusers import StableDiffusionPipeline
from PIL import Image
import base64

#pipe = StableDiffusionPipeline.from_single_file("/run/media/privateserver/NECTAC1TBSSD/a1111/stable-diffusion-webui/models/Stable-diffusion/sd15.safetensors", torch_dtype=torch.float16).to("cuda")

def stable_diffusion(prompt):
    # Generate an image using a prompt
    pipe.safety_checker = None
    pipe.requires_safety_checker = False
    image = pipe(prompt=prompt, num_inference_steps=30).images[0]

    # Save the image directly as PNG
    image.save("generated_image.png")

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

app = FastAPI()

origins = [
    "http://localhost:3000",  # React app address
    "http://127.0.0.1:3000"
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

from exllamav2 import(
    ExLlamaV2,
    ExLlamaV2Config,
    ExLlamaV2Cache,
    ExLlamaV2Cache_8bit,
    ExLlamaV2Tokenizer,
    model_init,
)

import argparse
import torch

from exllamav2.generator import (
    ExLlamaV2StreamingGenerator,
    ExLlamaV2Sampler
)

from examples.chat_formatting import CodeBlockFormatter
from examples.chat_prompts import prompt_formats
prompt_formats_list = list(prompt_formats.keys())

# Options

parser = argparse.ArgumentParser(description = "Simple Llama2 chat example for ExLlamaV2")
parser.add_argument("-dm", "--draft_model_dir", type = str, default = None, help = "Path to draft model directory")
parser.add_argument("-nds", "--no_draft_scale", action = "store_true", help = "If draft model has smaller context size than model, don't apply alpha (NTK) scaling to extend it")

parser.add_argument("-modes", "--modes", action = "store_true", help = "List available modes and exit.")
parser.add_argument("-mode", "--mode", choices = prompt_formats_list, help = "Chat mode. Use llama for Llama 1/2 chat finetunes.")
parser.add_argument("-un", "--username", type = str, default = "User", help = "Username when using raw chat mode")
parser.add_argument("-bn", "--botname", type = str, default = "Chatbot", help = "Bot name when using raw chat mode")
parser.add_argument("-sp", "--system_prompt", type = str, help = "Use custom system prompt")

parser.add_argument("-temp", "--temperature", type = float, default = 0.70, help = "Sampler temperature, default = 0.95 (1 to disable)")
parser.add_argument("-topk", "--top_k", type = int, default = 20, help = "Sampler top-K, default = 20 (0 to disable)")
parser.add_argument("-topp", "--top_p", type = float, default = 0.9, help = "Sampler top-P, default = 0.9 (0 to disable)")
parser.add_argument("-typical", "--typical", type = float, default = 1.0, help = "Sampler typical threshold, default = 1.0 (0 to disable)")
parser.add_argument("-repp", "--repetition_penalty", type = float, default = 1.15, help = "Sampler repetition penalty, default = 1.15 (1 to disable)")
parser.add_argument("-maxr", "--max_response_tokens", type = int, default = 4096, help = "Max tokens per response, default = 1000")
parser.add_argument("-resc", "--response_chunk", type = int, default = 250, help = "Space to reserve in context for reply, default = 250")
parser.add_argument("-ncf", "--no_code_formatting", action = "store_true", help = "Disable code formatting/syntax highlighting")

parser.add_argument("-c8", "--cache_8bit", action = "store_true", default = True, help = "Use 8-bit cache")

parser.add_argument("-pt", "--print_timings", action = "store_true", help = "Output timings after each prompt")

# Arrrgs

model_init.add_args(parser)
args = parser.parse_args()

# Prompt templates/modes

if args.modes:
    print(" -- Available formats:")
    for k, v in prompt_formats.items():
        print(f" --   {k:12} : {v().description}")
    sys.exit()

username = args.username
botname = args.botname
system_prompt = args.system_prompt

mode = 'chatml'

prompt_format = prompt_formats[mode]()
prompt_format.botname = botname
prompt_format.username = username
if system_prompt is None: system_prompt = prompt_format.default_system_prompt()

# Initialize model and tokenizer

args.model_dir = "modeldir"

model_init.check_args(args)
model_init.print_options(args)
print("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
print(args)
print("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
model, tokenizer = model_init.init(args, allow_auto_split = True)

# Initialize draft model if provided, assume it always fits on first device

draft_model = None
draft_cache = None

if args.draft_model_dir:

    print(f" -- Draft model: {args.draft_model_dir}")

    draft_config = ExLlamaV2Config()
    draft_config.model_dir = args.draft_model_dir
    draft_config.prepare()

    if draft_config.max_seq_len < model.config.max_seq_len:

        if args.no_draft_scale:
            print(f" !! Warning: Draft model native max sequence length is less than sequence length for model. Speed may decrease after {draft_config.max_seq_len} tokens.")
        else:
            ratio = model.config.max_seq_len / draft_config.max_seq_len
            alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2
            draft_config.scale_alpha_value = alpha
            print(f" -- Applying draft model RoPE alpha = {alpha:.4f}")

    draft_config.max_seq_len = model.config.max_seq_len
    draft_config.no_flash_attn = args.no_flash_attn

    print(" -- Loading draft model...")

    draft_model = ExLlamaV2(draft_config)
    draft_model.load()

    #if args.cache_8bit:
    draft_cache = ExLlamaV2Cache_8bit(draft_model)
    #else:
    #draft_cache = ExLlamaV2Cache(draft_model)

# Create cache

#if args.cache_8bit:
cache = ExLlamaV2Cache_8bit(model, lazy = not model.loaded)
#else:
    #cache = ExLlamaV2Cache(model, lazy = not model.loaded)

# Load model now if auto split enabled

if not model.loaded:

    print(" -- Loading model...")
    model.load_autosplit(cache)

# Chat context

def format_prompt(user_prompt, first):
    global system_prompt, prompt_format

    if first:
        return prompt_format.first_prompt() \
            .replace("<|system_prompt|>", system_prompt) \
            .replace("<|user_prompt|>", user_prompt)
    else:
        return prompt_format.subs_prompt() \
            .replace("<|user_prompt|>", user_prompt)

def encode_prompt(text):
    global tokenizer, prompt_format

    add_bos, add_eos, encode_special_tokens = prompt_format.encoding_options()
    return tokenizer.encode(text, add_bos = add_bos, add_eos = add_eos, encode_special_tokens = encode_special_tokens)

user_prompts = []
responses_ids = []

def get_tokenized_context(max_len):
    global user_prompts, responses_ids

    while True:

        context = torch.empty((1, 0), dtype=torch.long)

        for turn in range(len(user_prompts)):

            up_text = format_prompt(user_prompts[turn], context.shape[-1] == 0)
            up_ids = encode_prompt(up_text)
            context = torch.cat([context, up_ids], dim=-1)

            if turn < len(responses_ids):
                context = torch.cat([context, responses_ids[turn]], dim=-1)

        if context.shape[-1] < max_len: return context

        # If the context is too long, remove the first Q/A pair and try again. The system prompt will be moved to
        # the first entry in the truncated context

        user_prompts = user_prompts[1:]
        responses_ids = responses_ids[1:]

# Generator

generator = ExLlamaV2StreamingGenerator(model, cache, tokenizer, draft_model, draft_cache)

settings = ExLlamaV2Sampler.Settings()
settings.temperature = args.temperature
settings.top_k = args.top_k
settings.top_p = args.top_p
settings.typical = args.typical
settings.token_repetition_penalty = args.repetition_penalty

max_response_tokens = args.max_response_tokens
min_space_in_context = args.response_chunk

# Stop conditions

generator.set_stop_conditions(prompt_format.stop_conditions(tokenizer))

# ANSI color codes

col_default = "\u001b[0m"
col_user = "\u001b[33;1m"  # Yellow
col_bot = "\u001b[34;1m"  # Blue
col_error = "\u001b[31;1m"  # Magenta
col_sysprompt = "\u001b[37;1m"  # Grey

# Code block formatting

codeblock_formatter = None if args.no_code_formatting else CodeBlockFormatter()
in_code_block = False

delim_overflow = ""

# Other options

print_timings = args.print_timings

# Main loop

print(f" -- Prompt format: {args.mode}")
print(f" -- System prompt:")
print()
print(col_sysprompt + system_prompt.strip() + col_default)

from pydantic import BaseModel

class GenerateResponseModel(BaseModel):
    user_message: str
    username: str
    botname: str
    print_timings: Optional[bool] = False
    in_code_block: Optional[bool] = False

from fastapi.responses import StreamingResponse
from typing import Generator
import re

def get_first_prompt(text):
    match = re.search(r'\[image\](.*?)\[/image\]', text)
    return match.group(1) if match else None

def generate_prompt_response(user_message,username,botname,print_timings,in_code_block):
    col_default = "\u001b[0m"
    global responses_ids
    global user_prompts
    up = user_message
    response_output = ""
    responses_ids2 = responses_ids
    user_prompts2 = user_prompts
    user_prompts = []
    responses_ids = []
    user_prompts.append(up)
    # Send tokenized context to generator
    active_context = get_tokenized_context(model.config.max_seq_len - min_space_in_context)
    generator.begin_stream(active_context, settings)

    if prompt_format.print_bot_name():
        response_output += col_bot + botname + ": " + col_default

    if print_timings:
        time_begin_stream = time.time()
        if draft_model is not None:
            generator.reset_sd_stats()

    while True:
        chunk, eos, tokens = generator.stream()
        if eos:
            responses_ids = responses_ids2
            user_prompts = user_prompts2
            return response_output
        if len(response_output) == 0:
            chunk = chunk.lstrip()
        response_output += chunk
        continue

import re
@app.post("/generate_response/")
async def generate_response(data: GenerateResponseModel) -> StreamingResponse:
    def stream_output():
        global user_prompts
        global responses_ids
        user_message = data.user_message
        username = data.username
        botname = data.botname
        print_timings = data.print_timings
        in_code_block = data.in_code_block
        col_default = "\u001b[0m"

        if (user_message == "clear"):
            user_prompts = []
            responses_ids = []
            return
        pattern = r"inject_response=Q:(.*?)A:(.*)"
        match = re.match(pattern, user_message)

        if (user_message == "history"):
            ite = 0
            for response_tokens in responses_ids:
                response_text = tokenizer.decode(response_tokens[0])
                yield "USER: " + user_prompts[ite]
                yield "\n"
                yield "------------------"
                yield "\n"
                yield response_text
                yield "\n"
                ite = ite + 1
            return
        if match:
            question, response = match.groups()
            user_prompts = [question]
            response = tokenizer.encode(response)
            response_tokens = torch.tensor(response)
            responses_ids = [response_tokens]
            return

        phrases = ["generate an image", "create an image", "make an image", "want an image", "give me an image", "provide an image", "show me an image","generate me an image","generate a photo","create a photo","make a photo","generate a photo"]
        if any(phrase in user_message.lower() for phrase in phrases):
            final_message = '''
            I want you to parse an user message and put [image][/image] tags around image requests.

            Example input:
            "generate the image of a bear, my son wants to see it!"
            Output:
            [image]a bear[/image]
            Example input:
            "generate an image about oppeheimer, and write a 1000 word essay about him"
            Output:
            [image]oppeheimer[image]
            Example input:
            "I want an image of a beautiful woman in a red dress!"
            Output:
            [image]a beautiful woman in a red dress[/image]
            Example input:
            "Generate a photo of Obama."
            Output:
            [image]photography of Barack Obama[/image]

            Always using [image][/image] tags surrounding the parsed request!

            I need you to identify which image the user asked, and put [image][/image] prompts around it.
            Only provide a reply with the tags. Do not output anything else.

            The user asked this, identify the image and surround it with tags: '''"\""+user_message+"\""'''

            '''
            prompt = generate_prompt_response(final_message,username,botname,print_timings,in_code_block)
            prompt = get_first_prompt(prompt)
            try:
                stable_diffusion("best quality, 8k uhd,"+prompt)
            except:
                yield '''Sorry, there was a problem when I was trying to generate the image.'''
            # Read the 'generated_image.png' file that your code produced
            with open("generated_image.png", "rb") as image_file:
                # Convert the image to Base64
                encoded_string = base64.b64encode(image_file.read()).decode('utf-8')

            # Embed the Base64 string into the <img> tag
            if prompt:
                img_tag = f'''Generated an image with the prompt "{prompt}" : \n <img src="data:image/png;base64,{encoded_string}" alt="Embedded Image" />'''+"\n"
                yield img_tag
        up = user_message

        user_prompts.append(up)

        active_context = get_tokenized_context(model.config.max_seq_len - min_space_in_context)
        generator.begin_stream(active_context, settings)

        if prompt_format.print_bot_name():
            yield col_bot + botname + ": " + col_default

        response_tokens = 0
        response_text = ""
        responses_ids.append(torch.empty((1, 0), dtype = torch.long))

        if print_timings:
            time_begin_stream = time.time()
            if draft_model is not None:
                generator.reset_sd_stats()

        while True:
            chunk, eos, tokens = generator.stream()
            yield chunk
            if len(response_text) == 0: chunk = chunk.lstrip()
            response_text += chunk
            responses_ids[-1] = torch.cat([responses_ids[-1], tokens], dim=-1)

            chunk, codeblock_delimiter = (chunk, False) if codeblock_formatter is None else codeblock_formatter.process_delimiter(chunk)

            '''if not in_code_block and codeblock_delimiter:
                codeblock_formatter.begin()
                yield "\n"
                in_code_block = True
                codeblock_delimiter = False

            if in_code_block:
                yield codeblock_formatter.get_code_block(chunk)
            else:
                yield chunk

            if in_code_block and codeblock_delimiter:
                if eos: yield codeblock_formatter.get_code_block("\n")
                yield "\033[0m"
                in_code_block = False
                codeblock_delimiter = False
                '''

            if generator.full():
                active_context = get_tokenized_context(model.config.max_seq_len - min_space_in_context)
                generator.begin_stream(active_context, settings)

            response_tokens += 1
            if response_tokens == max_response_tokens:
                if tokenizer.eos_token_id in generator.stop_tokens:
                    responses_ids[-1] = torch.cat([responses_ids[-1], tokenizer.single_token(tokenizer.eos_token_id)], dim=-1)
                yield "\n"
                yield col_error + f" !! Response exceeded {max_response_tokens} tokens and was cut short." + col_default
                break

            if eos:
                if prompt_format.print_extra_newline():
                    yield "\n"
                break

        if print_timings:
            time_end_stream = time.time()
            speed = response_tokens / (time_end_stream - time_begin_stream)
            if draft_model is not None:
                eff, acc, _, _, _ = generator.get_sd_stats()
                sd_stats = f", SD eff. {eff*100:.2f}%, SD acc. {acc*100:.2f}%"
            else:
                sd_stats = ""
            yield "\n"
            yield col_sysprompt + f"(Response: {response_tokens} tokens, {speed:.2f} tokens/second{sd_stats})" + col_default

    return StreamingResponse(stream_output())

# If running the app directly
if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 11 12:00:16 2018

@author: soojunghong

"""

import scipy
from scipy.stats import pearsonr

x = scipy.array([-0.6549, 2.3464, 3.0])
y = scipy.array([-1.4604, 3.8653, 21.0])

r_row, p_value = pearsonr(x,y)
r_row
p_value


# oracle 접속 모듈을 만들어 보세요
#
# 단일 셀렉트(파라미터 있는거 , 없는거)
# 다건 셀렉트 (파라미터 있는거 , 없는거)
# 업데이트
# 인서트

import cx_Oracle

class DBManager:

    def __init__(self):
        self.conn = None
        self.get_connection()

    def __del__(self):
        try:
            print("소멸자")
            if self.conn:
                self.conn.close()
        except Exception as err:
            print("__del__ 예외", err)

    def get_connection(self):
        self.conn = cx_Oracle.connect("java", "oracle", "127.0.0.1:1521/XE", encoding="utf-8")

        return self.conn

    # def db_close(self) -> None:
    #     try:
    #         if self.conn:
    #             self.conn.close()
    #     except Exception as err:
    #         print(err)
    def makeDictFactory(self, cursor):
        columnNames = [d[0] for d in cursor.description]

        def createRow(*args):
            return dict(zip(columnNames, args))

        return createRow

    def selectOne(self, query):
        cursor = self.conn.cursor()
        cursor.execute(query)
        output = None
        for row in cursor:
            for i in range(len(row)):
                # print(row[i], end=" ")
                output = row[i]
        cursor.close()
        # self.db_close()
        return output

    def selectOneParam (self, query, t):

        cursor = self.conn.cursor()
        cursor.execute(query, t)
        output = None
        for row in cursor:
            for i in range(len(row)):
                # print(row[i], end=" ")
                output = row[i]
        cursor.close()
        return output

    def selectList(self, query):

        cursor = self.conn.cursor()
        cursor.execute(query)
        cursor.rowfactory = self.makeDictFactory(cursor)
        rows = cursor.fetchall()
        cursor.close()
        return rows

    def selectListParam(self, query, t):
        cursor = self.conn.cursor()
        cursor.execute(query, t)
        cursor.rowfactory = self.makeDictFactory(cursor)
        rows = cursor.fetchall()
        cursor.close()
        return rows

    def insert(self, query, t):
        cursor = self.conn.cursor()
        cursor.execute(query, t)
        self.conn.commit()
        cursor.close()

    def merge(self, query, t):
        cursor = self.conn.cursor()
        cursor.execute(query, t)
        self.conn.commit()
        cursor.close()

    def update(self, query, t):
        cursor = self.conn.cursor()
        cursor.execute(query, t)
        self.conn.commit()
        cursor.close()

if __name__ == '__main__':
    print("connect test")
    try:
        manager = DBManager()
        conn = manager.get_connection()
        print(conn.version)
    except Exception as e:
        print(str(e))
    finally:
        print("connect test 종료")
else:
    print("db 모듈 임포트")


import json

# with open('static/data/data_1000.json', 'r', encoding='utf-8') as f:
# 	data = json.load(f)
# 	for k, v in list(data.items()):
# 		if v['is_process']:
# 			print(k, v)
n = 215
cnt = 0
# data_1500.json
with open('static/data/data_1500.json', 'r', encoding='utf-8') as f:
	data = json.load(f)
	for k, v in list(data.items()):
		# if 'judge' not in v.keys():
		if not v['is_process']:
			print(k, v.keys(), v['is_process'])
		# if k == str(n):
		# 	for i, j in v.items():
		# 		print(i, j)
	# for k, v in list(data.items())[n:n+1]:
	# 	print(k)
	# 	for i, j in v.items():
	# 		print(i, j)
	# print(len(data.keys()))

# 	print(cnt)

# 130 false []

# 45 True
# 60 模型未找到
# 793