from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from datetime import datetime, timedelta
from operators.extract_operator import mysql_table_to_s3, pgsql_table_to_s3, s3_to_redshift
from airflow.operators.postgres_operator import PostgresOperator
from rvairflow import slack_hook as sh
from airflow.models import Variable

redshift_conn = 'cards-redshift-cluster'
# Default settings applied to all tasks
default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': datetime(2021, 5, 19),
    'email': ['mdey@redventures.com'],
    'email_on_failure': False,
    'email_on_retry': False,
    'on_failure_callback': sh.slack_failure_callback(slack_connection_id=Variable.get("slack-connection-name")),
    'retries': 0,
    'retry_delay': timedelta(minutes=5),
    'provide_context': True
}

# Using a DAG context manager, you don't have to specify the dag property of each task
with DAG('cccom-dw-sales-and-clicks',
         schedule_interval='0 * * * *',
         catchup=False,
         max_active_runs=1,
         dagrun_timeout=timedelta(hours=2),
         default_args=default_args) as dag:

    extract_affiliates = PythonOperator(
        task_id=f'extract-cccom-affiliates',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_affiliates.sql', 'key': 'affiliates.csv', 'compress': True},
        provide_context=True
    )

    load_affiliates = PythonOperator(
        task_id=f'load-cccom-affiliates',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_affiliates', 'key': 'affiliates.csv', 'compress': True},
        provide_context=True
    )

    merge_affiliates = PostgresOperator(
        task_id='merge-cccom-affiliates',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_affiliates.sql'
    )

    # This is a long-running job, so we up its priority so it
    # starts early in the DAG, and other tasks can happen in parallel.
    extract_click_transactions = PythonOperator(
        task_id='extract-cccom-click_transactions',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_click_transactions.sql',
                   'key': 'click_transactions.csv',
                   'compress': True},
        provide_context=True,
        priority_weight=5,
        execution_timeout=timedelta(minutes=40)
    )

    load_click_transactions = PythonOperator(
        task_id=f'load-cccom-click_trans',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_click_trans', 'key': 'click_transactions.csv', 'compress': True},
        provide_context=True
    )

    merge_click_transactions = PostgresOperator(
        task_id='merge-cccom-click_trans',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_clicks.sql'
    )

    extract_device_types = PythonOperator(
        task_id='extract-cccom-device_types',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_device_types.sql', 'key': 'device_types.csv', 'compress': True},
        provide_context=True,
        execution_timeout=timedelta(minutes=3)
    )

    load_device_types = PythonOperator(
        task_id=f'load-cccom-device_types',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_device_types', 'key': 'device_types.csv', 'compress': True},
        provide_context=True
    )

    merge_device_types = PostgresOperator(
        task_id='merge-cccom-device_types',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_device_types.sql'
    )

    extract_pages = PythonOperator(
        task_id='extract-cccom-pages',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_pages.sql', 'key': 'pages.csv', 'compress': True},
        provide_context=True,
        execution_timeout=timedelta(minutes=2)
    )

    load_pages = PythonOperator(
        task_id=f'load-cccom-pages',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_pages', 'key': 'pages.csv', 'compress': True},
        provide_context=True
    )

    merge_pages = PostgresOperator(
        task_id='merge-cccom-pages',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_pages.sql'
    )

    # This is a long-running job, so we up its priority so it
    # starts early in the DAG, and other tasks can happen in parallel.
    extract_sale_transactions = PythonOperator(
        task_id='extract-cccom-sale_transactions',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_sale_trans.sql', 'key': 'sale_trans.csv', 'compress': True},
        provide_context=True,
        execution_timeout=timedelta(minutes=20),
        priority_weight=5
    )

    load_sale_transactions = PythonOperator(
        task_id=f'load-cccom-sale_trans',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_sale_trans', 'key': 'sale_trans.csv', 'compress': True},
        provide_context=True
    )

    merge_sale_transactions = PostgresOperator(
        task_id='merge-cccom-sale_trans',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_sales.sql'
    )

    """Adding new tasks for sale trans from RMS"""
    extract_sale_rms_with_cutover_date = PythonOperator(
        task_id='extract-cccom-sales_rms-with-cutover-date',
        python_callable=pgsql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_rms_transactions.sql',
                   'key': 'rms_transactions.csv',
                   'compress': True},
        provide_context=True
    )

    load_sale_rms_with_cutover_date = PythonOperator(
        task_id=f'load-cccom-sales_rms-with-cutover-date',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_rms_transactions', 'key': 'rms_transactions.csv', 'compress': True},
        provide_context=True
    )

    merge_sales_rms_with_cutover_date = PostgresOperator(
        task_id='merge-cccom-sales_rms_with_cutover_date',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_sales_rms.sql'
    )

    """End of task defs for sale trans"""
    extract_transaction_types = PythonOperator(
        task_id='extract-cccom-transaction_types',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_transaction_types.sql',
                   'key': 'transaction_types.csv',
                   'compress': True},
        provide_context=True,
        execution_timeout=timedelta(minutes=3)
    )

    load_transaction_types = PythonOperator(
        task_id=f'load-cccom-trans_types',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_trans_types', 'key': 'transaction_types.csv', 'compress': True},
        provide_context=True
    )

    merge_transaction_types = PostgresOperator(
        task_id='merge-cccom-trans_types',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_trans_types.sql'
    )

    extract_keywords = PythonOperator(
        task_id='extract-cccom-keywords',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_keywords.sql', 'key': 'keywords.csv', 'compress': True},
        provide_context=True,
        execution_timeout=timedelta(minutes=3)
    )

    load_keywords = PythonOperator(
        task_id=f'load-cccom-keywords',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_keywords', 'key': 'keywords.csv', 'compress': True},
        provide_context=True
    )

    merge_keywords = PostgresOperator(
        task_id='merge-cccom-keywords',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_keywords.sql'
    )

    extract_websites = PythonOperator(
        task_id='extract-cccom-websites',
        python_callable=mysql_table_to_s3,
        op_kwargs={'extract_script': 'cccom/extract_websites.sql', 'key': 'websites.csv', 'compress': True},
        provide_context=True,
        execution_timeout=timedelta(minutes=3)
    )

    load_websites = PythonOperator(
        task_id=f'load-cccom-websites',
        python_callable=s3_to_redshift,
        op_kwargs={'table': 'cccom_dw.stg_websites', 'key': 'websites.csv', 'compress': True},
        provide_context=True
    )

    merge_websites = PostgresOperator(
        task_id='merge-cccom-websites',
        postgres_conn_id=redshift_conn,
        sql='/sql/merge/cccom/merge_websites.sql'
    )

extract_sale_transactions >> load_sale_transactions >> merge_sale_transactions

extract_sale_rms_with_cutover_date >> load_sale_rms_with_cutover_date >> merge_sales_rms_with_cutover_date

merge_sale_transactions >> merge_sales_rms_with_cutover_date

extract_transaction_types >> load_transaction_types >> merge_transaction_types

extract_affiliates >> load_affiliates >> merge_affiliates

extract_keywords >> load_keywords >> merge_keywords

extract_websites >> load_websites >> merge_websites

extract_click_transactions >> load_click_transactions >> merge_click_transactions

extract_device_types >> load_device_types >> merge_device_types

extract_pages >> load_pages >> merge_pages

merge_transaction_types >> merge_sale_transactions

merge_affiliates >> merge_websites

merge_affiliates >> merge_sale_transactions

merge_keywords >> merge_click_transactions

merge_keywords >> merge_sale_transactions

merge_websites >> merge_click_transactions

merge_device_types >> merge_click_transactions

merge_pages >> merge_click_transactions


import torch
import torch.nn.functional as F

def max_shape(data):
    """Gets the maximum length along all dimensions in a list of Tensors"""
    shapes = torch.Tensor([_.shape for _ in data])
    return torch.max(shapes.transpose(0, 1), dim=1)[0].int()

def pad_data_to_same_shape(tensor_list, pad_value=0):
    target_shape = max_shape(tensor_list)

    padded_dataset_shape = [len(tensor_list)] + list(target_shape)
    padded_dataset = torch.Tensor(*padded_dataset_shape).type_as(
        tensor_list[0])

    for i, data in enumerate(tensor_list):
        # Get how much padding is needed per dimension
        padding = reversed(target_shape - torch.Tensor(list(data.shape)).int())

        # Add 0 every other index to indicate only right padding
        padding = F.pad(padding.unsqueeze(0).t(), (1, 0, 0, 0)).view(-1, 1)
        padding = padding.view(1, -1)[0].tolist()

        padded_data = F.pad(data, padding, value=pad_value)
        padded_dataset[i] = padded_data

    return padded_dataset

def fill_diagonally_(matrix,
                     diagonal_index,
                     fill_value=0,
                     fill_method='below'):
    """Destructively fills an nxm tensor somehow with respect to a diagonal.
    :param matrix:
    :type matrix: torch.Tensor
    :param diagonal_index:
    :param fill_value:
    :type fill_value: numeric
    :param fill_method:
    :type fill_method: str
    :return:
    """
    num_rows = matrix.shape[0]
    if fill_method == 'symmetric':
        mask = torch.ones(matrix.shape)
        fill_diagonally_(mask,
                         diagonal_index - 1,
                         fill_method='between',
                         fill_value=0)
        matrix[mask.byte()] = fill_value
        return

    for i in range(num_rows):
        if fill_method == 'below':
            left_bound = 0
            right_bound = min(num_rows, max(i - diagonal_index + 1, 0))
        elif fill_method == 'above':
            left_bound = min(num_rows, max(i - diagonal_index, 0))
            right_bound = num_rows
        elif fill_method == 'between':
            left_bound = min(num_rows, max(i - diagonal_index, 0))
            right_bound = min(num_rows, min(i + diagonal_index + 1, num_rows))
        else:
            msg = ('{} is an invalid fill_method. The fill_method must be in '
                   '\'below\', \'above\', \'symmetric\', \'between\'')
            raise ValueError(msg.format(fill_method))

        matrix[i, left_bound:right_bound] = fill_value


import boto3
import json
import os
import pandas as pd
from botocore.exceptions import ClientError
from urllib.request import urlopen

def simple_function(url, sheet_name):
    file_name = url.split('/')[-1]
    print('file_name', file_name)
    with urlopen(url) as response, open(file_name, 'wb') as out_file:
        data = response.read()
        out_file.write(data)
    df = pd.read_excel(file_name, sheet_name=sheet_name)
    df.fillna('', inplace=True)
    data = list(df.T.to_dict().values())
    json_data = json.dumps(data)
    try:
        os.remove(file_name)
        print('removed file {}'.format(file_name))
    except OSError:
        pass
    try:
        s3 = boto3.resource('s3')
        s3.Bucket(S3_BUCKET_NAME).put_object(Key='data.json', Body=json_data)
        return True
    except ClientError as e:
        print(e)
        return False

simple_function(url=FILE_URL, sheet_name=SHEET_NAME)


# coding=utf-8
import scrapy

from scrapy_splash import  SplashRequest
from scrapy.loader import ItemLoader
from wyy.items import WyyItem
import gzip
import zlib
import requests, json
from wyy.spiders import Decrypt
'''  js=string.format([[x=document.getElementsByClassName('nm nm-icn f-thide s-fc0');x[%d].click()]],args.same_az_id)
  splash:evaljs(js)'''
script = '''
function main(splash, args)
  splash.images_enabled= false
  local header=assert(splash:go{args.url,headers=args.headers})
  assert(splash:wait(args.wait))
  return header.headers

end
'''
class WyySpiderSpider(scrapy.Spider):
    name = 'wyy_spider'
    allowed_domains = ['music.163.com']
    country_id = {1, 2, 6, 7, 4}  # 分别代表 华语、欧美、日本、韩国、其他
    gender_id = {1, 2, 3}   # 分别代表男歌手、女歌手、组合
    cookies = {
        '_ntes_nnid': '7eced19b27ffae35dad3f8f2bf5885cd, 1476521011210',
        '_ntes_nuid': '7eced19b27ffae35dad3f8f2bf5885cd',
        'usertrack': 'c+5+hlgB7TgnsAmACnXtAg==',
        'Province': '025',
        'City': '025',
        'NTES_PASSPORT': '6n9ihXhbWKPi8yAqG.i2kETSCRa.ug06Txh8EMrrRsliVQXFV_orx5HffqhQjuGHkNQrLOIRLLotGohL9s10wcYSPiQfI2wiPacKlJ3nYAXgM',
        'P_INFO': 'hourui93@163.com|1476523293|1|study|11&12|jis&1476511733&mail163#jis&320100#10#0#0|151889&0|g37_client_check&mailsettings&mail163&study&blog|hourui93@163.com',
        'NTES_SESS': 'Fa2uk.YZsGoj59AgD6tRjTXGaJ8_1_4YvGfXUkS7C1NwtMe.tG1Vzr255TXM6yj2mKqTZzqFtoEKQrgewi9ZK60ylIqq5puaG6QIaNQ7EK5MTcRgHLOhqttDHfaI_vsBzB4bibfamzx1.fhlpqZh_FcnXUYQFw5F5KIBUmGJg7xdasvGf_EgfICWV',
        'S_INFO': '1476597594|1|0&80##|hourui93',
        'NETEASE_AUTH_SOURCE': 'space',
        'NETEASE_AUTH_USERNAME': 'hourui93',
        '_ga': 'GA1.2.1405085820.1476521280',
        'JSESSIONID-WYYY': 'cbd082d2ce2cffbcd5c085d8bf565a95aee3173ddbbb00bfa270950f93f1d8bb4cb55a56a4049fa8c828373f630c78f4a43d6c3d252c4c44f44b098a9434a7d8fc110670a6e1e9af992c78092936b1e19351435ecff76a181993780035547fa5241a5afb96e8c665182d0d5b911663281967d675ff2658015887a94b3ee1575fa1956a5a%3A1476607977016',
        '_iuqxldmzr_': '25',
        '__utma': '94650624.1038096298.1476521011.1476595468.1476606177.8',
        '__utmb': '94650624.20.10.1476606177',
        '__utmc': '94650624',
        '__utmz': '94650624.1476521011.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',

    }
    def start_requests(self):
        for country in self.country_id:
            for gender in self.gender_id:
                for i in range(65, 91):
                    url = 'https://music.163.com/discover/artist/cat?id={}00{}&initial={}'.format(
                            country, gender, i) #获取所有国家所有类型所有姓氏的url
                    yield scrapy.Request(url=url, headers=self.settings.get('HEADERS'), callback=self.parse,
                                        cookies=self.cookies,meta={'dont_merge_cookies':True,'url':url})

    # def parse(self, response):
    #     item = WyyItem()
    #     html = response.body.decode('utf-8')
    #     singers = response.xpath('''.//a[@class='nm nm-icn f-thide s-fc0']''')
    #     for same_az_id,singer in enumerate(singers):
    #         yield SplashRequest(response.meta['url'], callback=self.singer_parse, endpoint='execute', args={
    #             'lua_source': script, 'same_az_id': same_az_id, 'wait': 1,'headers':self.settings.get('HEADERS')
    #         })
    # def singer_parse(self, response):
    #     # 有的不需要解压 加个判断就好了
    #     html = response.body.decode('utf-8')
    #     try:
    #         decode_data = gzip.decompress(response.body.decode('utf-8')).decode('utf-8')
    #         #decode_data = zlib.decompress(response.body, 16 + zlib.MAX_WBITS)
    #         # print('用了gzip啊：{}'.format(decode_data))
    #     except:  # 请求失败的链接返回的是json 不能用gzip
    #         decode_data = response.body.decode()
    #     music_list = decode_data.xpath('''.//table[@class='m-table m-table-1 m-table-4']/tbody//tr''')
    #     for music in music_list:
    #         music_name = music.xpath('./td[2]//a/b/@title').extract_first('没拿到歌名?')
    #         music_url = music.xpath('./td[2]//a/@href').extract_first('没拿到song的herf?')
    #         music_id = music_url.replace('/song?id=', '').strip()
    #         real_music_url = 'https://music.163.com/song?id={}&limit=200'.format(music_id)

    def parse(self, response):
        item = WyyItem()
        html = response.body.decode('utf-8')
        singers = response.xpath('''.//a[@class='nm nm-icn f-thide s-fc0']''')
        for i,singer in enumerate(singers):
            #if i == 0:   #调试只拿一个歌手
                singer_name = singer.xpath('./text()').extract_first('没拿到name')
                singer_url = singer.xpath('./@href').extract_first('没拿到singer的herf?')
                singer_id = singer_url .replace('/artist?id=', '').strip()  # 获取歌手ID
                real_singer_url = 'https://music.163.com/artist?id={}'.format(singer_id)

                item['singer_name'] = singer_name
                item['singer_id'] = singer_id

                yield scrapy.Request(url=real_singer_url, callback=self.singer_parse,
                                     headers=self.settings.get('HEADERS'), cookies=self.cookies,
                                     meta={'dont_merge_cookies':True,'item':item})

    def singer_parse(self, response):
        item = response.meta['item']
        html = response.body.decode('utf-8')
        music_list = response.xpath('''.//ul[@class='f-hide']//li''')
        for i, music in enumerate(music_list):
            #if i==0:   #调试只取一首歌
                music_name = music.xpath('./a/text()').extract_first('没拿到歌名?')
                music_url = music.xpath('./a/@href').extract_first('没拿到song的herf?')
                music_id = music_url.replace('/song?id=', '').strip()
                real_music_url = 'https://music.163.com/song?id={}'.format(music_id)
                music_comment_url = "https://music.163.com/weapi/comment/resource/comments/get?csrf_token="
                item['music_name'] = music_name
                item['music_id'] = music_id
                item['music_url'] = real_music_url
                first_param = Decrypt.get_comment_first_param(music_id, 1)
                getkey = Decrypt.GetMusicKeyParameter(first_param, Decrypt.second_param,
                                                      Decrypt.third_param, Decrypt.forth_param, Decrypt.iv)
                params = getkey.get_params()
                encSecKey = getkey.get_encSecKey()
                data = {
                    "params": params,
                    "encSecKey": encSecKey
                }
                headers = {
                    'Host': 'music.163.com',
                    'Connection': 'keep-alive',
                    #'Content-Length': '484',
                    'Cache-Control': 'max-age=0',
                    'Origin': 'http://music.163.com',
                    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36',
                    'Content-Type': 'application/x-www-form-urlencoded',
                    'Accept': '*/*',
                    'DNT': '1',
                    'Accept-Encoding': 'gzip, deflate',
                    'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
                    'Cookie': 'JSESSIONID-WYYY=b66d89ed74ae9e94ead89b16e475556e763dd34f95e6ca357d06830a210abc7b685e82318b9d1d5b52ac4f4b9a55024c7a34024fddaee852404ed410933db994dcc0e398f61e670bfeea81105cbe098294e39ac566e1d5aa7232df741870ba1fe96e5cede8372ca587275d35c1a5d1b23a11e274a4c249afba03e20fa2dafb7a16eebdf6%3A1476373826753; _iuqxldmzr_=25; _ntes_nnid=7fa73e96706f26f3ada99abba6c4a6b2,1476372027128; _ntes_nuid=7fa73e96706f26f3ada99abba6c4a6b2; __utma=94650624.748605760.1476372027.1476372027.1476372027.1; __utmb=94650624.4.10.1476372027; __utmc=94650624; __utmz=94650624.1476372027.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',
                }
                # json_text = Decrypt.get_json(music_comment_url, params, encSecKey)
                # json_dict = json.loads(json_text)
                # print(json_dict['data']['totalCount'])

                # 在headers 中加入Content-Length 会400 Bad Request
                yield scrapy.FormRequest(url=music_comment_url, callback=self.song_parse, headers=headers,
                                         formdata=data,meta={'dont_merge_cookies':True,'item':item})
                # yield scrapy.Request(url=music_comment_url, callback=self.song_parse,method='POST',
                #                      headers=headers, cookies=self.cookies,
                #                      meta={'dont_merge_cookies':True,'item':item},body=json.dumps(body_data))

    def song_parse(self, response):
        item = response.meta['item']
        json_text = response.body
        json_dict = json.loads(json_text)
        music_comment_num = json_dict['data']['totalCount']
        item['music_comment_num'] = music_comment_num
        yield item


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import numpy as np
import matplotlib.pyplot as plt
import xarray as xr

ds=xr.open_dataset("data_netcdf/subspace_trans.nc")
# print(ds)
# tau = 3.0

plt.style.use('../nature_style.txt')

fig=plt.figure(figsize=(3.3,4.5),dpi=600) # figsize=(width,height(inch)),dpi(dots per inch)
ax1=fig.add_subplot(211)
var=ds.jkee
# vmax=np.abs(var).max()
vmax=np.abs(ds.jkie+ds.jkei).max()*0.2
quad1=ax1.pcolormesh(ds.kx,ds.ky,var,cmap="seismic",shading="auto",vmin=-vmax,vmax=vmax)
fig.colorbar(quad1,shrink=0.83,aspect=10)
ax1.set_xlim(-4,4)
ax1.set_ylim(0,5)
ax1.set_xlabel(r"Radial wave number $k_x \rho_\mathrm{ti}$")
ax1.set_ylabel(r"Poloidal wave number $k_y \rho_\mathrm{ti}$")
ax1.set_aspect("equal")
ax1.set_title(r"Electron-scale, electron-scale coupling $J_{\bf k}^{\Omega_e,\Omega_e}$",fontsize=7)

ax2=fig.add_subplot(212)
var=ds.jkie+ds.jkei
# vmax=np.abs(var).max()
quad2=ax2.pcolormesh(ds.kx,ds.ky,var,cmap="seismic",shading="auto",vmin=-vmax,vmax=vmax)
fig.colorbar(quad2,shrink=0.83,aspect=10)
ax2.set_xlim(-4,4)
ax2.set_ylim(0,5)
ax2.set_xlabel(r"Radial wave number $k_x \rho_\mathrm{ti}$")
ax2.set_ylabel(r"Poloidal wave number $k_y \rho_\mathrm{ti}$")
ax2.set_aspect("equal")
ax2.set_title(r"Ion-scale, electron-scale coupling $2J_{\bf k}^{\Omega_i,\Omega_e}$",fontsize=7)

fig.text(0.06,0.87,"a",color="k", fontfamily="sans-serif", fontweight="bold", fontsize=8)
fig.text(0.06,0.455,"b",color="k", fontfamily="sans-serif", fontweight="bold", fontsize=8)

plt.savefig("fig5.jpg",dpi=600)
quad1.set_rasterized(True)
quad2.set_rasterized(True)
plt.savefig('fig5.svg',bbox_inches="tight")
import subprocess
subprocess.call('inkscape fig5.svg -M fig5.emf', shell=True)
subprocess.call('inkscape fig5.svg -E fig5.eps', shell=True)
subprocess.call('inkscape fig5.svg -A fig5.pdf', shell=True)
plt.show()

# In[ ]:


from collections import deque

n,m = map(int, input().split())

graph = [[] for _ in range(n)]

for _ in range(m):
    u,v = map(int, input().split())
    graph[u-1].append(v-1)
    graph[v-1].append(u-1)

isTwoGraph = True
queue = deque([0])
colors = [0 for _ in range(n)]
colors[0] = 1

while queue:
    v = queue.popleft()
    for i in graph[v]:
        if colors[i]==0:
            queue.append(i)
            colors[i] = -1 * colors[v]
        elif colors[i]==colors[v]:
            isTwoGraph=False
            break

white = colors.count(1)
black = colors.count(-1)

if isTwoGraph:
    ans = n*(n-1)/2 - m - white*(white-1)/2 - black*(black-1)/2
    print(int(ans))
else:
    print(0)


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 17 10:05:47 2020

@author: acer
"""

#Projet info EIVP

import datetime
import matplotlib.pyplot as plt
import numpy as np
import os

os.getcwd()
monfichier = open('EIVP_KM.csv','r')
L = []
for ligne in monfichier.readlines():
    L += ligne.split('\t')
L = L[6:]
L = [i for i in L]
#print(L)

#L : liste de ttes les données sans les entêtes

#passer d'une chaîne de liste à une liste de str
def str_list(chaine):
    L = []
    char = ""
    for c in chaine:
        if c != ";":
            char += c
        elif char != "":
            L.append(char)
            char = ""
    L.append(char)
    return L

lig = len(L)    #nombre de lignes
col = len(L[0]) #nombre de colonnes
#Transformer les lignes en listes de str pour récupérer les données
for i in range(lig):
    L[i] = str_list(L[i])

ID = []
Noise = []
Temp = []
Hum = []
Lum = []
CO2 = []
Sent_at = []

Donnees = [ID, Noise, Temp, Hum, Lum, CO2, Sent_at]

#Fonction pour passer d'une date en str à une date et heure en list
def datetimelist(chaine):
    Date = []
    nb = ''
    for c in chaine:
        if c in "0123456789":
            nb += c
        elif nb != '':
            Date.append(int(nb))
            nb = ''
    if nb != '':
        Date.append(int(nb))
    return Date

for i in range(lig):
    for j in range(col):
        donnee = L[i][j]
        if j != 6:
            donnee = float(donnee)  #passer de str à nombre si possible
        else:
            donnee = donnee[:-7]    #pour enlever "+0200\n"

            date = datetimelist(donnee)     #avoir la date et l'heure en liste
            donnee = datetime.datetime(date[0],date[1],date[2],date[3],date[4],date[5])
            #pour avoir la date en classe Datetime pour être correctement exploitée
        Donnees[j].append(donnee)  #insérer dans les bonnes listes les bonnes données

#Là on a les différentes listes ID, Noise, etc avec leurs données numériques
#Pour Sent_at, on a les dates et heures en datetime.datetime

def niveau_sonore():
    minNoise = min(Noise)
    maxNoise = max(Noise)
    medianeNoise = np.median(Noise)
#    moyNoise = np.mean(Noise)
#    varNoise = np.var(Noise)
#    ecartypeNoise = varNoise**0.5

    print("Niveau sonore \n",)
    print("Minimum = "+str(minNoise))
    print("Maximum = "+str(maxNoise))
    print("Médiane = "+str(medianeNoise))
#    print("Moyenne = "+str(moyNoise))
#    print("Variance = "+str(varNoise))
#    print("Ecart-type = "+str(ecartypeNoise))
    print("\n")

    plt.figure()
    plt.plot(Sent_at,Noise)
#    plt.plot(Sent_at, lig*[moyNoise + ecartypeNoise], c='orange', lw=2, label='µ + sigma')
#    plt.plot(Sent_at, lig*[moyNoise - ecartypeNoise], c='orange', lw=2, label='µ - sigma')
    plt.title('Niveau sonore en fonction du temps')
    plt.xlabel('Temps')
    plt.ylabel('Niveau sonore (dBA)')
#    plt.legend()
    plt.show()

def température():
    minTemp = min(Temp)
    maxTemp = max(Temp)
    medianeTemp = np.median(Temp)
    moyTemp = np.mean(Temp)
    varTemp = np.var(Temp)
    ecartypeTemp = varTemp**0.5

    print("Température \n",)
    print("Minimum = "+str(minTemp))
    print("Maximum = "+str(maxTemp))
    print("Médiane = "+str(medianeTemp))
    print("Moyenne = "+str(moyTemp))
    print("Variance = "+str(varTemp))
    print("Ecart-type = "+str(ecartypeTemp))
    print("\n")

    plt.figure()
    plt.plot(Sent_at, Temp, c='blue')
    plt.plot(Sent_at, lig*[moyTemp + ecartypeTemp], c='orange', lw=2, label='µ + sigma')
    plt.plot(Sent_at, lig*[moyTemp - ecartypeTemp], c='orange', lw=2, label='µ - sigma')
    plt.title('Température en fonction du temps')
    plt.xlabel('Temps')
    plt.ylabel('Température (°C)')
    plt.legend()
    plt.show()

def humidité_relative():
    minHum = min(Hum)
    maxHum = max(Hum)
    medianeHum = np.median(Hum)
    moyHum = np.mean(Hum)
    varHum = np.var(Hum)
    ecartypeHum = varHum**0.5

    print("Humidité relative \n",)
    print("Minimum = "+str(minHum))
    print("Maximum = "+str(maxHum))
    print("Médiane = "+str(medianeHum))
    print("Moyenne = "+str(moyHum))
    print("Variance = "+str(varHum))
    print("Ecart-type = "+str(ecartypeHum))
    print("\n")

    plt.figure()
    plt.plot(Sent_at, Hum)
    plt.plot(Sent_at, lig*[moyHum + ecartypeHum], c='orange', lw=2, label='µ + sigma')
    plt.plot(Sent_at, lig*[moyHum - ecartypeHum], c='orange', lw=2, label='µ - sigma')
    plt.title('Humidité relative en fonction du temps')
    plt.xlabel('Temps')
    plt.ylabel('Humidité relative (%)')
    plt.legend()
    plt.show()

def luminosité():
    minLum = min(Lum)
    maxLum = max(Lum)
    medianeLum = np.median(Lum)
    moyLum = np.mean(Lum)
    varLum = np.var(Lum)
    ecartypeLum = varLum**0.5

    print("Luminosité \n",)
    print("Minimum = "+str(minLum))
    print("Maximum = "+str(maxLum))
    print("Médiane = "+str(medianeLum))
    print("Moyenne = "+str(moyLum))
    print("Variance = "+str(varLum))
    print("Ecart-type = "+str(ecartypeLum))
    print("\n")

    plt.figure()
    plt.plot(Sent_at, Lum)
    plt.plot(Sent_at, lig*[moyLum + ecartypeLum], c='orange', lw=2, label='µ + sigma')
    plt.plot(Sent_at, lig*[moyLum - ecartypeLum], c='orange', lw=2, label='µ - sigma')
    plt.title('Niveau lumineux en fonction du temps')
    plt.xlabel('Temps')
    plt.ylabel('Niveau lumineux (lux)')
    plt.legend()
    plt.show()

def co2():
    minCO2 = min(CO2)
    maxCO2 = max(CO2)
    medianeCO2 = np.median(CO2)
    moyCO2 = np.mean(CO2)
    varCO2 = np.var(CO2)
    ecartypeCO2 = varCO2**0.5

    print("Quantité de CO2 \n",)
    print("Minimum = "+str(minCO2))
    print("Maximum = "+str(maxCO2))
    print("Médiane = "+str(medianeCO2))
    print("Moyenne = "+str(moyCO2))
    print("Variance = "+str(varCO2))
    print("Ecart-type = "+str(ecartypeCO2))
    print("\n")

    plt.figure()
    plt.plot(Sent_at, CO2)
    plt.plot(Sent_at, lig*[moyCO2 + ecartypeCO2], c='orange', lw=2, label='µ + sigma')
    plt.plot(Sent_at, lig*[moyCO2 - ecartypeCO2], c='orange', lw=2, label='µ - sigma')
    plt.title('Quantité de CO2 en fonction du temps')
    plt.xlabel('Temps')
    plt.ylabel('Quantité de CO2 (ppm)')
    plt.legend()
    plt.show()

def corrélation(A,B):
    A = np.array(A)
    B = np.array(B)

    C = np.vstack((A,B))
    correl = np.corrcoef(C)[0,1]
    print("Coefficient de corrélation : ",correl)


#!/usr/bin/python

# (C) 2016 Tyler William H Backman
# Purpose: find PAINS in input compounds

import argparse
from rdkit import Chem
import sys
import string
from rdkit.Chem import FilterCatalog

parser = \
    argparse.ArgumentParser(description='find pains')
parser.add_argument('-i', '--infile', help='input sdf',
    required=True)
parser.add_argument('-o', '--outfile', help='output file',
    required=True)
parser.add_argument('-u', '--unparseable', help='output file for unparsable ids',
    required=True)
args = vars(parser.parse_args())
infile = args['infile']
outfile = args['outfile']
ufile = args['unparseable']

suppl = Chem.SDMolSupplier(infile)

params = FilterCatalog.FilterCatalogParams()
params.AddCatalog(params.FilterCatalogs.PAINS_A)
params.AddCatalog(params.FilterCatalogs.PAINS_B)
params.AddCatalog(params.FilterCatalogs.PAINS_C)
catalog = FilterCatalog.FilterCatalog(params)

f = open(outfile, 'w')
u = open(ufile, 'w')
count = 0
for mol in suppl:
    count += 1
    if mol is None:
        u.write(str(count) + "\n")
        continue
    try:
        if catalog.HasMatch(mol):
            f.write(mol.GetProp("_Name") + "\n")
    except:
        print("error in compound " + mol.GetProp("_Name") + "\n")

f.close()
u.close()


import numpy as np
import matplotlib.pyplot as plt
import time
import os

def get_cpg_values(mu, offset, omega, d, num_samples=1000):
    r = np.sqrt(mu)
    phi = np.pi * d
    o = offset
    gamma = 0.1
    dt = 0.001

    samples = []

    # startup_samples = 5000

    for sample in range(num_samples):
        for iteration in range(int(0.001 / dt)):
            d_r = gamma * (mu - r * r) * r
            d_phi = omega
            d_o = 0

            r += dt * d_r
            phi += dt * d_phi
            o += dt * d_o

        two_pi = 2 * 3.141592654

        phi_L = 0
        phi_2pi = phi % two_pi
        if phi_2pi < (two_pi * d):
            phi_L = phi_2pi / (2 * d)
        else:
            phi_L = (phi_2pi + two_pi * (1 - 2 * d)) / (2 * (1 - d))

        action = r * np.cos(phi_L) + o
        # if sample >= startup_samples:
        samples.append(action)

    return samples

def main():
    times = np.arange(0, 1, 0.001)
    amplitude = 30**2
    freq = 1
    cpg_values_1 = np.array(get_cpg_values(amplitude, 0, freq * 2 * np.pi, 0.1))
    cpg_values_3 = np.array(get_cpg_values(amplitude, 0, freq * 2 * np.pi, 0.5))
    cpg_values_5 = np.array(get_cpg_values(amplitude, 0, freq * 2 * np.pi, 0.9))

    plt.figure(figsize=(4,3))

    ax = plt.gca()
    ax.grid(False)

    ax.yaxis.set_ticks_position('left') # this one is optional but I still recommend it...
    ax.xaxis.set_ticks_position('bottom')

    # 1
    plt.plot(times, cpg_values_1, label='d=0.1', color='#78B833')
    plt.plot(times, cpg_values_3, label='d=0.5', color='#DD7611')
    plt.plot(times, cpg_values_5, label='d=0.9', color='#16A6C9')

    plt.plot(times, [0] * len(times), linestyle=':', color='#b0b0b0', linewidth=0.3)

    # 2
    # plt.plot(times, cpg_values_1, label='d=0.1', color='#763369')
    # plt.plot(times, cpg_values_3, label='d=0.5', color='#D84451')
    # plt.plot(times, cpg_values_5, label='d=0.9', color='#F7D302')

    plt.ylim(cpg_values_1.min()*1.1, cpg_values_1.max()*1.1)

    plt.xlabel('Time (seconds)')
    plt.ylabel('Motor position (degrees)')

    plt.legend(loc='upper left', frameon=False)
    plt.tight_layout()

    plt.savefig('/Users/Siebe/Dropbox/Thesis/presentation_plots/cpg_duty_factor.png', dpi=300)

if __name__ == '__main__':
    main()


# coding: utf-8
# This is Edje Electronics' code with some minor adjustments
# Credit goes to his repo: https://github.com/EdjeElectronics/TensorFlow-Lite-Object-Detection-on-Android-and-Raspberry-Pi

import tflite_runtime.interpreter as tflite
import os
import argparse
import cv2
import numpy as np
import sys
import time
from threading import Thread
import importlib.util
from datetime import datetime
from datetime import timedelta
import csv

parser = argparse.ArgumentParser()
parser.add_argument('--model', help='Provide the path to the TFLite file, default is models/model.tflite',
                    default='models/model.tflite')
parser.add_argument('--labels', help='Provide the path to the Labels, default is models/labels.txt',
                    default='models/labels.txt')
parser.add_argument('--video', help='Name of the video to perform detection on',
                    default='videos/2021-09-16-09-18-27.mp4')
parser.add_argument('--threshold', help='Minimum confidence threshold for displaying detected objects',
                    default=0.65)
parser.add_argument('--keepDetections', help='Saves detections (boxes, classes and scores) on frames',
                    default=False)

args = parser.parse_args()

# PROVIDE PATH TO MODEL DIRECTORY
PATH_TO_MODEL_DIR = args.model

# PROVIDE PATH TO LABEL MAP
PATH_TO_LABELS = args.labels

# PROVIDE PATH TO VIDEO DIRECTORY
VIDEO_PATH = args.video

# PROVIDE THE MINIMUM CONFIDENCE THRESHOLD
MIN_CONF_THRESH = float(args.threshold)

# Save detections on frames or not
keepDetections = args.keepDetections

import time
print('Loading model...', end='')
start_time = time.time()

# LOAD TFLITE MODEL
interpreter = tflite.Interpreter(model_path=PATH_TO_MODEL_DIR)
# LOAD LABELS
with open(PATH_TO_LABELS, 'r') as f:
    labels = [line.strip() for line in f.readlines()]
end_time = time.time()
elapsed_time = end_time - start_time
print('Done! Took {} seconds'.format(elapsed_time))

interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]

floating_model = (input_details[0]['dtype'] == np.float32)

input_mean = 127.5
input_std = 127.5

# Initialize frame rate calculation
frame_rate_calc = 1
freq = cv2.getTickFrequency()
print('Running inference for {}... '.format(VIDEO_PATH), end='')
# Initialize video
video = cv2.VideoCapture(VIDEO_PATH)
imW = video.get(cv2.CAP_PROP_FRAME_WIDTH)
imH = video.get(cv2.CAP_PROP_FRAME_HEIGHT)

# Variables propres à la vidéo
video_name = os.path.basename(VIDEO_PATH)
fps = video.get(cv2.CAP_PROP_FPS)
number_of_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
print('\nNombre d\'images par secondes (FPS) : ', fps)
print('Total number of frames : ', number_of_frames)
print('Durée de la vidéo : ', number_of_frames/fps, ' secondes')

# Dossier pour sauvegarder les frames (le créé s'il n'existe pas) :
frame_path = os.getcwd() + '/saved_frames/'
if not os.path.exists(frame_path):
  os.mkdir(frame_path)

# Les frames sont sauvegardées dans un dossier qui a le même nom que la vidéo courante
frame_path += video_name
if not os.path.exists(frame_path):
  os.mkdir(frame_path)

# Converti la liste de labels en dictionnaire pour compter les occurences des espèces
counting_detections = dict.fromkeys(labels, 0)

# Garde les N frames denières détections pour supprimer les faux positifs
N_save = 5 # Par défaut, si on veut que l'oiseau y soit au moins 1 seconde, on fixe cette valeur à la variable "fps"

# Pour le fichier CSV, qui a pour nom : CSV_nomVidéo (sans .mp4)
CSV_path = os.getcwd() + '/CSV_' + video_name[:-4] + '.csv'
print('CSV_path :', CSV_path)
header = ['videotime', 'class', 'xmin', 'ymin', 'xmax', 'ymax']

# Créé le fichier CSV en écriture
with open(CSV_path, 'w', newline='') as f:
    # Entête du fichier CSV seulement à la création du fichier
    writer = csv.DictWriter(f, delimiter=';', fieldnames=header)
    writer.writeheader()

# Parcours de la vidéo frame par frame
while(video.isOpened()):
    # Start timer (for calculating frame rate)
    current_count=0
    t1 = cv2.getTickCount()

    # Grab frame from video stream
    ret, frame1 = video.read()

    # Si on a bien lu une frame
    if ret:
        # Acquire frame and resize to expected shape [1xHxWx3]
        frame = frame1.copy()
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame_resized = cv2.resize(frame_rgb, (width, height))
        input_data = np.expand_dims(frame_resized, axis=0)

        # Sauvegarde le temps de la frame courante en secondes
        milliseconds = video.get(cv2.CAP_PROP_POS_MSEC)
        timestamp = timedelta(milliseconds=milliseconds)

        # Numéro de la frame courante
        frame_number = video.get(cv2.CAP_PROP_POS_FRAMES)
        if frame_number % 50 == 0:
            print('Treating frame n°', frame_number)

        # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
        if floating_model:
            input_data = (np.float32(input_data) - input_mean) / input_std

        # Perform the actual detection by running the model with the image as input
        interpreter.set_tensor(input_details[0]['index'],input_data)
        interpreter.invoke()

        # Retrieve detection results
        boxes = interpreter.get_tensor(output_details[0]['index'])[0] # Bounding box coordinates of detected objects
        classes = interpreter.get_tensor(output_details[1]['index'])[0] # Class index of detected objects
        scores = interpreter.get_tensor(output_details[2]['index'])[0] # Confidence of detected objects
        #num = interpreter.get_tensor(output_details[3]['index'])[0]  # Total number of detected objects (inaccurate and not needed)

        # Loop over all detections and draw detection box if confidence is above minimum threshold
        detected_classes = []
        detected_scores = []
        keep_frame = True

        # Pour le fichier CSV
        rows = []
        # Loop over all detections and draw detection box if confidence is above minimum threshold
        for i in range(len(scores)):
            if ((scores[i] > MIN_CONF_THRESH) and (scores[i] <= 1.0)):

                # Get bounding box coordinates and draw box
                # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
                ymin = int(max(1,(boxes[i][0] * imH)))
                xmin = int(max(1,(boxes[i][1] * imW)))
                ymax = int(min(imH,(boxes[i][2] * imH)))
                xmax = int(min(imW,(boxes[i][3] * imW)))

                # Trace la bounding box sur l'image
                cv2.rectangle(frame, (xmin,ymin), (xmax,ymax), (10, 255, 0), 2)

                # Draw label
                object_name = labels[int(classes[i])] # Look up object name from "labels" array using class index
                score = int(scores[i]*100)
                label = '%s: %d%%' % (object_name, score) # Example: 'person: 72%'
                labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
                label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
                cv2.rectangle(frame, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED) # Draw white box to put label text in
                cv2.putText(frame, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # Draw label text
                current_count+=1

                # Mets à jour le compteur de la classe actuelle dans le dictionnaire
                counting_detections[object_name] += 1
                # Enregistre les noms de classes et scores détectés
                detected_classes.append(object_name)
                detected_scores.append(scores[i])
                # Vérifie si l'espèce détectée n'est pas présente sur au moins N frames, on ne la garde pas
                if counting_detections[object_name] < N_save :
                    keep_frame = False

                # Données pour la/les nouvelle(s) ligne(s) pour le CSV
                rows.append({'videotime': timestamp, 'class': object_name, 'xmin': xmin, 'ymin': ymin, 'xmax': xmax, 'ymax': ymax})

        # Calculate framerate
        t2 = cv2.getTickCount()
        time1 = (t2-t1)/freq
        frame_rate_calc= 1/time1

        # Draw framerate in corner of frame
        cv2.putText(frame,'FPS on Pi: {0:.2f}, Detection(s) count: {1}'.format(frame_rate_calc, str(current_count)),(15,25),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,55),2,cv2.LINE_AA)
        # Si on a détecté une espèce au moins :
        if detected_classes and keep_frame :
          # Sauvegarde la frame avec les détections et le temps sur la vidéo (si ce temps n'est pas nul)
          # TODO les 5 ou 6 dernières frames ont un temps égal à 0.0 avec cv2 ???
          if milliseconds != 0.0 and keepDetections:
            # temps de la frame ==> str(timestamp)
            saved_frame_name = 'frame_' + str(int(frame_number)) + '.jpg'
            cv2.imwrite(os.path.join(frame_path, saved_frame_name), frame)
          elif not keepDetections:
            # Affiche les détections à la volée
            cv2.imshow('Object detector', frame)

        # Ouverture du fichier CSV pour ajouter des données à la suite
        with open(CSV_path, 'a', newline='') as f:
            # Sauvegarde le nom de l'image, la/les espèce(s) et les coordonées des bounding boxes dans le CSV
            writer = csv.DictWriter(f, delimiter=';', fieldnames=header)
            # Si on a au moins une détection
            if rows != []:
                # Une ligne par espèce pour l'image courante
                for l in rows:
                    writer.writerow(l)

        # Press 'q' to quit
        if not keepDetections:
            # Attend moins de temps pour sauvegarder les images/frames
            if cv2.waitKey(1) == ord('q'):
                break
        else:
            # Affiche une image par seconde (1000ms)
            if cv2.waitKey(1000) == ord('q'):
                break

    # Si c'est la fin de la vidéo
    else:
        video.release()

# Affichage finaux
if os.path.exists(frame_path):
  print('Saved frames : ', len(os.listdir(frame_path)))

total_seconds = number_of_frames / frame_rate_calc
print('Average total time :', timedelta(seconds=total_seconds))
print('Detected species : ', counting_detections)

# Clean up
cv2.destroyAllWindows()
print("Done")


# -*- coding: utf-8 -*-

import time
from datetime import date, datetime, timedelta
import logging
import difflib
from openerp import models, fields, api, _
from openerp.osv import fields, osv
from openerp import models, fields
import re
import codecs
from openerp.tools.translate import _
from openerp.exceptions import except_orm, Warning, RedirectWarning
import math
from openerp.tools import float_compare
import openerp.addons.decimal_precision as dp
import time
from datetime import date, datetime, timedelta
_logger = logging.getLogger(__name__)

class Presupuesto(models.Model):
	_name = 'presupuesto.tools'
	_description = 'Reusable Methods'

	def get_saldo(self, presupuesto_move_id):

		saldo_total = 0
		move_val = 0
		saldo_rel = 0

		if presupuesto_move_id:

			saldo_total = presupuesto_move_id.amount_total

			presupuesto_moverubros_pool = self.env['presupuesto.moverubros']

			presupuesto_moverubros_ids = presupuesto_moverubros_pool.search([('move_id', '=', presupuesto_move_id.id),
										('move_id.state', '=', 'confirm'),
										('move_id.fiscal_year', '=', presupuesto_move_id.fiscal_year.id),
										])

			presupuesto_moverubros_relacionados_ids = presupuesto_moverubros_pool.search([('move_rel_id', '=', presupuesto_move_id.id),
							('move_id.state', '=', 'confirm'),
							('move_id.fiscal_year', '=', presupuesto_move_id.fiscal_year.id)])

			if presupuesto_moverubros_relacionados_ids:

				for x in presupuesto_moverubros_relacionados_ids:
					move_val = move_val + x.ammount

			if presupuesto_moverubros_ids:

				for x in presupuesto_moverubros_ids:
					saldo_rel = saldo_rel + x.ammount

		return (saldo_rel - move_val) if saldo_rel else move_val

	def get_saldo_obligaciones(self, presupuesto_move_id):

		saldo_total = 0
		move_val = 0
		total = 0

		if presupuesto_move_id:

			presupuesto_moverubros_pool = self.env['presupuesto.moverubros']

			presupuesto_move_ids = presupuesto_moverubros_pool.search([('move_rel_id', '=', presupuesto_move_id.id),
							('move_id.state', '=', 'confirm'),
							('move_id.fiscal_year', '=', presupuesto_move_id.fiscal_year.id)])

			saldo_total = presupuesto_move_id.amount_total
			total = saldo_total
			if presupuesto_move_ids:

				for x in presupuesto_move_ids:

					move_val = move_val + x.ammount

				total = saldo_total - move_val if move_val else saldo_total

			presupuesto_move_id.write({'saldo_sin_utilizar': total})
			_logger.info(total)
			return total

	"""
		metodo que nos sirve para saber si en los rubros que tiene el cdp, compromiso
		o obligacion el monto de los gastos es igual.

		recibe los ids de los rubros.

		retorna la diferencia entre el presupuesto y los gastos
	"""

	def _get_diff_money(self, ids):

		if ids:
			saldo_move = 0
			ammount = 0
			for data in ids:
				saldo_move += data.saldo_move
				ammount += data.ammount
			_logger.info(saldo_move)
			_logger.info(ammount)
			return saldo_move - ammount

	def update_old_values(self):

		presupuesto_move_pool = self.env['presupuesto.move']
		presupuesto_moverel_pool = self.env['presupuesto.moverubros']
		presupuesto_move_pool_ids = []

		sql = """
			SELECT id AS id
			FROM presupuesto_move
			WHERE move_rel IS NOT NULL
		"""

		self.env.cr.execute( sql )
		res = self.env.cr.dictfetchall(  )

		if res:

			for x in res:

				move_id = presupuesto_move_pool.browse(x.get('id'))

				if move_id:

					values = {

						'presupuesto_rel_move': [(6, 0, [move_id.move_rel.id])]

					}

					move_id.write(values)

					presupuesto_moverel_id = presupuesto_moverel_pool.search([('move_id', '=', x.get('id'))])

					if presupuesto_moverel_id:

						for z in presupuesto_moverel_id:

							z.write({'move_rel_id': move_id.move_rel.id})

	def uptdate_old_values_account_invoice(self):

		sql = """
			SELECT id AS id
			FROM account_invoice
			WHERE rp IS NOT NULL
		"""

		self.env.cr.execute( sql )
		res = self.env.cr.dictfetchall(  )

		account_invoice_pool = self.env['account.invoice']

		if res:

			for x in res:

				account_invoice_id = account_invoice_pool.browse(x.get('id'))

				if account_invoice_id:

					values = {

						'rp_move_rel': [(6, 0, [account_invoice_id.rp.id])]

					}

					account_invoice_id.write(values)

	def uptdate_old_values_account_voucher(self):

		sql = """
			SELECT id AS id
			FROM account_voucher
			WHERE obl IS NOT NULL
		"""

		self.env.cr.execute( sql )
		res = self.env.cr.dictfetchall(  )

		account_voucher_pool = self.env['account.voucher']

		if res:

			for x in res:

				account_voucher_id = account_voucher_pool.browse(x.get('id'))

				if account_voucher_id:

					values = {

						'obl_move_rel': [(6, 0, [account_voucher_id.obl.id])]

					}

					account_voucher_id.write(values)

	def uptdate_old_values_contact(self):

		sql = """
			SELECT id AS id
			FROM hr_contract
			WHERE cdp IS NOT NULL
		"""

		self.env.cr.execute( sql )
		res = self.env.cr.dictfetchall(  )

		hr_contract_pool = self.env['hr.contract']

		if res:

			for x in res:

				hr_contract_id = hr_contract_pool.browse(x.get('id'))

				if hr_contract_id:

					values = {

						'cdp_move_rel': [(6, 0, [hr_contract_id.cdp.id])]

					}

					hr_contract_id.write(values)

	def uptdate_old_values_payslip(self):

		sql = """
			SELECT id AS id
			FROM hr_payslip
			WHERE obl IS NOT NULL
		"""

		self.env.cr.execute( sql )
		res = self.env.cr.dictfetchall(  )

		hr_payslip_pool = self.env['hr.payslip']

		if res:

			for x in res:

				hr_payslip_id = hr_payslip_pool.browse(x.get('id'))

				if hr_payslip_id:

					values = {

						'obl_move_rel': [(6, 0, [hr_payslip_id.obl.id])]

					}

					hr_payslip_id.write(values)


import re

from pygments import highlight
from pygments.formatters import HtmlFormatter, ClassNotFound
from pygments.lexers import get_lexer_by_name
import misaka

class HighlighterRenderer(misaka.HtmlRenderer):
    def blockcode(self, text, lang):
        try:
            lexer = get_lexer_by_name(lang, stripall=True)
        except ClassNotFound:
            lexer = None

        if lexer:
            formatter = HtmlFormatter()
            return highlight(text, lexer, formatter)
        # default
        return '\n
{}
\n'.format(text.rstrip())

md = misaka.Markdown(
    HighlighterRenderer(),
    extensions=('fenced-code', 'tables'))
section_tpl = '
{section}
'
hline_re = re.compile('\n---\s*\n')

def read_file(path):
    content = open(path).read()
    for chunk in hline_re.split(content):
        yield chunk

def to_html(chunk_id, chunk):
    section = md(chunk)
    return section_tpl.format(id=chunk_id, section=section)

def main():
    content = '\n'.join(
        to_html(cid, chunk)
        for cid, chunk in enumerate(read_file('tanker.md')))
    page_tpl = open('tanker-tpl.html').read()
    print(page_tpl % {'article': content, 'title': 'Tanker'})

if __name__ == '__main__':
    main()


import re

from setuptools import setup, find_packages

with open('daiquiri_sync/__init__.py') as f:
    metadata = dict(re.findall(r'__(.*)__ = [\']([^\']*)[\']', f.read()))

setup(
    name='django-daiquiri-sync',
    version=metadata['version'],
    author=metadata['author'],
    author_email=metadata['email'],
    maintainer=metadata['author'],
    maintainer_email=metadata['email'],
    license=metadata['license'],
    url='https://github.com/aipescience/django-daiquiri-sync',
    description=u'Daiquiri Sync is a python library meant to be used with the Daiquiri Framework.',
    long_description=open('README.rst').read(),
    install_requires=[
        'ansible',
        'paramiko',
        'PyYAML'
    ],
    classifiers=[],
    packages=find_packages(),
    include_package_data=True
)


import logging
import numpy as np
import pandas as pd
from pathlib import Path
from src.data.read_dataset import get_data
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.compose import ColumnTransformer
from src.data.text_preprocessing import convert_labels

project_dir = Path(__file__).resolve().parents[2]

class BaselinePredict:
    """
    Provides a classic baseline for comparison
    """

    def __init__(self, model_name, data_source):
        self._data = data_source
        self._model = self.train(model_name)

    def my_predict(self):
        """Predict the binary class of a sentence using a Logistic Regression
        Args:
            Data (pd.DataFrame): DataFrame
        Returns:
            binary class (str): Apartment (class 0) | House (class 1)
        """
        # predict
        return self._model

    def train(self, model) -> str:
        """Train a logistic regression method"""
        #try:

        # Get data
        df_train, _ = get_data(self._data)

        # Removing label
        train = df_train.drop(["label"], axis=1)
        #train = train.drop(["ppl"], axis=1)

        # Numerical transformations
        numeric_transformer = Pipeline(steps=[
            ("imputer", SimpleImputer(strategy="median"))
        ])
        # Categorical transformations
        categorical_transformer = Pipeline(steps=[
            ("encoder", OneHotEncoder(handle_unknown="ignore"))
        ])
        # Text transformations
        text_transformer = Pipeline(steps=[
            ('vect', CountVectorizer())
            #('tfidf', TfidfTransformer())
        ])

        # Numerical features
        # numeric_features = train.select_dtypes(include=['int64', 'float64']).columns

        # Categorical features
        categorical_features = train.select_dtypes(include=['object']).columns

        # Convert in a column
        preprocessor = ColumnTransformer(
            transformers=[
                #('num', numeric_transformer, 'ppl'),
                ('tex', text_transformer, 'text'),
                ('cat', categorical_transformer, categorical_features)
            ])

        # Pipeline
        pipeline = Pipeline(steps=[
            ('preprocessor', preprocessor),
            ('lr', LogisticRegression(max_iter=1000, multi_class="multinomial")) #solver="sag",
        ])

        # labels
        label2int = convert_labels(df_train["label"])
        train_labels = df_train['label'].apply(lambda x: label2int[x])

        # fit
        pipeline.fit(train, train_labels)
        print(pipeline.feature_names_in_)

        return pipeline

        #except Exception:
        #    logging.error(f'directory or model is invalid or does not exist: {self._model}')


import pytest
from graph_scheduler import *

def test_execution_engine_main(tmpdir):

    import modeci_mdf.execution_engine
    from modelspec.utils import FORMAT_NUMPY, FORMAT_TENSORFLOW
    import numpy as np

    mdf_formats = ["json", "yaml"]
    array_formats = [FORMAT_NUMPY, FORMAT_TENSORFLOW]

    # For now, don't make tensorflow a requiremnt...
    try:
        import tensorflow
    except:
        array_formats = [FORMAT_NUMPY]

    for mdf_format in mdf_formats:
        for array_format in array_formats:

            eg = modeci_mdf.execution_engine.main(
                "examples/MDF/Simple.%s" % mdf_format, array_format=array_format
            )
            output = (
                eg.enodes["processing_node"].evaluable_outputs["output_1"].curr_value
            )
            assert output == 0.6016871801828567

            eg = modeci_mdf.execution_engine.main(
                "examples/MDF/ABCD.%s" % mdf_format, array_format=array_format
            )
            output = eg.enodes["D"].evaluable_outputs["output_1"].curr_value
            assert -1.7737500239216304 - output < 1e-9

            eg = modeci_mdf.execution_engine.main(
                "examples/MDF/Arrays.%s" % mdf_format, array_format=array_format
            )
            output = eg.enodes["middle_node"].evaluable_outputs["output_1"].curr_value
            assert output[0, 0] == 0.5
            assert output[1, 1] == 4

            eg = modeci_mdf.execution_engine.main(
                "examples/MDF/States.%s" % mdf_format, array_format=array_format
            )
            output = eg.enodes["counter_node"].evaluable_outputs["out_port"].curr_value
            assert output == 1
            output = eg.enodes["sine_node"].evaluable_outputs["out_port"].curr_value
            assert output == 0

def test_execution_engine_onnx(tmpdir):

    import modeci_mdf.execution_engine
    import numpy as np

    mdf_formats = ["json", "yaml"]
    from modelspec.utils import FORMAT_NUMPY

    array_format = FORMAT_NUMPY

    for mdf_format in mdf_formats:

        eg = modeci_mdf.execution_engine.main(
            "examples/ONNX/ab.%s" % mdf_format, array_format=array_format
        )
        output = eg.enodes["Mul_3"].evaluable_outputs["_4"].curr_value
        assert output == 5

_abc_conditions_expected_output = [
    {"input0"},
    {"A"},
    {"A"},
    {"B"},
    {"A"},
    {"C"},
    {"A"},
    {"B"},
    {"A"},
    {"A"},
    {"C", "B"},
    {"A"},
]

@pytest.mark.parametrize(
    "fi, expected_output",
    [
        ("examples/MDF/abc_conditions.json", _abc_conditions_expected_output),
        ("examples/MDF/abc_conditions.yaml", _abc_conditions_expected_output),
    ],
)
def test_condition_scheduler_main(fi, expected_output):

    import modeci_mdf.execution_engine

    eg = modeci_mdf.execution_engine.main(fi)
    output = [{n.id for n in nodes} for nodes in eg.scheduler.execution_list[None]]
    assert output == expected_output

    assert eg.enodes["A"].evaluable_parameters["count_A"].curr_value == 7
    assert eg.enodes["B"].evaluable_parameters["count_B"].curr_value == 3


from flask import Flask,render_template,request

app=Flask(__name__)
app.config['SECRET_KEY'] = "cc7c83c25c81641609b40dbaf2038e12"

@app.route('/', methods=['GET', 'POST'])
def home():
    return render_template("index.html")

@app.route('/about', methods=['GET', 'POST'])
def about():
    return render_template("about.html")

@app.route('/help', methods=['GET', 'POST'])
def help():
    return render_template("help.html")

@app.route('/CheckExam', methods=['GET', 'POST'])
def CheckExam():
    examtime=0
    result=0
    seatlocation=0
    print(type(request.form.get("Examdatedept")))
    print(request.form.get("Examdateyr"))
    print(request.form.get("checkseatrollno"))
    print(request.form.get("checkseatdept"))
    print(request.form.get("checkseatyr"))

    if request.form.get("Examdatedept")=="BSCIT" or request.form.get("Examdatedept")=="BSCCS":
        if request.form.get("Examdateyr")=="first":
            examtime=[["10-apr","11.00 am-1.00 pm","Maths"],["11-apr","11.00 am-1.00 pm","Microprocessor"],["12-apr","11.00 am-1.00 pm","Graphics"],
                        ["14-apr","11.00 am-1.00 pm","OS"],["15-apr","11.00 am-1.00 pm","Network"],["16-apr","11.00 am-1.00 pm","OOPS"]]
        elif request.form.get("Examdateyr")=="second":
            examtime=[["10-apr","09.00 am-11.00 am","Java"],["11-apr","09.00 am-11.00 am","Python"],["12-apr","09.00 am-11.00 am","embedded"],
                        ["14-apr","09.00 am-11.00 am","Computing"],["15-apr","09.00 am-11.00 am","WebProj"],["16-apr","09.00 am-11.00 am","Network"]]
        elif request.form.get("Examdateyr")=="final":
            examtime=[["10-apr","02.00 pm-04.00 pm","IOT"],["11-apr","02.00 pm-04.00 pm","Cyberlaw"],["12-apr","02.00 pm-04.00 pm","NGT"],
                        ["14-apr","02.00 pm-04.00 pm","SIC"],["15-apr","02.00 pm-04.00 pm","GIS"],["16-apr","02.00 pm-04.00 pm","BI"]]
        else:
            examtime=0
        return render_template("exam.html",exam=1,seat=0,examdetails=examtime,location=seatlocation)

    seatingarr={"bscit":{"first":[{"roomno":"IT-10","rollno":[0,25]},{"roomno":"IT-11","rollno":[25,50]}],
                    "second":[{"roomno":"IT-20","rollno":[0,25]},{"roomno":"IT-21","rollno":[25,50]}],
                    "final":[{"roomno":"IT-30","rollno":[0,25]},{"roomno":"IT-31","rollno":[25,50]}]
                    },
                "bsccs":{"first":[{"roomno":"CS-10","rollno":[0,25]},{"roomno":"CS-11","rollno":[25,50]}],
                        "second":[{"roomno":"CS-20","rollno":[0,25]},{"roomno":"CS-21","rollno":[25,50]}],
                        "final":[{"roomno":"CS-30","rollno":[0,25]},{"roomno":"CS-31","rollno":[25,50]}]
                    }
                }

    if request.form.get("checkseatdept")=="BSCIT":
        if str(request.form.get("checkseatrollno"))!="None" and str(request.form.get("checkseatrollno"))!="":
            rollno=str(request.form.get("checkseatrollno"))
            if rollno[0:2]=="IT":
                result=seatingarr["bscit"]
            if rollno[0:2]=="CS":
                result=seatingarr["bsccs"]
            if rollno[2:4]=="01":
                result=result["first"]
            if rollno[2:4]=="02":
                result=result["second"]
            if rollno[2:4]=="03":
                result=result["final"]
            lastdigit=rollno[4:6]
            for norange in result:
                if int(lastdigit)<=norange["rollno"][1] and int(lastdigit)>norange["rollno"][0]:
                    seatlocation=norange["roomno"]
        return render_template("exam.html",exam=0,seat=1,location=seatlocation, examdetails=examtime)
    return render_template("exam.html",exam=0,seat=0,location=seatlocation,examdetails=examtime)

@app.route('/FillFees', methods=['GET', 'POST'])
def FillFees():
    return render_template("adminfees.html")

@app.route('/RegisterEvents', methods=['GET', 'POST'])
def RegisterEvents():
    return render_template("events.html")

if __name__=="__main__":
    app.run(host='0.0.0.0', port=8080, debug=True)


''''
SuperCRUNCH: Contamination_Filter module

    Contamination_Filter: Create a blast database from 'contamination' sequences
    and blast an empirical file to this database. Any query sequence that scores
    a pident of > 95% with length > 100 bp is considered to be a match to a
    'contamination' sequence. These bad sequences are written to a separate fasta file,
    whereas all other passing sequences are written to a new filtered fasta file.

    For example, use human mtDNA as the contamination seq database and blast
    sequences of a different group of organisms (reptiles, amphibians, birds,
    etc) to the database to look for human contaminated sequences.

    The type of blast search is defined by the user. For the purpose of finding
    contaminated sequences, megablast may be preferable.

    Input fasta files should be labeled as 'NAME.fasta' or 'NAME.fa'. The
    NAME portion should not contain any periods or spaces, but can contain
    underscores. Output files are labeled using a prefix identical to NAME.

    Note - The blast output format 6 is tab-delimited with the following
           'hidden' headers:
    qseqid sseqid pident length mismatch gapopen qstart qend sstart send evalue bitscore

-------------------------
Compatible with Python 2.7 & 3.7
Dependencies:
	-blast+ (installed in path)
Python packages required:
	-BioPython
-------------------------

SuperCRUNCH project
https://github.com/dportik/SuperCRUNCH
Written by Daniel Portik
daniel.portik@gmail.com
January 2019
Distributed under the
GNU General Public Lincense
'''
import shutil
import os
import subprocess as sp
import argparse
import operator
from datetime import datetime
from Bio import SeqIO

def get_args():
    """
    Get arguments from command line.
    """
    parser = argparse.ArgumentParser(
            description="""---------------------------------------------------------------------------
    Contamination_Filter: Create a blast database from 'contamination' sequences
    and blast an empirical file to this database. Any query sequence that scores
    a pident of > 95% with length > 100 bp is considered to be a match to a
    'contamination' sequence. These bad sequences are written to a separate fasta file,
    whereas all other passing sequences are written to a new filtered fasta file.

    For example, use human mtDNA as the contamination seq database and blast
    sequences of a different group of organisms (reptiles, amphibians, birds,
    etc) to the database to look for human contaminated sequences.

    The type of blast search is defined by the user. For the purpose of finding
    contaminated sequences, megablast may be preferable.

    Input fasta files should be labeled as 'NAME.fasta' or 'NAME.fa'. The
    NAME portion should not contain any periods or spaces, but can contain
    underscores. Output files are labeled using a prefix identical to NAME.

    DEPENDENCIES: Python: BioPython; Executables in path: blast+ tools
    (makeblastdb, blastn).

	---------------------------------------------------------------------------""")

    parser.add_argument("-i", "--indir",
                            required=True,
                            help="REQUIRED: The full path to a directory containing the "
                            "reference and the empirical fasta files")

    parser.add_argument("-o", "--outdir",
                            required=True,
                            help="REQUIRED: The full path to an existing directory to write "
                            "output files.")

    parser.add_argument("-e", "--emp_fasta",
                            required=False,
                            help="OPTION 1: The name of an empirical fasta to blast to "
                            "the database to identify 'contaminated' sequences. File name "
                            "only (rather than full path), should be located in the input "
                            "directory. Follow labeling format: NAME.fasta")

    parser.add_argument("-d", "--blast_db",
                            required=False,
                            help="OPTION 1: The name of the reference fasta file (of "
                            "sequences from the contamination source) that "
                            "will be used to create the blast database. File name only "
                            "(rather than full path), should be located in the input directory.")

    parser.add_argument("--multisearch",
                            required=False,
                            help="OPTION 2: If multiple empirical fasta + blast database "
                            "pairs are present in the -i (input) directory, provide the full path "
                            "to a tab-delimited text file containing the pair information "
                            "to allow all of them to run sequentially. See documentation.")

    parser.add_argument("-b", "--blast_task",
                            required=True,
                            choices=["blastn", "blastn-short", "dc-megablast", "megablast"],
                            help="REQUIRED: The type of blast search to conduct.")

    parser.add_argument("--max_hits",
                            type=int,
                            default=None,
                            help="OPTIONAL: The maximum number of blast matches allowed "
                            "per input sequence.")

    return parser.parse_args()

def make_db(blast_db):
    """
    Generates call string for makeblastd and then
    runs using sp.call() with shell option.
    """
    print("\n---------------------------------------------------------------------------\n")

    mdb_str =  "makeblastdb -in {} -dbtype nucl".format(blast_db)

    print("\n\n{}".format(mdb_str))
    proc = sp.call(mdb_str, shell=True)

def blastn_to_db(task, blast_db, emp_fasta, outname, max_hits):
    """
    Generates call string for blastn for file (f) using the
    supplied arguments. Runs using sp.call() with shell
    option. Returns the name of the output file generated by
    the program.
    """
    print("\n---------------------------------------------------------------------------\n")

    if max_hits is None:
    	blast_str = ("blastn -task {0} -db {1} -query {2} -outfmt 6 > {3}"
                         .format(task, blast_db, emp_fasta, outname))
    else:
    	blast_str = ("blastn -task {0} -db {1} -query {2} -outfmt 6 -max_target_seqs {3} > {4}"
                         .format(task, blast_db, emp_fasta, max_hits, outname))

    print("\n\n{}".format(blast_str))
    proc = sp.call(blast_str, shell=True)

def parse_blastn_output6(outname):
    """
    Get information from blast output file.
    """
    print("\n---------------------------------------------------------------------------\n")

    #Store the file contents of the blast output file
    #as a split and stripped list.
    print("Parsing contents in {}...\n".format(outname))
    file_contents = []

    with open(outname, 'r') as fh_in:
        for line in fh_in:
            temp_list = []
            #list comprehension to strip whitespace from all items results from splitting
            #by the tab character, resulting in a list of file line contents
            parts = [l.strip() for l in line.split('\t')]
            file_contents.append(parts)

    #sort blast output sublists by reverse order of 'pident'
    file_contents.sort(key=operator.itemgetter(2), reverse=True)

    #create accession lists for contaminated and good seqs
    bad_seqs = []
    #iterate over sorted file contents list
    for l in file_contents:
        #test if pident is > 95% and length is > 100bp (matching contaminated seq)
        if float(l[2]) > 95.0 and int(l[3]) > 100:
            bad_seqs.append(l[0])

    return bad_seqs

def pull_records_from_accn_lists(emp_fasta, bad_seqs):
    """
    Function to write sequences to new fasta file based
    on whether they passed the contam filter or not.
    """
    #load the fasta file using the indexing function
    #will create a dictionary with accn as keys
    record_dict = SeqIO.index(emp_fasta, "fasta")

    prefix = emp_fasta.split('.')[0]

    #initiate counts for records written
    goodcount, badcount = int(0), int(0)

    print("\n---------------------------------------------------------------------------")
    print("\nWriting sequences that passed contamination filter...\n")

    #first write sequences that passed filter
    goodfasta = "{}_contfiltered.fasta".format(prefix)
    with open(goodfasta, 'a') as fh:
        #iterate over seq records
        for acc in record_dict:
            #check if accession number is not present in bad seq list
            if acc not in bad_seqs:
                #test if number of accns processed is divisible by 100, if so print number (ie progress report for big files)
                if goodcount % int(100) == 0:
                    print("\t\t\tFinished writing {:,} records...".format(goodcount))
                #write record to fasta
                fh.write(">{}\n{}\n".format(record_dict[acc].description, record_dict[acc].seq))
                #add to counter
                goodcount += 1

    print("\n---------------------------------------------------------------------------")
    print("\nWriting contaminated sequences...\n")

    #second write any sequences that failed the filter
    badfasta = "{}_contaminated.fasta".format(prefix)
    with open(badfasta, 'a') as fh:
        if len(bad_seqs) >= int(1):
            #iterate over list with accns and coordinates
            for acc in bad_seqs:
                #write record to fasta
                fh.write(">{}\n{}\n".format(record_dict[acc].description, record_dict[acc].seq))
                #add to counter
                badcount += 1
            print("\t\t\tFinished writing {} records...".format(badcount))

    #log for bad record IDs
    logname = "{}_Contaminated_Seq_Summary.txt".format(prefix)
    with open(logname, 'a') as fh_out:
        fh_out.write("Wrote a total of {0} sequences to {1}\n\n".format(goodcount, goodfasta))
        fh_out.write("Wrote a total of {0} sequences to {1}, including:\n\n".format(badcount, badfasta))
        if len(bad_seqs) >= int(1):
            for acc in bad_seqs:
                #write record to fasta
                fh_out.write("{}\n".format(record_dict[acc].description))

    print("\n\nWrote a total of {0:,} sequences to {1}".format(goodcount, goodfasta))
    print("Wrote a total of {0:,} sequences to {1}".format(badcount, badfasta))

def cleanup(blastdir, cdir, fdir):
    """
    Moves relevant output files to their output directories.
    """
    print("\nCleaning up output files...")

    [os.remove(f) for f in os.listdir('.') if f.endswith('contaminated.fasta') and os.stat(f).st_size == 0]
    [shutil.move(f, cdir) for f in os.listdir('.') if f.endswith("contaminated.fasta")]
    [shutil.move(f, blastdir) for f in os.listdir('.') if f.endswith(("blast_results.txt", "Contaminated_Seq_Summary.txt"))]
    [shutil.move(f, fdir) for f in os.listdir('.') if f.endswith("contfiltered.fasta")]
    [os.remove(f) for f in os.listdir('.') if f.endswith((".nsq", ".nhr", ".nin", ".ndb", ".not", ".ntf", ".nto"))]

    print("\tDone!\n")

def make_dirs(outdir):
    """
    Creates directory path names and makes output directories.
    Returns directory paths, which are used to move around
    output files during cleanup steps.
    """
    os.chdir(outdir)
    curpath = os.getcwd()

    blastdir = os.path.join(curpath, "Blast-Output-Files")
    if not os.path.exists(blastdir):
        os.mkdir(blastdir)

    cdir = os.path.join(curpath, "Contaminated-Sequences")
    if not os.path.exists(cdir):
        os.mkdir(cdir)

    fdir = os.path.join(curpath, "Filtered-Fasta-Files")
    if not os.path.exists(fdir):
        os.mkdir(fdir)

    return blastdir, cdir, fdir

def main():
    args = get_args()
    tb = datetime.now()

    blastdir, cdir, fdir = make_dirs(args.outdir)

    os.chdir(args.indir)
    '''
    make_db(args.blast_db)

    outname = "{}_blast_results.txt".format((args.emp_fasta).split('.')[0])
    blastn_to_db(args.blast_task, args.blast_db, args.emp_fasta, outname, args.max_hits)

    bad_seqs = parse_blastn_output6(outname)
    pull_records_from_accn_lists(args.emp_fasta, bad_seqs)

    cleanup(blastdir, cdir, fdir)

    tf = datetime.now()
    te = tf - tb
    print("\n\n--------------------------------------------------------------------------------------")
    print("\nTotal time to process {0}: {1} (H:M:S)\n".format(args.emp_fasta, te))
    print("--------------------------------------------------------------------------------------\n\n")
    '''
    if not args.multisearch:
        print("\n\n======================================================================================")
        print("\nProcessing fasta file: {}\n".format(args.emp_fasta.split('.')[0]))
        print("======================================================================================\n\n")
        make_db(args.blast_db)
        outname = "{}_blast_results.txt".format((args.emp_fasta).split('.')[0])
        blastn_to_db(args.blast_task, args.blast_db, args.emp_fasta, outname, args.max_hits)

        bad_seqs = parse_blastn_output6(outname)
        pull_records_from_accn_lists(args.emp_fasta, bad_seqs)

        cleanup(blastdir, cdir, fdir)
        tf = datetime.now()
        te = tf - tb
        print("\n\n--------------------------------------------------------------------------------------")
        print("\nTotal time to process {0}: {1} (H:M:S)\n".format(args.emp_fasta, te))
        print("--------------------------------------------------------------------------------------\n\n")

    elif args.multisearch:
        with open(args.multisearch, 'r') as fh:
            pairs = [l.strip().split('\t') for l in fh if l.strip()]
        if pairs:
            for p in pairs:
                print("\n\n======================================================================================")
                print("\nProcessing fasta file: {}\n".format(p[0]))
                print("======================================================================================\n\n")
                b = datetime.now()
                make_db(p[1])
                outname = "{}_blast_results.txt".format((p[0]).split('.')[0])
                blastn_to_db(args.blast_task, p[1], p[0], outname, args.max_hits)

                bad_seqs = parse_blastn_output6(outname)
                pull_records_from_accn_lists(p[0], bad_seqs)

                cleanup(blastdir, cdir, fdir)
                f = datetime.now()
                e = f - b
                print("\nElapsed time: {0} (H:M:S)\n\n".format(e))

        else:
            print("\n\nSomething went wrong while attempting to parse file: {}".format(args.multisearch.split('/')[-1]))
            print("Please ensure this is a tab-delimited file with empirical fasta name in column 1 and "
                      "the database fasta name in column 2, and all files are within the -i directory.\n\n")

        tf = datetime.now()
        te = tf - tb
        print("\n\n--------------------------------------------------------------------------------------")
        print("\nTotal time to process {0} file(s): {1} (H:M:S)\n".format(len(pairs), te))
        print("--------------------------------------------------------------------------------------\n\n")

if __name__ == '__main__':
    main()


import argparse
import json
import logging

from torch.utils.data import Dataset
from torch.utils.data.dataset import T_co
from tqdm.auto import tqdm
from transformers import AutoTokenizer, AutoModelForMaskedLM, pipeline

from utils.file_io import read_lm_kbc_jsonl

logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s -  %(message)s",
    datefmt="%m/%d/%Y %H:%M:%S",
    level=logging.INFO,
)
logger = logging.getLogger(__name__)

class PromptSet(Dataset):
    def __init__(self, prompts):
        self.prompts = prompts

    def __len__(self):
        return len(self.prompts)

    def __getitem__(self, index) -> T_co:
        return self.prompts[index]

def create_prompt(subject_entity: str, relation: str, mask_token: str) -> str:
    """
    Depending on the relation, we fix the prompt
    """

    prompt = mask_token

    if relation == "CountryBordersWithCountry":
        prompt = f"{subject_entity} shares border with {mask_token}."
    elif relation == "CountryOfficialLanguage":
        prompt = f"The official language of {subject_entity} is {mask_token}."
    elif relation == "StateSharesBorderState":
        prompt = f"{subject_entity} shares border with {mask_token} state."
    elif relation == "RiverBasinsCountry":
        prompt = f"{subject_entity} river basins in {mask_token}."
    elif relation == "ChemicalCompoundElement":
        prompt = f"{subject_entity} consists of {mask_token}, " \
                 f"which is an element."
    elif relation == "PersonLanguage":
        prompt = f"{subject_entity} speaks in {mask_token}."
    elif relation == "PersonProfession":
        prompt = f"{subject_entity} is a {mask_token} by profession."
    elif relation == "PersonInstrument":
        prompt = f"{subject_entity} plays {mask_token}, which is an instrument."
    elif relation == "PersonEmployer":
        prompt = f"{subject_entity} is an employer at {mask_token}, " \
                 f"which is a company."
    elif relation == "PersonPlaceOfDeath":
        prompt = f"{subject_entity} died at {mask_token}."
    elif relation == "PersonCauseOfDeath":
        prompt = f"{subject_entity} died due to {mask_token}."
    elif relation == "CompanyParentOrganization":
        prompt = f"The parent organization of {subject_entity} is {mask_token}."

    return prompt

def run(args):
    # Load the model
    model_type = args.model
    logger.info(f"Loading the model \"{model_type}\"...")

    tokenizer = AutoTokenizer.from_pretrained(model_type)
    model = AutoModelForMaskedLM.from_pretrained(model_type)

    pipe = pipeline(
        task="fill-mask",
        model=model,
        tokenizer=tokenizer,
        top_k=args.top_k,
        device=args.gpu
    )

    mask_token = tokenizer.mask_token

    # Load the input file
    logger.info(f"Loading the input file \"{args.input}\"...")
    input_rows = read_lm_kbc_jsonl(args.input)
    logger.info(f"Loaded {len(input_rows):,} rows.")

    # Create prompts
    logger.info(f"Creating prompts...")
    prompts = PromptSet([create_prompt(
        subject_entity=row["SubjectEntity"],
        relation=row["Relation"],
        mask_token=mask_token,
    ) for row in input_rows])

    # Run the model
    logger.info(f"Running the model...")
    outputs = []
    for out in tqdm(pipe(prompts, batch_size=8), total=len(prompts)):
        outputs.append(out)
    results = []
    for row, prompt, output in zip(input_rows, prompts, outputs):
        result = {
            "SubjectEntity": row["SubjectEntity"],
            "Relation": row["Relation"],
            "Prompt": prompt,
            "ObjectEntities": [
                seq["token_str"]
                for seq in output if seq["score"] > args.threshold],
        }
        results.append(result)

    # Save the results
    logger.info(f"Saving the results to \"{args.output}\"...")
    with open(args.output, "w") as f:
        for result in results:
            f.write(json.dumps(result) + "\n")

def main():
    parser = argparse.ArgumentParser(
        description="Probe a Language Model and "
                    "Run the Baseline Method on Prompt Outputs"
    )

    parser.add_argument(
        "-m",
        "--model",
        type=str,
        default="bert-large-cased",
        help="HuggingFace model name (default: bert-large-cased)",
    )
    parser.add_argument(
        "-i",
        "--input",
        type=str,
        default="data/dev.jsonl",
        required=True,
        help="Input test file (required)",
    )
    parser.add_argument(
        "-o",
        "--output",
        type=str,
        default="predictions/baseline.pred.jsonl",
        required=True,
        help="Output file (required)",
    )
    parser.add_argument(
        "-k",
        "--top_k",
        type=int,
        default=100,
        help="Top k prompt outputs (default: 100)",
    )
    parser.add_argument(
        "-t",
        "--threshold",
        type=float,
        default=0.5,
        help="Probability threshold (default: 0.5)",
    )
    parser.add_argument(
        "-g",
        "--gpu",
        type=int,
        default=-1,
        help="GPU ID, (default: -1, i.e., using CPU)"
    )

    args = parser.parse_args()

    run(args)

if __name__ == '__main__':
    main()


from csv import DictReader
from pytube import YouTube
import librosa
import librosa.display
import matplotlib.pyplot as plt
import sklearn
import sys
import os

class Frame:

	def create_image(self, title, start, end):
		wav, s = librosa.core.load(title + '.mp4', sr = 44100)

		wav = wav[start:end]
		#MFCC  feature extraction automatically performs framing . Th default value for the noOfMFCC = 20

		mfcc = librosa.feature.mfcc(wav, sr = 44100)

		# Feature scaling , so that the feature with higher magnitude does not become main deciding feature
		mfcc = sklearn.preprocessing.scale(mfcc, axis=1)

		#Creation of the image
		fig = plt.figure()

		ax = fig.add_subplot(111)

		ax.axes.get_xaxis().set_visible(False)

		ax.axes.get_yaxis().set_visible(False)

		ax.set_frame_on(False)

		librosa.display.specshow(mfcc, sr=44100, x_axis='time' , cmap = 'gray_r')

		#Saving the image
		fig.savefig(title + '.png')

	#Function to delete the .mp4 files from the directory
	def delete_mp4(self):
		dir = os.getcwd()
		for file in os.listdir(dir):
			if file.endswith('.mp4'):
				os.remove(os.path.join(dir, file))

	#Function to parse the csv files and create list of those youtube items with the required Label.
	def list_of_cry(self, csv_file, label):

		l1 = []

		with open(csv_file) as file:
			csv_res = DictReader(file)
			for elem in csv_res:
				s = elem['positive_labels']
				x = s.split(',')
				for st in x:
					if st == label:
						l1.append(elem)
		return l1

	def download_yt(self, list_of_csv, count_of_images):

		i = 1
		dick = {}
		flag = False
		for row in list_of_csv:
			vid = row['YTID']
			start = int(float(row['start_seconds']))*44100
			end = int(float(row['end_seconds']))*44100
			url = 'https://www.youtube.com/watch?v='+vid
			j = 0
			while True:
				#Many times downloading fails, this is to specify attempt 20 times before giving up for a video
				j = j + 1
				if j > 20:
					break
				try:
					yt = YouTube(url)
					stream = yt.streams.filter(only_audio=True).all()
					stream = stream[0]
					print(yt.title)
					stream.download(filename=str(i))
					flag = True
				except Exception as e:
					print(e)
					print('Attempt ' + str(j))
					continue
				break
			if flag == True:
				self.create_image(str(i), start, end)
				i = i + 1
				if i > count_of_images:
					break
			flag = False

def main():

	label = sys.argv[1]
	limit_for_images = int(sys.argv[2])
	f = Frame()
	cry = f.list_of_cry('test.csv', label)
	f.download_yt(cry, limit_for_images)
	f.delete_mp4()

if __name__ == '__main__':
	main()


import sys
import os
import time

from urllib.parse import urljoin
from datetime import timedelta
from tzlocal import get_localzone
from dateutil import parser as dateparser
from requests.exceptions import HTTPError

import lxml.html

from .admin import AdminInfo, AdminRedashInfo
from .queue import QueueEntry
from .logs import LogEntry
from .review import Review
from .session import AmoSession
from .validation import ValidationReport
from .utils import AMO_BASE, AMO_CONFIG, AMO_EDITOR_BASE, AMO_DEVELOPER_BASE, \
    AMO_TIMEZONE, VALIDATION_WAIT, UPLOAD_PLATFORM, csspath

class AddonsService:
    def __init__(self, login_prompter=None):
        self.session = AmoSession(self, login_prompter)

    def persist(self):
        self.session.persist()

    def get_review(self, id_or_url, unlisted=False):
        review = Review(self, id_or_url, unlisted)
        review.get()
        return review

    def get_admin_info(self, id_or_url, getall=True):
        admininfo = AdminInfo(self, id_or_url)
        if getall:
            admininfo.get_all_versions()
        else:
            admininfo.get()
        return admininfo

    @staticmethod
    def get_user_addons(user):
        api_key = AMO_CONFIG.get('auth', 'redash_key', fallback=None)
        if not api_key:
            raise Exception("A redash API key is required in the config (auth.redash_key)")

        redash = AdminRedashInfo(api_key)
        return redash.get_user_addons(user)

    def _unpaginate(self, url, func, params=None, limit=sys.maxsize):
        things = []
        lastlen = -1

        while url and len(things) < limit:
            req = self.session.get(url, stream=True, params=params)
            req.raw.decode_content = True
            doc = lxml.html.parse(req.raw).getroot()
            lastlen = len(things)
            nexturl = func(things, doc)
            thislen = len(things)

            if lastlen == thislen:
                break

            if thislen > limit:
                things = things[:limit]

            # Get the next url and make sure to unset parameters, since they
            # will be provided in the next url anyway.
            url = urljoin(AMO_EDITOR_BASE, nexturl) if nexturl else None
            params = None

        return things

    def get_queue(self, name_or_url):
        if name_or_url.startswith(AMO_BASE) or name_or_url.startswith(AMO_EDITOR_BASE):
            name = "/".join(name_or_url.split("/")[-2])
        else:
            name = name_or_url

        def page(queue, doc):
            queuerows = doc.xpath(csspath('#addon-queue > tbody > .addon-row'))
            for row in queuerows:
                queue.append(QueueEntry(self.session, row))

            nextlink = doc.xpath(csspath('.data-grid-top > .pagination > li > a[rel="next"]'))
            return nextlink[0].attrib['href'] if len(nextlink) > 0 else None

        url = '%s/%s' % (AMO_EDITOR_BASE, name)
        return self._unpaginate(url, page)

    def get_logs(self, loglist, start=None, end=None, query=None, limit=sys.maxsize):
        # pylint: disable=too-many-arguments
        payload = {
            'search': query
        }

        dtstart = None
        dtend = None
        localtz = get_localzone()

        # We need to offset the date a bit so the returned results are in the
        # user's local timezone. If specific times were passed then don't
        # expand the end date to the end of the day.
        if start:
            dtstart = localtz.localize(dateparser.parse(start))
            payload['start'] = dtstart.astimezone(AMO_TIMEZONE).strftime('%Y-%m-%d')

        if end:
            dtend = localtz.localize(dateparser.parse(end))
            if dtend.hour == 0 and dtend.minute == 0 and dtend.second == 0:
                dtend += timedelta(days=1)
            payload['end'] = dtend.astimezone(AMO_TIMEZONE).strftime('%Y-%m-%d')

        def page(logs, doc):
            logrows = doc.xpath(csspath('#log-listing > tbody > tr[data-addonid]'))
            for row in logrows:
                entry = LogEntry(self.session, row)
                if (not dtstart or entry.date >= dtstart) and \
                   (not dtend or entry.date <= dtend):
                    logs.append(entry)

            nextlink = doc.xpath(csspath('.pagination > li > a[rel="next"]'))
            return nextlink[0].attrib['href'] if len(nextlink) > 0 else None

        url = '%s/%s' % (AMO_EDITOR_BASE, loglist)
        return self._unpaginate(url, page, params=payload, limit=limit)

    def upload(self, addonid, xpi, platform='all'):
        if platform not in UPLOAD_PLATFORM:
            raise Exception("Unknown platform %s" % platform)

        uploadurl = None
        url = '%s/addon/%s/versions' % (AMO_DEVELOPER_BASE, addonid)
        req = self.session.get(url, stream=True)
        req.raw.decode_content = True
        doc = lxml.html.parse(req.raw).getroot()
        token = doc.xpath(csspath('form input[name="csrfmiddlewaretoken"]'))[0].attrib['value']

        with open(xpi, 'rb') as xpifd:
            payload = {
                'csrfmiddlewaretoken': (None, token),
                'upload': (os.path.basename(xpi), xpifd, 'application/x-xpinstall')
            }

            uploadurl = '%s/addon/%s/upload-listed' % (AMO_DEVELOPER_BASE, addonid)

            req = self.session.post(uploadurl, files=payload, allow_redirects=False, timeout=None)
            if req.status_code != 302:
                raise Exception('Could not upload %s' % xpi)
            uploadurl = req.headers['location']

        return self.wait_for_validation(addonid, uploadurl, platform)

    def wait_for_validation(self, addonid, uploadurl, platform='all', interval=VALIDATION_WAIT):
        if platform not in UPLOAD_PLATFORM:
            raise Exception("Unknown platform %s" % platform)

        report = None
        while not report or not report.completed:
            req = self.session.get(uploadurl, timeout=None)
            report = ValidationReport(addonid, req.json(), platform)

            if not report.success:
                return report
            elif not report.completed:
                print("Waiting for validation...")
                time.sleep(interval)

        return report

    def add_xpi_to_version(self, addonid, report, platform, source=None, beta=False):
        # pylint: disable=too-many-locals,too-many-branches,too-many-arguments
        sourcefd = None
        try:
            url = '%s/addon/%s/versions' % (AMO_DEVELOPER_BASE, addonid)
            req = self.session.get(url, stream=True)
            req.raw.decode_content = True
            doc = lxml.html.parse(req.raw).getroot()
            token = doc.xpath(csspath('form input[name="csrfmiddlewaretoken"]'))[0].attrib['value']

            ver_exists = doc.xpath(csspath(".item_wrapper a") +
                                   "[contains(text(), 'Version %s')]" % report.version)
            if len(ver_exists) > 0:
                final_version_url = urljoin(AMO_DEVELOPER_BASE, ver_exists[0].attrib['href'])
                add_version_url = final_version_url + "/submit-file/"
            else:
                final_version_url = None  # will fill this in later
                add_version_url = '%s/addon/%s/versions/submit/' % (AMO_DEVELOPER_BASE, addonid)

            payload = {
                'csrfmiddlewaretoken': (None, token),
                'upload': (None, report.reportid),
                'supported_platforms': (None, UPLOAD_PLATFORM[platform]),
            }

            if beta:
                payload['beta'] = (None, "on")

            if source:
                sourcefd = open(source, 'rb')
                payload['source'] = (os.path.basename(source), sourcefd, 'application/octet-stream')
            else:
                payload['source'] = ("", "", 'application/octet-stream')

            req = self.session.post(add_version_url, files=payload,
                                    allow_redirects=False, timeout=(60.0, 60.0))

            if not final_version_url:
                locparts = req.headers['location'].split("/")
                final_version_url = urljoin(AMO_DEVELOPER_BASE,
                                            "addon/%s/versions/%s" % (addonid, locparts[-2]))

            return final_version_url
        except HTTPError as e:
            if e.response.status_code != 400:
                raise e
            msg = None
            try:
                msg = "Error: %s" % " ".join(e.response.json()['__all__'])
            except (ValueError, KeyError):
                msg = ("Error: %s" % e.response.text)

            raise Exception(msg) from e
        finally:
            if sourcefd:
                sourcefd.close()