import json
from topojson.core.extract import Extract
from shapely import geometry
import geopandas
import geojson
from geojson import Feature, Polygon, FeatureCollection
import fiona

# extract copies coordinates sequentially into a buffer
def test_extract_linestring():
    data = {
        "foo": {"type": "LineString", "coordinates": [[0, 0], [1, 0], [2, 0]]},
        "bar": {"type": "LineString", "coordinates": [[0, 0], [1, 0], [2, 0]]},
    }
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 2

# assess if a multipolygon with hole is processed into the right number of rings
def test_extract_multipolygon():
    # multipolygon with hole
    data = {
        "foo": {
            "type": "MultiPolygon",
            "coordinates": [
                [
                    [[0, 0], [20, 0], [10, 20], [0, 0]],  # CCW
                    [[3, 2], [10, 16], [17, 2], [3, 2]],  # CW
                ],
                [[[6, 4], [14, 4], [10, 12], [6, 4]]],  # CCW
                [[[25, 5], [30, 10], [35, 5], [25, 5]]],
            ],
        }
    }
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 3
    assert len(topo["linestrings"]) == 4

# a LineString without coordinates is an empty polygon geometry
def test_extract_empty_linestring():
    data = {"empty_ls": {"type": "LineString", "coordinates": None}}
    topo = Extract(data).to_dict()

    assert topo["objects"]["empty_ls"]["arcs"] == None

# invalid polygon geometry
def test_extract_invalid_polygon():
    data = {
        "wrong": {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [2, 0], [0, 0]]]},
        "valid": {"type": "Polygon", "coordinates": [[[0, 0], [2, 0], [1, 1], [0, 0]]]},
    }
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 1

# test multilinestring
def test_extract_multilinestring():
    data = {
        "foo": {
            "type": "MultiLineString",
            "coordinates": [
                [[0.0, 0.0], [1, 1], [3, 3]],
                [[1, 1], [0, 1]],
                [[3, 3], [4, 4], [0, 1]],
            ],
        }
    }
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 3

# test nested geojson geometrycollection collection
def test_extract_nested_geometrycollection():
    data = {
        "foo": {
            "type": "GeometryCollection",
            "geometries": [
                {
                    "type": "GeometryCollection",
                    "geometries": [
                        {"type": "LineString", "coordinates": [[0.1, 0.2], [0.3, 0.4]]}
                    ],
                },
                {
                    "type": "Polygon",
                    "coordinates": [[[0.5, 0.6], [0.7, 0.8], [0.9, 1.0]]],
                },
            ],
        }
    }
    topo = Extract(data).to_dict()

    assert len(topo["objects"]["foo"]["geometries"][0]["geometries"][0]["arcs"]) == 1

# test geometry collection + polygon
def test_extract_geometrycollection_polygon():
    data = {
        "bar": {"type": "Polygon", "coordinates": [[[0, 0], [1, 1], [2, 0]]]},
        "foo": {
            "type": "GeometryCollection",
            "geometries": [
                {"type": "LineString", "coordinates": [[0.1, 0.2], [0.3, 0.4]]}
            ],
        },
    }
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 2

# test feature type
def test_extract_features():
    data = {
        "foo": {
            "type": "Feature",
            "geometry": {"type": "LineString", "coordinates": [[0.1, 0.2], [0.3, 0.4]]},
        },
        "bar": {
            "type": "Feature",
            "geometry": {
                "type": "Polygon",
                "coordinates": [[[0.5, 0.6], [0.7, 0.8], [0.9, 1.0]]],
            },
        },
    }
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 2

# test feature collection including geometry collection
def test_extract_featurecollection():
    data = {
        "collection": {
            "type": "FeatureCollection",
            "features": [
                {
                    "type": "Feature",
                    "geometry": {
                        "type": "LineString",
                        "coordinates": [[0.1, 0.2], [0.3, 0.4]],
                    },
                },
                {
                    "type": "Feature",
                    "geometry": {
                        "type": "GeometryCollection",
                        "geometries": [
                            {
                                "type": "Polygon",
                                "coordinates": [[[0.5, 0.6], [0.7, 0.8], [0.9, 1.0]]],
                            }
                        ],
                    },
                },
            ],
        }
    }
    topo = Extract(data).to_dict()

    assert len(topo["objects"]) == 2
    assert len(topo["bookkeeping_geoms"]) == 2
    assert len(topo["linestrings"]) == 2
    assert topo["objects"]["feature_0"]["type"] == "LineString"
    assert topo["objects"]["feature_1"]["geometries"][0]["type"] == "Polygon"

# test to parse feature collection from a geojson file through geojson library
def test_extract_geojson_feat_col_geom_col():
    with open("tests/files_geojson/feature_collection.geojson") as f:
        data = geojson.load(f)
    topo = Extract(data).to_dict()

    assert len(topo["objects"]) == 1
    assert len(topo["bookkeeping_geoms"]) == 3
    assert len(topo["linestrings"]) == 3

# test to parse a feature from a geojson file through geojson library
def test_extract_geojson_feature_geom_col():
    with open("tests/files_geojson/feature.geojson") as f:
        data = geojson.load(f)
    topo = Extract(data).to_dict()

    assert len(topo["objects"]) == 1
    assert len(topo["bookkeeping_geoms"]) == 3
    assert len(topo["linestrings"]) == 3

# test feature collection including geometry collection
def test_extract_geopandas_geoseries():
    data = geopandas.GeoSeries(
        [
            geometry.Polygon([(0, 0), (1, 0), (1, 1)]),
            geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]),
            geometry.Polygon([(2, 0), (3, 0), (3, 1), (2, 1)]),
        ]
    )
    topo = Extract(data).to_dict()

    assert len(topo["objects"]) == 3
    assert len(topo["bookkeeping_geoms"]) == 3
    assert len(topo["linestrings"]) == 3

# test shapely geometry collection.
def test_extract_shapely_geometrycollection():
    data = geometry.GeometryCollection(
        [
            geometry.Polygon([[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]),
            geometry.Polygon([[1, 0], [2, 0], [2, 1], [1, 1], [1, 0]]),
        ]
    )
    topo = Extract(data).to_dict()

    assert len(topo["objects"]) == 1
    assert len(topo["bookkeeping_geoms"]) == 2
    assert len(topo["linestrings"]) == 2

def test_extract_geo_interface_from_list():
    data = [
        {"type": "LineString", "coordinates": [[0, 0], [1, 0], [2, 0]]},
        {"type": "LineString", "coordinates": [[0, 0], [1, 0], [2, 0]]},
    ]
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 2

def test_extract_shapely_geo_interface_from_list():
    data = [
        geometry.Polygon([[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]),
        geometry.Polygon([[1, 0], [2, 0], [2, 1], [1, 1], [1, 0]]),
    ]
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 2
    assert isinstance(topo["objects"][0], dict)

# duplicate rotated geometry bar with hole interior in geometry foo
def test_extract_geomcol_multipolygon_polygon():
    data = {
        "foo": {
            "type": "GeometryCollection",
            "geometries": [
                {
                    "type": "MultiPolygon",
                    "coordinates": [
                        [
                            [[10, 20], [20, 0], [0, 0], [3, 13], [10, 20]],
                            [[3, 2], [10, 16], [17, 2], [3, 2]],
                        ],
                        [[[10, 4], [14, 4], [10, 12], [10, 4]]],
                    ],
                },
                {
                    "type": "Polygon",
                    "coordinates": [[[20, 0], [35, 5], [10, 20], [20, 0]]],
                },
            ],
        }
    }
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 4

def test_extract_geo_interface_shapefile():
    import shapefile

    data = shapefile.Reader("tests/files_shapefile/southamerica.shp")
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 15

def test_extract_points():
    data = [
        {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]]},
        {"type": "Point", "coordinates": [0.5, 0.5]},
    ]
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_coords"]) == 1
    assert len(topo["bookkeeping_geoms"]) == 1
    assert topo["coordinates"][0].tolist() == [[0.5, 0.5]]
    assert "coordinates" in topo["objects"][1].keys()

def test_extract_single_polygon():
    data = geometry.Polygon([[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]])
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 1

def test_extract_single_linestring():
    data = geometry.LineString([[0, 0], [1, 0], [1, 1], [0, 1]])
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 1

def test_extract_single_multilinestring():
    data = geometry.MultiLineString([[[0, 0], [1, 1]], [[-1, 0], [1, 0]]])
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 2

def test_extract_single_multilinestring_list():
    data = [geometry.MultiLineString([[[0, 0], [1, 1]], [[-1, 0], [1, 0]]])]
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 2

def test_extract_geopandas_geodataframe():
    data = geopandas.read_file(
        "tests/files_geojson/naturalearth_alb_grc.geojson", driver="GeoJSON"
    )
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 3

# dict should have a valued key:geom_object. Otherwise key:value is removed
def test_extract_invalid_dict_item():
    data = {
        "type": "MultiPolygon",
        "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]],
    }
    topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 0

def test_extract_fiona_file():
    with fiona.open("tests/files_shapefile/southamerica.shp") as data:
        topo = Extract(data).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 15

def test_extract_fiona_file_gpkg():
    with fiona.open("tests/files_shapefile/rivers.gpkg") as col:
        feats = []
        for idx, feat in enumerate(col):
            if idx in [24, 25]:
                feats.append(feat)

    topo = Extract(feats).to_dict()

    assert len(topo["bookkeeping_geoms"]) == 4

# test to check if original data is not modified
def test_extract_dict_org_data_untouched():
    data = {
        "foo": {"type": "LineString", "coordinates": [[0, 0], [1, 0], [2, 0]]},
        "bar": {"type": "LineString", "coordinates": [[0, 0], [1, 0], [2, 0]]},
    }
    topo = Extract(data).to_dict()
    topo_foo = topo["objects"]["foo"]
    data_foo = data["foo"]

    assert "arcs" in topo_foo.keys()
    assert "arcs" not in data_foo.keys()

# test to check if original data is not modified
def test_extract_list_org_data_untouched():
    data = [
        geometry.Polygon([[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]),
        geometry.Polygon([[1, 0], [2, 0], [2, 1], [1, 1], [1, 0]]),
    ]
    topo = Extract(data).to_dict()
    topo_0 = topo["objects"][0]
    data_0 = data[0]

    assert "arcs" in topo_0.keys()
    assert data_0.geom_type == "Polygon"

# test to check if original data is not modified
def test_extract_gdf_org_data_untouched():
    data = geopandas.read_file(
        "tests/files_geojson/naturalearth_alb_grc.geojson", driver="GeoJSON"
    )
    topo = Extract(data).to_dict()
    topo_0 = topo["objects"][0]
    data_0 = data.iloc[0]

    assert "arcs" in topo_0.keys()
    assert data_0.geometry.geom_type == "Polygon"

# test to check if original data is not modified
def test_extract_shapely_org_data_untouched():
    data = geometry.LineString([[0, 0], [1, 0], [1, 1], [0, 1]])
    topo = Extract(data).to_dict()
    topo_0 = topo["objects"][0]

    assert "arcs" in topo_0.keys()
    assert data.geom_type == "LineString"

# test to check if original data is not modified
def test_extract_shapefile_org_data_untouched():
    import shapefile

    data = shapefile.Reader("tests/files_shapefile/southamerica.shp")
    topo = Extract(data).to_dict()
    topo_0 = topo["objects"]["feature_00"]
    data_0 = data.__geo_interface__["features"][0]["geometry"]

    assert "arcs" in topo_0.keys()
    assert "arcs" not in data_0.keys()

# issue 137 do not modify source data
def test_extract_source_data_modify():
    # prepare data
    feat_1 = Feature(
        geometry=Polygon([[[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]]),
        properties={"name": "abc"},
    )
    feat_2 = Feature(
        geometry=Polygon([[[1, 0], [2, 0], [2, 1], [1, 1], [1, 0]]]),
        properties={"name": "def"},
    )
    data = FeatureCollection([feat_1, feat_2])

    # before Topology()
    assert "geometry" in data["features"][0].keys()

    # apply Topology()
    topo = Extract(data)

    # after Topology()
    assert "geometry" in data["features"][0].keys()

# issue 151 properties are not kept in geojson data
def test_extract_keep_properties():
    # prepare data
    feat_1 = Feature(
        geometry=Polygon([[[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]]]),
        properties={"name": "abc"},
    )
    feat_2 = Feature(
        geometry=Polygon([[[1, 0], [2, 0], [2, 1], [1, 1], [1, 0]]]),
        properties={"name": {"def": "ghi"}},
    )
    data = FeatureCollection([feat_1, feat_2])
    topo = Extract(data).to_dict()

    assert topo["objects"]["feature_0"]["properties"]["name"] == "abc"
    assert topo["objects"]["feature_1"]["properties"]["name"]["def"] == "ghi"

# why cannot load geojson file using json module?
def test_extract_read_geojson_from_json_dict():
    with open("tests/files_geojson/naturalearth_lowres.geojson") as f:
        data = json.load(f)
    topo = Extract(data).to_dict()

    assert len(topo["linestrings"]) == 287

def test_extract_read_multiple_gdf_object_name():
    world = geopandas.read_file(geopandas.datasets.get_path("naturalearth_lowres"))
    world = world[["continent", "geometry", "pop_est"]]
    continents = world.dissolve(by="continent", aggfunc="sum")

    topo = Extract(
        data=[world, continents], options={"object_name": ["world", "continents"]}
    ).to_dict()

    assert len(topo["objects"]) == len(world) + len(continents)

def test_extract_read_multiple_gjson_object_name():

    with open('tests/files_geojson/geojson_1.json', 'r') as gj_1:
        geojson_1 = json.load(gj_1)

    with open('tests/files_geojson/geojson_2.json', 'r') as gj_2:
        geojson_2 = json.load(gj_2)

    topo = Extract(
        data=[geojson_1, geojson_2],
        options={"object_name": ['gjson_1', 'gjson_2']}
    ).to_dict()

    assert len(topo["objects"]) == (
        len(geojson_1['features']) + len(geojson_2['features'])
    )


"""
meetups models
"""

from datetime import datetime

meetups = []
rsvps = []

class Meetup():
    """
    define all meetup attributes and methods
    """

    def __init__(self, occuring_on=None, host=None, topic=None, summary=None, tags=None, location=None):
        '''
        initialize class
        '''
        self.meetupId = str(len(meetups) + 1)
        self.created_on = datetime.now()
        self.occuring_on = occuring_on
        self.host = host
        self.topic = topic
        self.summary = summary
        self.tags = tags
        self.location = location

    def createMeetup(self):
        '''
        Method for creating a new meetup record
        '''

        meetup = {
            "meetupId": self.meetupId,
            "host": self.host,
            "created_on": self.created_on,
            "occuring_on": self.occuring_on,
            "topic": self.topic,
            "summary": self.summary,
            "location": self.location,
            "tags": self.tags
        }

        meetups.append(meetup)
        return meetup, {"message": "Meetup created successfully"}

    def getMeetup(self, meetupId):
        '''
        Method for getting one meetup record
        '''
        for meetup in meetups:
            if meetup['meetupId'] == meetupId:
                return meetup, {"message": "success"}

    def allMeetups(self):
        '''method for getting all meetup records'''
        return meetups

    def meetupRsvp(self, userId, meetupId, response):
        '''
        Method for getting rsvp meetup
        '''
        rsvp_data = {
            "rsvpId": str(len(rsvps) + 1),
            "meetupId": meetupId,
            "userId": userId,
            "response": response
        }

        rsvps.append(rsvp_data)
        return rsvp_data, {"message": "attendance status confirmed"}


from discord.ext import commands
import os
import logging
import bot_tokens as bt

#define what will trigger bot commands
#https://discordpy.readthedocs.io/en/stable/ext/commands/commands.html
bot = commands.Bot(command_prefix='!')

#keeps a running event log while bot is on
#https://discordpy.readthedocs.io/en/stable/logging.html
logger = logging.getLogger('discord')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

#posts message to console once the bot has succesfully logged into Discord
#https://discordpy.readthedocs.io/en/stable/quickstart.html
@bot.event
async def on_ready():
	print(f'{bot.user} Initialized')

for filename in os.listdir('./cogs'):
	if filename.endswith('.py'):
		bot.load_extension(f'cogs.{filename[:-3]}')

#!reload (command) to reload edited cogs from discord
@bot.command()
async def reload(ctx):
    #reloads the cog file to update without restarting bot
	bot.reload_extension('cogs.tracking')
	await ctx.send('Cog reloaded.')

#bot's authentication token with Discord
bot.run(bt.DISCORD_TOKEN)


"""Tests for houdini_toolbox.logging.shellio module."""

# =============================================================================
# IMPORTS
# =============================================================================

# Standard Library
import logging

# Third Party
import pytest

# Houdini Toolbox
import houdini_toolbox.logging.shellio

# Houdini
import hou

# =============================================================================
# FIXTURES
# =============================================================================

@pytest.fixture
def init_handler(mocker):
    """Fixture to initialize a handler."""
    mocker.patch.object(
        houdini_toolbox.logging.shellio.PythonShellHandler,
        "__init__",
        lambda x, y: None,
    )

    def _create():
        return houdini_toolbox.logging.shellio.PythonShellHandler(None)

    return _create

# =============================================================================
# TESTS
# =============================================================================

class Test_PythonShellHandler:
    """Test houdini_toolbox.logging.shellio.PythonShellHandler object."""

    def test_emit__shellio(self, init_handler, mocker):
        """Test when sys.stdout is an instance of hou.ShellIO."""
        mock_format = mocker.patch.object(
            houdini_toolbox.logging.shellio.PythonShellHandler, "format"
        )

        mock_message = mocker.MagicMock(spec=str)

        mock_format.return_value = mock_message

        mock_record = mocker.MagicMock(spec=logging.LogRecord)
        inst = init_handler()

        mock_stream = mocker.patch("sys.stdout", spec=hou.ShellIO)
        inst.emit(mock_record)

        mock_format.assert_called_with(mock_record)

        calls = [mocker.call(mock_format.return_value), mocker.call("\n")]

        mock_stream.write.assert_has_calls(calls)
        mock_stream.flush.assert_called()

    def test_emit__not_shellio(self, init_handler, mocker):
        """Test when sys.stdout is not an instance of hou.ShellIO."""
        mock_format = mocker.patch.object(
            houdini_toolbox.logging.shellio.PythonShellHandler, "format"
        )

        mock_message = mocker.MagicMock(spec=str)

        mock_format.return_value = mock_message

        mock_record = mocker.MagicMock(spec=logging.LogRecord)
        inst = init_handler()

        inst.emit(mock_record)

        mock_format.assert_not_called()

    def test_emit__keyboardinterrupt(self, init_handler, mocker):
        """Test when KeyboardInterrupt is raised."""
        mock_format = mocker.patch.object(
            houdini_toolbox.logging.shellio.PythonShellHandler, "format"
        )
        mock_format.side_effect = KeyboardInterrupt

        mock_record = mocker.MagicMock(spec=logging.LogRecord)
        inst = init_handler()

        mocker.patch("sys.stdout", spec=hou.ShellIO)

        with pytest.raises(KeyboardInterrupt):
            inst.emit(mock_record)

        mock_format.assert_called_with(mock_record)

    def test_emit__systemexit(self, init_handler, mocker):
        """Test when SystemExit is raised."""
        mock_format = mocker.patch.object(
            houdini_toolbox.logging.shellio.PythonShellHandler, "format"
        )
        mock_format.side_effect = SystemExit

        mock_record = mocker.MagicMock(spec=logging.LogRecord)
        inst = init_handler()

        mocker.patch("sys.stdout", spec=hou.ShellIO)

        with pytest.raises(SystemExit):
            inst.emit(mock_record)

        mock_format.assert_called_with(mock_record)

    def test_emit__generic_exception(self, init_handler, mocker):
        """Test when an generic exception is raised."""
        mock_format = mocker.patch.object(
            houdini_toolbox.logging.shellio.PythonShellHandler, "format"
        )
        mock_format.side_effect = Exception

        mock_handle = mocker.patch.object(
            houdini_toolbox.logging.shellio.PythonShellHandler, "handleError"
        )

        mock_record = mocker.MagicMock(spec=logging.LogRecord)
        inst = init_handler()

        mocker.patch("sys.stdout", spec=hou.ShellIO)

        inst.emit(mock_record)

        mock_format.assert_called_with(mock_record)
        mock_handle.assert_called_with(mock_record)


#!/usr/bin/env python

"""Tests for `blast2xl` package."""

from os.path import abspath
from pathlib import Path

from click.testing import CliRunner

from blast2xl import cli

def test_command_line_interface():
    """Test the CLI."""
    runner = CliRunner()
    help_result = runner.invoke(cli.main, ['--help'])
    assert help_result.exit_code == 0
    assert 'blast2xl: BLAST XLSX Report Creator' in help_result.stdout

    result = runner.invoke(cli.main)
    assert result.exit_code == 2
    assert 'Missing option' in result.output

    blast_tsv_dir = abspath('tests/data/blast_tsv')
    fasta_dir = abspath('tests/data/fastas')
    with runner.isolated_filesystem():
        excel_report = 'blast-report.xlsx'
        seq_outdir = 'seq-outdir'
        result = runner.invoke(cli.main, ['--blast-tsv-dir', blast_tsv_dir,
                                          '--blast-tsv-sample-name-pattern', r'^blastn-(.+)-vs-nt.*',
                                          '--seq-dir', fasta_dir,
                                          '--top-n-results', 5,
                                          '-o', excel_report,
                                          '-O', seq_outdir,
                                          '-vvv'])

        assert result.exit_code == 0
        path_seq_outdir = Path(seq_outdir)
        assert path_seq_outdir.exists()
        output_fastas = list(path_seq_outdir.glob('**/*.fasta'))
        assert len(output_fastas) > 2

        fasta_path = path_seq_outdir / 'FMDV' / 'Foot_and_mouth_disease_virus___type_O-12118' / 'FMDV.fasta'
        assert fasta_path.exists()
        assert fasta_path.stat().st_size > 0
        assert Path(excel_report).exists()
        assert Path(excel_report).stat().st_size > 0


from django.urls import path
from .views import *

urlpatterns = [
    path('', PostListView.as_view(), name='index'),
    path('post/<slug:slug>/', PostDetail.as_view(), name='detail'),
    path('category/<slug:slug>/', CategoryView.as_view(), name='category'),
    path('add/', AddPostView.as_view(template_name='spot_posts/add_post.html'), name='add_post'),
    path('search', SearchView.as_view(), name='search'),
    path('authors/<int:authors_id>/', AuthorPostView.as_view(), name='authors'),
    path('add_bookmark/<slug:post_slug>/', AddBookmark.as_view(), name='add_bookmark'),
    path('remove_bookmark/<slug:post_slug>/', RemoveBookmark.as_view(), name='remove_bookmark'),
    path('bookmarks/', BookmarksView.as_view(), name='bookmarks'),

]


# -*- coding: utf-8 -*-
import json
import logging
import os
import time

from datetime import datetime
from datetime import timedelta
import requests
import pandas as pd
from pandas import DataFrame, Series
from Open.PpdTask import PpdTask
from Open.PpdOpenClient import PpdOpenClient
import queue

class FetchListingInfoTask(PpdTask):
    def __init__(self, input_queue, logger=None):
        logger = logger or logging.getLogger(__name__)
        PpdTask.__init__(self, logger)
        self.config_file = json.load(open("../config.json", "r"))
        self.input_queue = input_queue

        if os.path.exists("listingInfo.csv"):
            self.df = pd.read_csv("listingInfo.csv", encoding="utf-8")
        else:
            self.df = None
        self.client = PpdOpenClient()

        self.save_to_file_queue = queue.Queue(maxsize=1000)

    def get_listing_info(self, listing_ids):
        for x in range(0, len(listing_ids), 10):
            sub_listing_ids = listing_ids[x: x + 10]
            result = self.client.get_listing_info(sub_listing_ids)
            json_data = json.loads(result)
            if "Result" not in json_data:
                self.logger.warn(f"fetch loan list info result not exist: {json_data}")
                continue

            if json_data["Result"] == 1:
                new_frame = DataFrame(json_data["LoanInfos"])
                new_listings_info = new_frame[~new_frame["ListingId"].isin(self.df["ListingId"])]
                self.output_queue.put(new_listings_info.to_dict('records'))

                self.save_to_file_queue.put(new_frame)
                # self.output_queue.put(json_data["LoanInfos"])
                # self.df = pd.concat([self.df, new_frame], sort=False).drop_duplicates("ListingId")
                # self.df.to_csv("listingInfo.csv", encoding="utf-8", index=False)
            else:
                self.logger.warn(f"fetch loan list info result != 1: {json_data}")

    def task_body(self):
        if self.input_queue.empty():
            return

        item = self.input_queue.get(timeout=2)
        if item is None:
            return

        start = time.time()
        self.get_listing_info(item)
        # self.logger.info(f"get_listing_info: {time.time() - start}")

def main():

    pass

if __name__ == "__main__":
    main()


#@+leo-ver=5-thin
#@+node:ekr.20220823200700.1: * @file ../plugins/leofts.py
import os
from whoosh.index import create_in, open_dir
from whoosh.fields import ID, TEXT, Schema
from whoosh.qparser import MultifieldParser
from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter

g = None

#@+others
#@+node:ekr.20220823205609.1: ** set_leo
def set_leo(gg):
    global g
    g = gg
    g._fts = None

#@+node:ekr.20220823205609.2: ** init
def init():

    print("bigdash init")
    import leo.core.leoGlobals as g

    set_leo(g)
    ok = g.app.gui.guiName() == "qt"
    g._fts = None
    g._gnxcache = GnxCache()

    return ok

#@+node:ekr.20220823205609.3: ** get_fts
def get_fts():
    if g._fts is None:
        g._fts = LeoFts(g.app.homeLeoDir + "/fts_index")
    return g._fts

#@+node:ekr.20220823205609.4: ** all_positions_global
def all_positions_global():
    for c in g.app.commanders():
        for p in c.all_unique_positions():
            yield(c, p)

#@+node:ekr.20220823205609.5: ** class GnxCache
class GnxCache:
    """ map gnx => vnode """
    #@+others
    #@+node:ekr.20220823205610.1: *3* __init__
    def __init__(self):
        self.clear()
    #@+node:ekr.20220823205610.2: *3* update_new_cs
    def update_new_cs(self):
        for c in g.app.commanders():
            if c.hash() not in self.cs:
                for p in c.all_unique_positions():
                    k = p.gnx
                    self.ps[k] = c, p.v
                self.cs.add(c.hash())

    #@+node:ekr.20220823205610.3: *3* get
    def get(self, gnx):
        if not self.ps:
            self.update_new_cs()
        res = self.ps.get(gnx, None)
        return res
    #@+node:ekr.20220823205610.4: *3* get_p
    def get_p(self, gnx):
        r = self.get(gnx)
        if r:
            c, v = r
            for p in c.all_unique_positions():
                if p.v is v:
                    return c, p.copy()

        print("Not in gnx cache, slow!")

        for c, p in all_positions_global():

            if p.gnx == gnx:
                return c, p.copy()
        return None

    #@+node:ekr.20220823205610.5: *3* clear
    def clear(self):
        self.ps = {}
        self.cs = set()

    #@-others
#@+node:ekr.20220823205610.6: ** class LeoFts
class LeoFts:
    #@+others
    #@+node:ekr.20220823205610.7: *3* __init__
    def __init__(self, idx_dir):
        self.idx_dir = idx_dir
        if not os.path.exists(idx_dir):
            os.mkdir(idx_dir)
            self.create()
        else:
            self.ix = open_dir(idx_dir)

    #@+node:ekr.20220823205610.8: *3* schema
    def schema(self):
        my_analyzer = RegexTokenizer("[a-zA-Z_]+") | LowercaseFilter() | StopFilter()
        schema = Schema(
            h=TEXT(stored=True, analyzer=my_analyzer),
            gnx=ID(stored=True), b=TEXT(analyzer=my_analyzer),
            parent=ID(stored=True),
            doc=ID(stored=True),
        )
        return schema

    #@+node:ekr.20220823205610.9: *3* create
    def create(self):

        schema = self.schema()
        self.ix = create_in(self.idx_dir, schema)

    #@+node:ekr.20220823205610.10: *3* index_nodes
    def index_nodes(self, c):
        writer = self.ix.writer()
        doc = c.mFileName
        for p in c.all_unique_positions():
            if p.hasParent():
                par = p.parent().get_UNL()
            else:
                par = c.mFileName

            writer.add_document(h=p.h, b=p.b, gnx=p.gnx, parent=par, doc=doc)

        writer.commit()
        g._gnxcache.clear()

    #@+node:ekr.20220823205610.11: *3* drop_document
    def drop_document(self, docfile):
        writer = self.ix.writer()
        print("Drop index", docfile)
        writer.delete_by_term("doc", docfile)
        writer.commit()

    #@+node:ekr.20220823205610.12: *3* statistics
    def statistics(self):
        r = {}
        with self.ix.searcher() as s:
            r['documents'] = list(s.lexicon("doc"))  # pylint:disable=no-member
        print("stats", r)
        return r

    #@+node:ekr.20220823205610.13: *3* search
    def search(self, searchstring, limit=30):

        res = []
        g._gnxcache.update_new_cs()
        with self.ix.searcher() as searcher:
            query = MultifieldParser(["h", "b"], schema=self.schema()).parse(searchstring)
            results = searcher.search(query, limit=limit)
            print(results)
            for r in results:
                rr = r.fields()

                gnx = rr["gnx"]
                tup = g._gnxcache.get(gnx)
                if tup:
                    rr['f'] = True
                    cont = tup[1].b

                    hl = r.highlights("b", text=cont)
                    rr["highlight"] = hl

                else:
                    rr['f'] = False
                res.append(rr)

        return res

    #@+node:ekr.20220823205610.14: *3* close
    def close(self):
        self.ix.close()

    #@-others
#@+node:ekr.20220823205610.15: ** main
def main():
    fts = LeoFts("c:/t/ltest")
    fts.create()

#@-others
if __name__ == '__main__':
    main()

#@@language python
#@@tabwidth -4
#@-leo


import itertools

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import torch
import torch.utils.data
import torchvision
from torch.utils.data import Dataset

class ImbalancedDatasetSampler(torch.utils.data.sampler.Sampler):
    """Samples elements randomly from a given list of indices for imbalanced dataset
    Arguments:
        indices (list, optional): a list of indices
        num_samples (int, optional): number of samples to draw
    """

    def __init__(self, dataset, indices=None, num_samples=None):

        # if indices is not provided,
        # all elements in the dataset will be considered
        self.indices = list(range(len(dataset))) \
            if indices is None else indices

        # if num_samples is not provided,
        # draw `len(indices)` samples in each iteration
        self.num_samples = len(self.indices) \
            if num_samples is None else num_samples

        # distribution of classes in the dataset
        label_to_count = {}
        for idx in self.indices:
            label = self._get_label(dataset, idx)
            if label in label_to_count:
                label_to_count[label] += 1
            else:
                label_to_count[label] = 1

        # weight for each sample
        weights = [1.0 / label_to_count[self._get_label(dataset, idx)]
                   for idx in self.indices]
        self.weights = torch.tensor(weights, dtype=torch.float)

    def _get_label(self, dataset, idx):
        dataset_type = type(dataset)
        if dataset_type is torchvision.datasets.MNIST:
            return dataset.train_labels[idx].item()
        elif dataset_type is torchvision.datasets.ImageFolder:
            return dataset.imgs[idx][1]
        else:
            return dataset[idx][1]

    def __iter__(self):
        return (self.indices[i] for i in torch.multinomial(
            self.weights, self.num_samples, replacement=True))

    def __len__(self):
        return self.num_samples

class Dictionary:
    def __init__(self):
        self.word2idx = {}
        self.idx2word = {}
        self.idx = 0

    def add_word(self, word: str):
        if word not in self.word2idx:
            self.word2idx[word] = self.idx
            self.idx2word[self.idx] = word
            self.idx += 1

    def __len__(self):
        return len(self.word2idx)

class RawDataset(Dataset):
    """
    Intentionally to use only
    """

    def __init__(self, df: pd.DataFrame):
        super().__init__()
        self.df = df.copy()
        self.len = len(self.df)

    def __getitem__(self, index):
        record = self.df.iloc[index]
        return record.Phrase, record.Sentiment

    def __len__(self):
        return self.len

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    print(cm)
    plt.figure(figsize=(10, 10))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.tight_layout()

def make_var(names):
    ans = []
    for name in sorted(names, reverse=True, key=len):
        name = name.lower()
        tmp = [ord(chr) for chr in name]
        tmp = torch.tensor(tmp, dtype=torch.long)
        ans.append(tmp)
    return torch.nn.utils.rnn.pad_sequence(ans, batch_first=True)

def count_non_zero_length(aaa):
    """
    aaa = [[116, 105, 114,  97, 115],
         [ 97, 110, 110,   0,   0],
         [101, 108,   0,   0,   0]]
    ans -> [5, 3, 2]
    """
    bbb = []
    for item in aaa:
        counting = 0
        for element in item:
            if element != 0:
                counting += 1
        bbb.append(counting)
    return bbb

def str2ascii_arr(name):
    """
    0-255
    """
    arr = [ord(c) for c in name]
    return arr, len(arr)


import sys
from PyQt6.QtWidgets import QApplication
import model
import view
import app

class Controller:
    def __init__(self):
        self.model = model.MyLanguageTool()
        self.view = view.View(self)

    def reset(self) -> None:
        """
        ruft die reset-methods von den jeweiligen Klassen auf
        :return:
        """
        #self.model.reset()
        self.view.reset()

    def execute(self) -> None:
        """
        bringt das ganze Programm zum Laufen, also model und view
        und setzt auch die Werte jenachdem ein
        :return:
        """
        if self.view.get_eingabe()!="" :
            self.view.set_text_statusbar("Erkennung erfolgreich durchgeführt")
            a=self.view.get_eingabe()
            b=self.model.execute(a)
            self.view.set_ausgabe(b)

if __name__ == '__main__':
    app = QApplication([])
    c = Controller()
    c.view.show()
    sys.exit(app.exec())

exec("exec-test")
eval("eval-test")

username = input("Enter username:")
print("Username is: {}".format(username))
pickle.loads(this)


from typing import Dict

from torch import tensor, cat, max, clamp, mean

from allennlp.models import Model
from allennlp.modules.feedforward import FeedForward
from allennlp.nn.activations import Activation
from allennlp.modules.text_field_embedders import TextFieldEmbedder
from allennlp.modules.similarity_functions.cosine import CosineSimilarity
from allennlp.data.vocabulary import Vocabulary

class CitationRanker(Model):
    def __init__(self,
                 vocab: Vocabulary,
                 title_embedder: TextFieldEmbedder,
                 abstract_embedder: TextFieldEmbedder,
                 dense_dim=75) -> None:

        super().__init__(vocab)

        self.title_embedder = title_embedder
        self.abstract_embedder = abstract_embedder
        self.intermediate_dim = 6
        self.n_layers = 3
        self.layer_dims = [dense_dim for i in range(self.n_layers-1)]
        self.layer_dims.append(1)

        self.activations = [Activation.by_name("elu")(),Activation.by_name("elu")(),Activation.by_name("sigmoid")()]
        self.layers = FeedForward(input_dim=self.intermediate_dim,
                                  num_layers=self.n_layers,
                                  hidden_dims=self.layer_dims,
                                  activations=self.activations)

    def forward(self,
                query_title: Dict[str, tensor],
                query_abstract: Dict[str, tensor],
                candidate_title: Dict[str, tensor],
                candidate_abstract: Dict[str, tensor],
                candidate_citations: tensor,
                title_intersection: tensor,
                abstract_intersection: tensor,
                cos_sim: tensor,
                label: tensor = None) -> Dict[str,tensor]:

        query_title_embed = self.title_embedder.forward(query_title["tokens"])
        query_abstract_embed = self.abstract_embedder.forward(query_abstract["tokens"])

        candidate_title_embed = self.title_embedder.forward(candidate_title["tokens"])
        candidate_abstract_embed = self.abstract_embedder.forward(candidate_title["tokens"])

        title_cos_sim = CosineSimilarity().forward(query_title_embed,candidate_title_embed).unsqueeze(-1)

        abstract_cos_sim = CosineSimilarity().forward(query_abstract_embed,candidate_abstract_embed).unsqueeze(-1)

        intermediate_output = cat((title_cos_sim, abstract_cos_sim, candidate_citations, title_intersection, abstract_intersection, cos_sim), dim = -1)

        pred = self.layers.forward(intermediate_output)

        output = {"cite_prob":pred}

        if label is not None:
            output["loss"] = self._compute_loss(pred,label)

        return output

    def _compute_loss(self, pred: tensor, label: tensor) -> tensor:
        #in existing implementation training examples with even indices are positive/odd indices are negative
        positive = pred[::2]
        negative = pred[1::2]

        #"margin is given by the difference in label"
        margin = label[::2] - label[1::2]
        delta = clamp(margin + negative - positive,min=0)

        return mean(delta)


import openai

openai.api_key = "" #Replace openAI key

import openai

def mental_wellbeing(user_query):
    goal = """
    You are a mental wellbeing assistant. Your role is to provide comfort, mindfulness exercises,
    or general advice to users talking about their feelings or challenges. Remember, you are not a
    replacement for professional help.
    """

    messages = [
        {
            "role": "system",
            "content": goal,
        },
        {
            "role": "user",
            "content": user_query,
        }
    ]

    try:
        response = openai.ChatCompletion.create(
            model="gpt-4",
            messages=messages,
        )
        return response['choices'][0]['message']['content']
    except Exception as e:
        return f"Sorry, there was an error processing your request. Error: {e}"


import mmcv

import argparse
import copy
import json
import os
from collections import defaultdict
from collections import OrderedDict
import os.path as osp
import Polygon as plg

import numpy as np
from PIL import Image
import editdistance

def group_by_key(detections, key):
    groups = defaultdict(list)
    for d in detections:
        groups[d[key]].append(d)
    return groups

def get_union(pa, pb):
    pa_area = pa.area()
    pb_area = pb.area()
    return pa_area + pb_area - get_intersection(pa, pb)

def get_intersection(pa, pb):
    pInt = pa & pb
    if len(pInt) == 0:
        return 0
    else:
        return pInt.area()

def cat_best_hmean(gt, predictions, thresholds):
    num_gts = len([g for g in gt if g['ignore'] == False])
    image_gts = group_by_key(gt, 'name')
    image_gt_boxes = {k: np.array([b['bbox'] for b in boxes])
                      for k, boxes in image_gts.items()}
    image_gt_trans = {k: np.array([b['trans'] for b in boxes])
                      for k, boxes in image_gts.items()}
    image_gt_ignored = {k: np.array([b['ignore'] for b in boxes])
                        for k, boxes in image_gts.items()}
    image_gt_checked = {k: np.zeros((len(boxes), len(thresholds)))
                        for k, boxes in image_gts.items()}
    predictions = sorted(predictions, key=lambda x: x['score'], reverse=True)

    # go down dets and mark TPs and FPs
    nd = len(predictions)
    tp = np.zeros((nd, len(thresholds)))
    fp = np.zeros((nd, len(thresholds)))
    ned = np.zeros((nd, len(thresholds)))
    for i, p in enumerate(predictions):
        pred_polygon = plg.Polygon(np.array(p['bbox']).reshape(-1, 2))
        ovmax = -np.inf
        jmax = -1
        try:
            gt_boxes = image_gt_boxes[p['name']]
            gt_ignored = image_gt_ignored[p['name']]
            gt_checked = image_gt_checked[p['name']]
            gt_trans = image_gt_trans[p['name']]
        except KeyError:
            gt_boxes = []
            gt_checked = None

        if len(gt_boxes) > 0:
            ovmax = 0
            jmax = 0
            for j, gt_box in enumerate(gt_boxes):
                gt_polygon = plg.Polygon(np.array(gt_box).reshape(-1, 2))
                union = get_union(pred_polygon, gt_polygon)
                inter = get_intersection(pred_polygon, gt_polygon)
                overlap = inter / (union + 1e-6)
                if overlap > ovmax:
                    ovmax = overlap
                    jmax = j

        for t, threshold in enumerate(thresholds):
            if ovmax > threshold:
                if gt_checked[jmax, t] == 0:
                    if gt_ignored[jmax]:
                        tp[i, t] = 0.
                        ned[i, t] = 0
                    else:
                        tp[i, t] = 1.
                        ned[i, t] = 1 - editdistance.eval(p['trans'], gt_trans[jmax]) / \
                                    max(len(p['trans']), len(gt_trans[jmax]))
                    gt_checked[jmax, t] = 1
                else:
                    fp[i, t] = 1.
                    ned[i, t] = 0
            else:
                fp[i, t] = 1.

    # compute precision recall
    fp = np.cumsum(fp, axis=0)
    tp = np.cumsum(tp, axis=0)
    ned = np.cumsum(ned, axis=0) / (fp + num_gts + np.finfo(np.float64).eps)

    recalls = tp / float(num_gts)
    # avoid divide by zero in case the first detection matches a difficult
    # ground truth
    precisions = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)

    fmeasures = 2 * precisions * recalls / (precisions + recalls + 1e-6)
    best_i = np.argmax(fmeasures)
    print('[Best F-Measure] p: {:.2f}, r: {:.2f}, f: {:.2f}, 1-ned: {:.2f}, best_score_th: {:.3f}'.format(
        float(precisions[best_i]) * 100, float(recalls[best_i]) * 100, float(fmeasures[best_i]) * 100,
        float(ned[best_i]) * 100, predictions[best_i]['score']))

    best_i = np.argmax(ned)
    print('[Best 1-NED]     p: {:.2f}, r: {:.2f}, f: {:.2f}, 1-ned: {:.2f}, best_score_th: {:.3f}'.format(
        float(precisions[best_i]) * 100, float(recalls[best_i]) * 100, float(fmeasures[best_i]) * 100,
        float(ned[best_i]) * 100, predictions[best_i]['score']))

def trans_pred_format(pred):
    bdd = []
    img_name = pred['img_name']
    ponits = pred['points']
    scores = pred['scores']
    texts = pred['texts']
    for i in range(len(ponits)):
        bdd_i = {
            'category': 'text',
            'timestamp': 1000,
            'name': img_name,
            'bbox': np.array(ponits[i]).reshape(-1).tolist(),
            'score': scores[i],
            'trans': texts[i]
        }
        bdd.append(bdd_i)
    return bdd

def trans_gt_format(img_name, gt):
    bdd = []
    for i in range(len(gt)):
        bdd_i = {
            'category': 'text',
            'timestamp': 1000,
            'name': img_name,
            'bbox': gt[i]['points'],
            'score': 1,
            'ignore': gt[i]['ignore'],
            'trans': gt[i]['transcription']
        }
        bdd.append(bdd_i)
    return bdd

def main():
    cfg = mmcv.Config.fromfile('local_configs/rects_ae_textspotter_lm_r50_1x.py')
    gt_root = cfg.data.val.data_root + cfg.data.val.ann_file
    preds = mmcv.load('results.json')

    preds_bdd = []
    gts_bdd = []
    for pred in preds:
        img_name = pred['img_name']
        pred_bdd = trans_pred_format(pred)
        preds_bdd.extend(pred_bdd)

        gt_path = gt_root + img_name.replace('.jpg', '.json')
        gt = mmcv.load(gt_path)['lines']
        gt_bdd = trans_gt_format(img_name, gt)
        gts_bdd.extend(gt_bdd)

    cat_gt = group_by_key(gts_bdd, 'category')
    cat_pred = group_by_key(preds_bdd, 'category')
    thresholds = [0.5]
    cat_best_hmean(cat_gt['text'], cat_pred['text'], thresholds)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 20 17:05:31 2018

@author: Thomas Schatz

Command-line interface to precompute mp-scores for all result files
in a specified folder.

Usage example:
   cd path/2/ABX/results/folder/
   mkdir minimal_pair
   python precompute_mp_scores.py ./ minimal_pair
"""

import argparse
import os.path as path
import scone_phobia.utils.mp_scores as mp_scores

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('in_dir', help="ABX results files folder")
    parser.add_argument('out_dir', help="Folder where to store mp scores")
    parser.add_argument('--overwrite', action='store_true',
                        help=("use this if you want to overwrite existing"
                              " pickles in out_dir"))
    parser.add_argument('--mp_type', default='spk_first',
                        help="Type of minimal-pair scores to compute")
    args = parser.parse_args()
    assert path.exists(args.in_dir), \
        "Input folder {} missing".format(args.in_dir)
    assert path.exists(args.out_dir), \
        "Output folder {} missing".format(args.out_dir)
    if args.overwrite:
        filt = None
        print(("precompute_mp_scores.py: overwriting any pre-existing"
               " pickle in {}".format(args.out_dir)))
    else:
        filt = lambda x: not(path.exists(path.join(args.out_dir, x+'.pickle')))
        print(("precompute_mp_scores.py: mp scores in pre-existing pickles"
               " in {} will not be computed again, use the --overwrite"
               " switch if you want to force them to be computed again."
               ).format(args.out_dir))
    mp_scores.precompute_mp_scores(args.in_dir, args.out_dir, filt=filt, mp_type=args.mp_type)


"""Compute genre-weighted typicality as described by Askin and Mauskapf"""

import numpy as np
import pandas as pd
from scipy.spatial.distance import cosine
import string
import ast
import copy
import math
import string

fpath='data/genre_set.tsv'
df=pd.read_csv(fpath,sep='\t',encoding='utf8')

# Convert the release dates to datetime objects
df.release=pd.to_datetime(df.release)
dates=df.release.unique()

# Discogs genre
mat=df.discog_genre.as_matrix()
mat=[ast.literal_eval(x) for x in mat]
genres=['Blues', "Children's", 'Classical', 'Electronic','Folk, World, & Country', 'Funk / Soul', 'Hip Hop', 'Jazz',
'Latin', 'Non-Music', 'Pop', 'Reggae', 'Rock','Stage & Screen']

# Temporal genre data frame
df_tg=pd.DataFrame(columns=genres+['weight'])
missing=[]

def format_echo(tempo,time_signature):
	"""Simple method to normalize tempo and time signature"""
	return (tempo/max(tempo)).as_matrix(), ((time_signature==4)).as_matrix()

df.tempo,df.time_signature=format_echo(df.tempo,df.time_signature)

def key_match(a,b):
	"""Check for key match"""
	return sum([x==y for x in a for y in b])/float(len(a)*len(b))

# Iterate through dates and get the average distance between songs from genres over time
for d in dates:
	gv=[]
	# Iterate through genres
	for g in genres:
		# Find the mean values for tracks with release up to 1 year before date and with a specific genre
		record=df.loc[(df.release<=d) & (df.release>=d-np.timedelta64(52,'W'))&(df.discog_genre.str.contains(g)),['acousticness',
			'danceability','energy','instrumentalness','liveness','speechiness','tempo','valence','mode']].copy()
		if len(record)==0:
			missing.append((d,g))
			#print('No records found for genre '+g+' at date '+str(d))
			continue
		#key=record['key'].tolist()
		record=record.mean()
		gv.append([record,g])

	# Calculate the weight between genres
	for i in range(len(gv)):
		for j in range(i+1,len(gv)):
			genre_1=copy.copy(gv[i])
			genre_2=copy.copy(gv[j])

			#genre_1[0].key=1
			#genre_2[0].key=key_match(genre_1[1],genre_2[1])

			# Cosine similarity
			weight=1-cosine(genre_1[0],genre_2[0])

			# Store weights in DataFrame
			row=np.zeros(len(genres))
			row[genres.index(genre_1[1])]=1
			row[genres.index(genre_2[1])]=1
			row=row.tolist()+[weight]
			row=pd.DataFrame([row],columns=df_tg.columns)
			row=row.xs(0)
			row.name=d
			df_tg=df_tg.append(row)
	print("progress: "+str(np.where(dates==d)[0][0]/float(len(dates))))

def avg_typ(track_1,track_r,weights,genres):
	"""Find genre-weighted average typicality

		Parameters
		----------
		track_1 : Pandas DataFrame element
		Track of interest
		track_r : Pandas DataFrame
		Related tracks that were active one year from the track's release
		weights : Pandas DataFrame
		Genre weights at the release date
		genres : List
		Genres used
	"""
	# Song record that can be manipulated
	og_track_1=track_1.copy()

	# Aggregate typicality
	agr_typ=0

	# Iterate through comparison tracks
	for x in track_r.iterrows():
		track_1=og_track_1.copy()
		track_2=x[1].copy()

		# Matching genres, no need to change weight
		if track_1.discog_genre==track_2.discog_genre:
			weight=1
		# Different genres, find average distance between genres
		else:
			wm=[]
			genre_1=[x for x in genres if x in track_1.discog_genre]
			genre_2=[x for x in genres if x in track_2.discog_genre]

			for g1 in genre_1:
				for g2 in genre_2:
					if g1==g2:
						wm.append(1)
					else:
						wm.append(weights[(weights[g1]==1)&(weights[g2]==1)]['weight'].values[0])
			# Normalize weight
			weight=sum(wm)/len(genre_1)/len(genre_2)

		# track_1.key=1
		# track_2.key=key_match([track_1.key],[track_2.key])

		agr_typ+=weight*(1-cosine(track_1[:-1].astype(float),track_2[:-1].astype(float)))

	# Normalize aggregate similarity
	return agr_typ/len(track_r)

# Iterate through files alphabetically
file_list=['number']+list(string.ascii_uppercase)

# Typicality
df['typ']=0

# Missing records
missing_typ=[]

for fname in file_list:
	if fname=='number':
		numbers=[False if x in string.ascii_uppercase else True for x in df.artist.str[0]]
		df_sub=df.copy()[numbers]
	else:
		df_sub=df.copy()[df.artist.str[0]==fname]

	for x in df_sub.iterrows():
		artist=x[1].artist
		title=x[1].title
		release=x[1].release

		# Song record
		track_1=x[1][['acousticness','danceability','energy','instrumentalness','liveness','speechiness',
			'tempo','valence','mode','discog_genre']]

		# Songs released in the previous 52 weeks
		track_r=df.loc[(df.release<=release) & (df.release>=release-np.timedelta64(52,'W')),['acousticness','danceability','energy',
			'instrumentalness','liveness','speechiness','tempo','valence','mode','discog_genre']].drop(x[0])

		# Genre weights from past 52 weeks
		weights=df_tg.loc[df_tg.index==release]

		try:
			typ_score=avg_typ(track_1,track_r,weights,genres)
		except:
			print("No artists released in the previous year for track "+title+" by artist "+artist)
			missing_typ.append((artist,title))
			df_sub.drop(x[0],inplace=True)

		df_sub.loc[(df_sub.title == title) & (df_sub.artist == artist),'typ']=typ_score
	df_sub.to_csv('data/typ/'+fname+'.tsv',sep='\t',index=False,encoding='utf8')
	print(fname)

# Write missing records to file
with open('data/typ/missing.tsv','wb') as f:
	f.write('date\tgenre\n')
	for x in missing:
		date=str(x[0]).encode('utf8')
		genre=x[1].encode('utf8')
		f.write(date+'\t'+genre+'\n')


# -*- coding: utf-8 -*-
"""
: Sat Mar 25 2020, 08:30:56

version: Version 1

Description:
    1. Developed a function to compute Earth repeat orbits geometry when
        a or i, j, k, e are provided as inputs
    2. Low fidelity approach (default) where Earth-repeat orbit solutions are
        computed cosindering the effect of $J_2$ perturbation on $\Omega$ only
    3. High fidelity approach where Earth-repeat orbit solutions are
        computed cosindering the effect of $J_2$ perturbation on $\Omega$,
        $\omega$, and $M$
    4. Either altitude or inclination values (minimum,maximum,step size) are
        to be provided.
    5. Low fidelity approach does not work when the inclination is known
        while altitude is unknown
    6. For eccentric orbits, altitude values (input and output) should
        corrospond to the perigee altitude.

"""

import numpy as np2

def EarthRepeatOrbits(jk,e,Variable,VarType,isHighFidelity=False,printStatus=False):
    # importing the required modules
    import spiceypy as spice
    import math
    import numpy as np

    Result = 0
    Nsolutions = 0

    # Error handling
    jkSize = jk.shape
    if (int(jkSize[1])!=2):
        print('Incorrect matrix size for jk matrix. Only two coloumns are required.')
        return Result
    elif (VarType != 'Alti') and (VarType != 'Inclin'):
        print('Inrecognized input for the argument specifying the variable type.')
        return Result
    elif e >= 1.0 or e < 0:
        print('Only circular or elliptical orbits are possible. Check the value of eccentricity.')
        return Result
    elif (Variable[1]-Variable[0])% Variable[2] !=0:
        print('Integer number of steps are not possible. Check inputs for the argument - "Variable".')
        return Result

    # Extracting the parameters
    spice.furnsh("./External_files/Spice_kernels/kernel_load.txt")
    muE = spice.bodvrd('Earth','GM',1)
    mu = muE[1][0] #km3/s2 for earth
    J2 = 1082.63E-6 #J2 for earth - import from spice
    RE = spice.bodvrd('EARTH','RADII',3)
    Re= RE[1][0]
    De = 86164.1004 # s, Sidereal day, to be extracted from spice
    k2 = 0.75 * J2 * math.sqrt(mu)* Re*Re
    r2d = 1/spice.rpd() # Radian to degree conversion

    # Creating storage
    steps = int((Variable[1]-Variable[0])/Variable[2])+1
    Result = np.zeros(int(jkSize[0])*int(steps)*6)
    Result.shape= [int(jkSize[0]),int(steps),6]

    # Computations
    for count in range(0,jkSize[0]):
#        print(count)
        for rowCount in range(0,int(steps)):
            j = jk[count,0]
            k = jk[count,1]

            # Extracting the value of variable
            var = Variable[0]+rowCount*Variable[2]

            # Storing values known so far
            Result[count,rowCount,0] = j
            Result[count,rowCount,1] = k
            Result[count,rowCount,2] = e
            Result[count,rowCount,3] = var
            Result[count,rowCount,4] = math.nan # to be computed
            Result[count,rowCount,5] = math.nan # to be computed

            # computations
            if isHighFidelity == False:
#                print('Approach 1')
                if VarType == 'Alti':
#                    print('Inclination is unknown')
#                    if e == 0:
#                        a = Re + var
#                        T = 2 * math.pi * math.sqrt(a**3/mu)
#                    else:
#                        print('Problem not defined yet for non-circular orbits.')
#                        sys.exit # DOES NOT WORK PROPERLY, LOOP IS NOT BROKEN
                    a = (Re + var)/(1-e)
                    T = 2 * math.pi * math.sqrt(a**3/mu)

                    DeltaL1 = -2 * math.pi * (T/De)
                    # DeltaL2 = C1 * cosi
                    C1 = (-3*math.pi * J2 * Re**2)/(a**2 * (1 - e**2)**2)
                    C2 = -2 * math.pi* k/j  - DeltaL1
                    C3 = 2 * math.pi* k/j  - DeltaL1

                    # Inverse cosine
                    if (C2 <0) & (abs(C2/C1) <=1):
                        DeltaL2 = C2
                        i = math.acos(DeltaL2/C1) * r2d # for i = [0, 90] deg
                        Result[count,rowCount,4] = i
                        Nsolutions += 1
                    elif (C2 > 0) & (abs(C2/C1) <=1):
                        DeltaL2 = C2
                        i = math.acos(DeltaL2/C1) * r2d # for i = [90, 180] deg
                        Result[count,rowCount,4] = i
                        Nsolutions += 1
                    elif ( C3 < 0) & (abs(C3/C1) <=1): # this was missing in MATLAB script
                        DeltaL2 = C3
                        i = math.acos(DeltaL2/C1) * r2d # for i = [0, 90] deg
                        Result[count,rowCount,5] = i
                        Nsolutions += 1
                    elif (C3>0) & (abs(C3/C1) <=1):
                        DeltaL2 = C3
                        i = math.acos(DeltaL2/C1) * r2d # for i = [90, 180] deg
                        Result[count,rowCount,5] = i
                        Nsolutions += 1
                elif VarType == 'Inclin':
                    print('Function is not defined for low fidelity + unknown inclination case.')
                    return Result
            else:
#                print('Approach 2')
                if VarType == 'Alti':
#                    if e == 0:
#                        a = Re + var
#                    else:
#                        print('Problem not defined yet for non-circular orbits.')
#                        sys.exit
                    a = (Re + var)/(1-e)
                    C1 = -2 * k2 * (a**-3.5) * ((1-e**2)**-2) * De*r2d  # RAAN_dot = C1 * cos(i)
                    C2 = k2 * (a**-3.5) *((1-e**2)**-2) * De*r2d # omega_dot = C2 *(5*(cosd(i))^2 -1)
                    C3 = k2 * (a**-3.5) *((1-e**2)**-1.5) * De * r2d # M_dot = C3 *(3*(cosd(i))^2 -1)
                    n = De * r2d * math.sqrt(mu/a**3);

                    # For the quadratic equation A*x^2 + B*x +C = 0 with x = cos(i)
                    A = 5* C2 + 3 * C3
                    B = C1 * j/k
                    C = n - 360 *j/k - (C2 + C3)
                    D = B**2-4*A*C

                    if D >=0:
                        if (D == 0) & (abs(-B/(2*A)) <=1):
                            i1 = math.acos(-B/(2*A))
                            Result[count,rowCount,4] = i1*r2d
                            Nsolutions += 1
                        elif (D > 0):
                            x1 = (-B + math.sqrt(D))/ (2 * A)
                            x2 = (-B - math.sqrt(D))/ (2 * A)
                            if abs(x1) <=1:
                                i1 = math.acos(x1)
                                Result[count,rowCount,4] = i1*r2d
                                Nsolutions += 1
                            if abs(x2) <=1:
                                i2 = math.acos(x2)
                                Result[count,rowCount,5] = i2*r2d
                                Nsolutions += 1
                elif VarType == 'Inclin':

                    a0 = math.pow((mu*De**2*k**2/(4*math.pi**2*j**2)),1/3)
                    iterations = 0
                    i = var
                    L_dot = 360

                    while True:
                        iterations = iterations+1

                        RAAN_dot = - 2 * k2 * math.pow(a0,-3.5) * math.cos(i/r2d) * math.pow(1-e**2,-2) * De * r2d
                        #       print(RAAN_dot)
                        omega_dot = k2 * math.pow(a0,-3.5) * (5*(math.cos(i/r2d))**2 -1) * math.pow(1-e**2,-2) * De * r2d
                        #       print(omega_dot)
                        M_dot = k2 * math.pow(a0,-3.5) * (3*(math.cos(i/r2d))**2 -1) * math.pow(1-e**2,-1.5) * De * r2d
                        #       print(M_dot)
                        n = (j/k) * (L_dot - RAAN_dot) -(omega_dot + M_dot)
                        #       print(n)
                        a1 = (mu/(n/(De*r2d))**2)**(1/3)
                        #       print('a1 is:',a1)

#                      # print('At iteration:',iterations,' a1 was ',a1, 'a0 was', a0)
                        if iterations > 1000:
#                            print('Maximum number of iterations executed')
                            a0 = a1
                            break
                        elif (abs(a1-a0) < 10**(-10)):
#                            print('Convergence reached after ',iterations, 'iterations')
                            a0 = a1
                            break
                        else:
                            a0 = a1

                    # Only tracking the feasible solutions
                    if ((a0 * (1-e)) - Re) > 0:
                        Nsolutions += 1
                        Result[count,rowCount,4] = (a0 * (1-e)) - Re

    # Printing the status of solutions obtained
    if printStatus == True:
        print(Nsolutions, ' solutions are obtained for the Earth-repeat orbits.')

#    print(Result.shape)
    return Result

##########################
######### Executing runs
##########################

Run = 1 # 1 - Approach 1 with i as unknown
        # 2 - Approach 2 with i as unknown
        # 3 - Approach 2 with altitude as unknown

# Approach 1, i as unknown
if Run == 1:
#    jk = np.matrix('14, 1; 43, 3; 29, 2; 59, 4; 74, 5 ; 15,1')
    jk = np2.matrix('39, 3; 40, 3; 41, 3; 42, 3; 43,3; 44, 3 ; 45,3; 46, 3; 47,3 ; 48, 3')
    e = 0.1
    a = np2.array([200,2000,5])
    Result = EarthRepeatOrbits(jk,e,a,'Alti',False,True)

# Approach 2, i as unknown
elif Run == 2:
    jk = np2.matrix('39, 3; 40, 3; 41, 3; 42, 3; 43,3; 44, 3 ; 45,3; 46, 3; 47,3 ; 48, 3')
    e = 0.1
    a = np2.array([200,2000,5])
    Result = EarthRepeatOrbits(jk,e,a,'Alti',True,True)

# Approach 2 with altitude as unknown
elif Run == 3:
    jk = np2.matrix('14, 1; 43, 3; 29, 2; 59, 4; 74, 5 ; 15,1')
    e = 0
    a = np2.array([28,29,1])
    Result = EarthRepeatOrbits(jk,e,a,'Inclin',True,True)


import traceback
from typing import Any, Callable, Dict, List

from .abc import (
    CallbackClientABC,
    DataABC,
    LogABC,
    RpcServerABC,
)
from .definitions import Definition, DefinitionMemsync
from .mempkg import Mempkg
from .typeguard import typechecked

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# CALLBACK CLIENT CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

@typechecked
class CallbackClient(CallbackClientABC):
    """
    Representing a callback function
    """

    def __init__(
        self,
        name: str,
        handler: Callable,
        rpc_server: RpcServerABC,
        data: DataABC,
        log: LogABC,
        argtypes: List[Definition],
        restype: Definition,
        memsyncs: List[DefinitionMemsync],
    ):

        self._name = name
        self._handler = handler
        self._data = data
        self._log = log

        self._argtypes = argtypes
        self._restype = restype
        self._memsyncs = memsyncs

        rpc_server.register_function(self, public_name=name)

    def __call__(self, packed_args: List[Any], packed_mempkgs: List[Dict]) -> Dict:

        self._log.info(f'[callback-client] Trying to call callback routine "{self._name:s}" ...')

        try:
            args = self._data.unpack_args(packed_args, self._argtypes)
            retval = None
            mempkgs = [Mempkg.from_packed(packed_mempkg) for packed_mempkg in packed_mempkgs]
            DefinitionMemsync.unpkg_memories(
                args = args,
                retval = retval,
                mempkgs = mempkgs,
                memsyncs = self._memsyncs,
                is_server = True,
            )
        except Exception as e:
            self._log.error("[callback-client] ... call preparation failed!")
            self._log.error(traceback.format_exc())
            raise e

        try:
            retval = self._handler(*args)
        except Exception as e:
            self._log.error("[callback-client] ... call failed!")
            self._log.error(traceback.format_exc())
            return {
                "args": packed_args,  # unchanged
                "retval": retval,  # unchanged, still None
                "mempkgs": packed_mempkgs,  # unchanged
                "success": False,
                "exception": e,
            }

        try:
            DefinitionMemsync.update_memories(
                args = args,
                retval = retval,
                mempkgs = mempkgs,
                memsyncs = self._memsyncs,
            )
            self._log.info("[callback-client] ... done.")
            return {
                "args": self._data.pack_args(args, self._argtypes),
                "retval": self._data.pack_retval(retval, self._restype),
                "mempkgs": [mempkg.as_packed() for mempkg in mempkgs],
                "success": True,
                "exception": None,
            }
        except Exception as e:
            self._log.error("[callback-client] ... call post-processing failed!")
            self._log.error(traceback.format_exc())
            raise e


import requests
from bs4 import BeautifulSoup
import re
import time
import pandas as pd

start = time.perf_counter()

session = requests.Session()
sitemap_response = session.get('YOUR_SITEMAP_URL')
sitemap_soup = BeautifulSoup(sitemap_response.content, 'lxml')
sitemap_urls = sitemap_soup.find_all("loc")

unique_urls = set()
for sitemap_url in sitemap_urls:
    unique_urls.add(sitemap_url.text)

search_terms = {
    'Your keyword1': False,
    'Your keyword2': False,
    'Your keyword3': False,
    'Your keyword4': False
}

results = []
for url in unique_urls:
    print(f'Requesting {url}')
    source_response = session.get(url)
    source_soup = BeautifulSoup(source_response.text, 'html.parser')

    for search_term in search_terms:
        search_terms[search_term] = bool(source_soup(text=re.compile(search_term)))

    results.append({
        'url': url,
        **search_terms
    })
print(results)
results_df = pd.DataFrame(results)
results_df.to_excel('YOUR_PATH/results.xlsx')

elapsed = time.perf_counter() - start
print(f'Time elapsed: {elapsed:0.6f} seconds')


import copy
import logging
import numpy as np
from tqdm import tqdm
import torch
import nni
from nni.utils import merge_parameter
import dgl

from graphmae.utils import (
    build_args,
    create_optimizer,
    set_random_seed,
    TBLogger,
    get_current_lr,
    load_best_configs
)
from graphmae.datasets.data_util import load_dataset
from graphmae.evaluation import node_classification_evaluation, link_prediction_for_transductive, \
    clustering_for_transductive, compute_auc, DotPredictor
from graphmae.models import build_model

logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO)

def pretrain(model, decoder, graph, x, optimizer, max_epoch, device, scheduler, logger=None, mute=False):
    logging.info("start training..")

    decoder.to(device)
    criterion = torch.nn.BCEWithLogitsLoss()
    graph = graph.to(device)
    x = x.to(device)

    num_nodes = graph.number_of_nodes()
    u, v = graph.edges()
    num_edges = graph.number_of_edges()
    eids = torch.randperm(num_edges).to(device)
    test_size = int(eids.shape[0] * 0.1)
    test_pos_u, test_pos_v = u[eids[:test_size]], v[eids[:test_size]]
    train_pos_u, train_pos_v = u[eids], v[eids]

    num_samples = num_edges
    neg_u, neg_v = dgl.sampling.global_uniform_negative_sampling(graph, num_samples, exclude_self_loops=True)
    neg_eids = torch.randperm(num_edges)
    test_neg_u, test_neg_v = neg_u[neg_eids[:test_size]], neg_v[neg_eids[:test_size]]
    train_neg_u, train_neg_v = neg_u[neg_eids], neg_v[neg_eids]

    # train_g = dgl.remove_edges(graph, eids[:test_size])
    train_g = graph
    # train_g = dgl.add_self_loop(train_g)
    # test_g = graph.edge_subgraph(eids[:test_size])
    train_pos_g = dgl.graph((train_pos_u, train_pos_v), num_nodes=num_nodes)
    train_neg_g = dgl.graph((train_neg_u, train_neg_v), num_nodes=num_nodes)
    test_pos_g = dgl.graph((test_pos_u, test_pos_v), num_nodes=num_nodes)
    test_neg_g = dgl.graph((test_neg_u, test_neg_v), num_nodes=num_nodes)

    best_test_auc = 0
    best_epoch = 0
    best_model = None

    epoch_iter = tqdm(range(max_epoch))
    for epoch in epoch_iter:
        model.train()
        loss, loss_dict = model(graph, x)
        out = model.embed(train_g, train_g.ndata["feat"])
        pos_score = decoder(train_pos_g, out)
        neg_score = decoder(train_neg_g, out)
        loss_lp = criterion(pos_score, neg_score)
        loss = loss + loss_lp
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if scheduler is not None:
            scheduler.step()

        epoch_iter.set_description(f"# Epoch {epoch}: train_loss: {loss.item():.4f}")
        if logger is not None:
            loss_dict["lr"] = get_current_lr(optimizer)
            logger.note(loss_dict, step=epoch)

        if (epoch + 1) % 100 == 0:
            with torch.no_grad():
                model.eval()
                out = model.embed(train_g, train_g.ndata["feat"])
                pos_score_test = decoder(test_pos_g, out)
                neg_score_test = decoder(test_neg_g, out)
                test_auc = compute_auc(pos_score_test, neg_score_test)
                # test_auc = compute_ap(pos_score_test, neg_score_test)
                pos_score_train = decoder(train_pos_g, out)
                neg_score_train = decoder(train_neg_g, out)
                train_auc = compute_auc(pos_score_train, neg_score_train)
                # train_auc = compute_ap(pos_score_train, neg_score_train)
                test_loss = criterion(pos_score_test, neg_score_test)
            if test_auc >= best_test_auc:
                best_test_auc = test_auc
                # print('best', best_test_auc)
                best_epoch = epoch
                # print(best_epoch)
                best_model = copy.deepcopy(decoder)

            if not mute:
                epoch_iter.set_description(
                    f"# Epoch: {epoch}, train_loss:{loss.item(): .4f}, train_auc:{train_auc: .4f}, "
                    f"test_loss:{test_loss.item(): .4f}, test_auc:{test_auc: .4f}")

            best_model.eval()
            with torch.no_grad():
                pos_score = best_model(test_pos_g, out)
                neg_score = best_model(test_neg_g, out)
                estp_test_auc = compute_auc(pos_score, neg_score)
                # estp_test_auc = compute_ap(pos_score, neg_score)
            if mute:
                print(
                    f"# IGNORE: --- TestAUC: {test_auc:.4f}, early-stopping-TestAUC: {estp_test_auc:.4f} in epoch {best_epoch} --- ")
            else:
                print(
                    f"--- TestAUC: {test_auc:.4f}, early-stopping-TestAUC: {estp_test_auc:.4f} in epoch {best_epoch} --- ")

    # (final_acc, es_acc, best_acc)
    return test_auc, estp_test_auc

def main(args):
    device = args.device if args.device >= 0 else "cpu"
    seeds = args.seeds
    dataset_name = args.dataset
    max_epoch = args.max_epoch
    max_epoch_lp = args.max_epoch_lp
    num_hidden = args.num_hidden
    num_layers = args.num_layers
    encoder_type = args.encoder
    decoder_type = args.decoder
    replace_rate = args.replace_rate
    optim_type = args.optimizer
    loss_fn = args.loss_fn
    lr = args.lr
    weight_decay = args.weight_decay
    lr_lp = args.lr_lp
    weight_decay_lp = args.weight_decay_lp
    logs = args.logging
    use_scheduler = args.scheduler

    graph, (num_features, num_classes) = load_dataset(dataset_name)
    args.num_features = num_features

    auc_list = []
    estp_auc_list = []

    for i, seed in enumerate(seeds):
        print(f"####### Run {i} for seed {seed}")
        set_random_seed(seed)

        model = build_model(args)
        decoder = DotPredictor()
        # print(model)
        optimizer = torch.optim.Adam(
            list(decoder.parameters()) + list(model.parameters()), lr=lr, weight_decay=weight_decay
        )
        model.to(device)

        if use_scheduler:
            logging.info("Use schedular")
            scheduler = lambda epoch: (1 + np.cos((epoch) * np.pi / max_epoch)) * 0.5
            # scheduler = lambda epoch: epoch / warmup_steps if epoch < warmup_steps \
            # else ( 1 + np.cos((epoch - warmup_steps) * np.pi / (max_epoch - warmup_steps))) * 0.5
            scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=scheduler)
        else:
            scheduler = None

        x = graph.ndata["feat"]

        final_auc, estp_auc = pretrain(model,decoder, graph, x, optimizer, max_epoch, device, scheduler, )
        # model = model.cpu()
        #
        # model = model.to(device)
        # model.eval()
        #
        # final_auc, estp_auc = link_prediction_for_transductive(model, graph, x, lr_lp, weight_decay_lp,
        #                                                        max_epoch_lp,
        #                                                        device, test_ratio=0.1, mute=True)
        nni.report_final_result({'default': final_auc})
        auc_list.append(final_auc)
        estp_auc_list.append(estp_auc)
        #
        # if logger is not None:
        #     logger.finish()

    final_auc, final_auc_std = np.mean(auc_list), np.std(auc_list)
    estp_auc, estp_auc_std = np.mean(estp_auc_list), np.std(estp_auc_list)
    print(f"# final_auc: {final_auc:.4f}±{final_auc_std:.4f}")
    print(f"# early-stopping_auc: {estp_auc:.4f}±{estp_auc_std:.4f}")

# Press the green button in the gutter to run the script.
if __name__ == "__main__":
    args = build_args()
    args = load_best_configs(args, "configs_lp.yml")

    nni_params = nni.get_next_parameter()
    args = merge_parameter(args, nni_params)
    print(args)

    main(args)


from django.views.generic import TemplateView
from django.contrib.auth.models import User, Group
from django.views.generic.edit import CreateView, UpdateView
from .models import BaseRegisterForm
from django.shortcuts import redirect
from django.contrib.auth.decorators import login_required

class BaseRegisterView(CreateView):
    model = User
    form_class = BaseRegisterForm
    success_url = "/"

class IndexView(TemplateView):
    template_name = 'login/index.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['is_not_author'] = not self.request.user.groups.filter(name='author').exists()
        return context

@login_required
def upgrade_me(request):
    user = request.user
    author_group = Group.objects.get(name='author')
    if not request.user.groups.filter(name='author').exists():
        author_group.user_set.add(user)
    return redirect('/')


# Definition for a binary tree node.
from typing import List

class TreeNode:
	def __init__(self, val=0, left=None, right=None):
		self.val = val
		self.left = left
		self.right = right

class Solution:
	def binaryTreePaths(self, root: TreeNode) -> List[str]:
		if not root.right and not root.left:
			return [f'{root.val}']

		childPaths = []
		if root.right:
			childPaths.extend(self.binaryTreePaths(root.right))
		if root.left:
			childPaths.extend(self.binaryTreePaths(root.left))
		return [f'{root.val}->{path}' for path in childPaths]

print(Solution().binaryTreePaths(TreeNode(1, TreeNode(2, None, TreeNode(5)), TreeNode(3))))


#!env/bin/python
from flask import Flask, jsonify
from tello import Tello

app = Flask(__name__)
tello = Tello()
tello.initialize()

@app.route('/drone/command/takeoff', methods=['POST'])
def drone_takeoff():
    print('Drone now taking off...')
    tello.takeoff()
    return jsonify({'status': 'OK'})

@app.route('/drone/command/land', methods=['POST'])
def drone_land():
    print('Drone now landing...')
    tello.land()
    return jsonify({'status': 'OK'})

if __name__ == '__main__':
    app.run(debug=False, host='10.0.0.3')
    tello.end()


import json
from typing import Any
from unittest.mock import MagicMock, Mock

import pytest
from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
from airbyte_cdk.sources.streams import Stream
from requests import Response
from source_monday.components import IncrementalSingleSlice, IncrementalSubstreamSlicer
from source_monday.extractor import MondayIncrementalItemsExtractor

def _create_response(content: Any) -> Response:
    response = Response()
    response._content = json.dumps(content).encode("utf-8")
    return response

def test_slicer():
    date_time_dict = {"updated_at": 1662459010}
    slicer = IncrementalSingleSlice(config={}, parameters={}, cursor_field="updated_at")
    slicer.close_slice(date_time_dict, date_time_dict)
    assert slicer.get_stream_state() == date_time_dict
    assert slicer.get_request_headers() == {}
    assert slicer.get_request_body_data() == {}
    assert slicer.get_request_body_json() == {}

@pytest.mark.parametrize(
    "last_record, expected, records",
    [
        (
            {"first_stream_cursor": 1662459010},
            {"parent_stream_name": {"parent_cursor_field": 1662459010}, "first_stream_cursor": 1662459010},
            [{"first_stream_cursor": 1662459010}],
        ),
        (None, {}, []),
    ],
)
def test_sub_slicer(last_record, expected, records):
    parent_stream = Mock(spec=Stream)
    parent_stream.name = "parent_stream_name"
    parent_stream.cursor_field = "parent_cursor_field"
    parent_stream.stream_slices.return_value = [{"a slice": "value"}]
    parent_stream.read_records = MagicMock(return_value=records)

    parent_config = ParentStreamConfig(
        stream=parent_stream,
        parent_key="id",
        partition_field="first_stream_id",
        parameters={},
        config={},
    )

    slicer = IncrementalSubstreamSlicer(
        config={}, parameters={}, cursor_field="first_stream_cursor", parent_stream_configs=[parent_config], nested_items_per_page=10
    )
    stream_slice = next(slicer.stream_slices()) if records else {}
    slicer.close_slice(stream_slice, last_record)
    assert slicer.get_stream_state() == expected

def test_null_records(caplog):
    extractor = MondayIncrementalItemsExtractor(
        field_path=["data", "boards", "*"],
        config={},
        parameters={},
    )
    content = {
        "data": {
            "boards": [
                {"board_kind": "private", "id": "1234561", "updated_at": "2023-08-15T10:30:49Z"},
                {"board_kind": "private", "id": "1234562", "updated_at": "2023-08-15T10:30:50Z"},
                {"board_kind": "private", "id": "1234563", "updated_at": "2023-08-15T10:30:51Z"},
                {"board_kind": "private", "id": "1234564", "updated_at": "2023-08-15T10:30:52Z"},
                {"board_kind": "private", "id": "1234565", "updated_at": "2023-08-15T10:30:43Z"},
                {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z"},
                None,
                None,
            ]
        },
        "errors": [{"message": "Cannot return null for non-nullable field Board.creator"}],
        "account_id": 123456,
    }
    response = _create_response(content)
    records = extractor.extract_records(response)
    warning_message = "Record with null value received; errors: [{'message': 'Cannot return null for non-nullable field Board.creator'}]"
    assert warning_message in caplog.messages
    expected_records = [
        {"board_kind": "private", "id": "1234561", "updated_at": "2023-08-15T10:30:49Z", "updated_at_int": 1692095449},
        {"board_kind": "private", "id": "1234562", "updated_at": "2023-08-15T10:30:50Z", "updated_at_int": 1692095450},
        {"board_kind": "private", "id": "1234563", "updated_at": "2023-08-15T10:30:51Z", "updated_at_int": 1692095451},
        {"board_kind": "private", "id": "1234564", "updated_at": "2023-08-15T10:30:52Z", "updated_at_int": 1692095452},
        {"board_kind": "private", "id": "1234565", "updated_at": "2023-08-15T10:30:43Z", "updated_at_int": 1692095443},
        {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z", "updated_at_int": 1692095454},
    ]
    assert records == expected_records


import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import pandas as pd

### Importation de la base de données

database = pd.read_csv("/Users/jeanprolhac/Desktop/Projet/Projet 16 - IA et fraude/Detection-de-fraudes/labelled_database.tsv", sep="\t")

### Création du trainset et du testset

trainset = []
testset = []

traindata = database.loc[1:50]
testdata = database.loc[50:100]
trainincome = list(traindata["income"])
traintaxclass = list(traindata["taxclass"])
trainfraud = list(traindata["fraud"])
testincome = list(testdata["income"])
testtaxclass = list(testdata["taxclass"])
testfraud = list(testdata["fraud"])

for i in range(len(trainincome)) :
    trainset.append((torch.tensor([[float(trainincome[i]), float(traintaxclass[i])]]), torch.tensor([trainfraud[i]])))
    testset.append((torch.tensor([[float(testincome[i]), float(testtaxclass[i])]]), torch.tensor([testfraud[i]])))

### Début duu réseau de neurones

class Net(nn.Module) :

    #define the layers
    def __init__(self) :
        super().__init__()
        self.fc1 = nn.Linear(2, 64)
        self.fc2 = nn.Linear(64, 64)
        self.fc3 = nn.Linear(64, 64)
        self.fc4 = nn.Linear(64, 2)

    def forward(self, x) :
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        x = self.fc4(x)
        return F.log_softmax(x, dim=1)

net = Net()

### Début de l'entraînement

optimizer = optim.Adam(net.parameters(), lr =0.001)

# Le nombre de fois qu'on va parcourir le trainset pour entrainer notre algo

EPOCHS = 5

for epoch in range(EPOCHS) :
    for data in trainset :
        X, y = data
        net.zero_grad()
        output = net(X.view(-1, 1*2))
        loss = F.nll_loss(output, y)
        loss.backward()
        optimizer.step()
    print(loss)

### Début des mesures de performance

correct = 0
total = 0

with torch.no_grad() :
    for data in testset :
        X, y = data
        output = net(X.view(-1, 2))
        for idx, i in enumerate(output):
            if torch.argmax(i) == y[idx]:
                correct += 1
            total += 1

print("Accuracy: ", round(correct/total, 3))

### Test sur un cas

X = torch.tensor([50000., 50000.])
X = X.view(-1, 1*2)
output = net(X)
print(output)
for idx, i in enumerate(output):
    if torch.argmax(i) == 0:
        print("Non fraudeur")
    else :
        print("Fraudeur")


# parse.py

import requests
import json
import datetime

headers = {
	"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0",
	"X-Requested-With": "XMLHttpRequest"
}

def get_data():
	start_time = datetime.datetime.now()
	url = "https://roscarservis.ru/catalog/legkovye/?set_filter=Y&sort%5Bprice%5D=asc&PAGEN_1=1"
	r = requests.get(url=url, headers=headers)

	#with open("index.html", "w") as file:
	#	file.write(r.text)
	#print(r.json())

	#with open("data.json", "w") as file:
	#	json.dump(r.json(), file, indent=4, ensure_ascii=False)

	pages_count = r.json()["pagesCount"]

	data_list = []

	for page in range(1, pages_count + 1):
		url = f"https://roscarservis.ru/catalog/legkovye/?set_filter=Y&sort%5Bprice%5D=asc&PAGEN_1={page}"
		r = requests.get(url=url, headers=headers)
		data = r.json()
		items = data["items"]
		possible_stores = ["discountStores", "externalStores", "commonStores"]

		for item in items:
			total_amount = 0

			item_id = item["id"]
			item_name = item["name"]
			item_url = f'https://roscarservis.ru{item["url"]}'
			item_season = item["season"]
			item_price = item["price"]
			item_image = f'https://roscarservis.ru{item["imgSrc"]}'

			stores = []

			for ps in possible_stores:
				if ps in item:
					if item[ps] is None or len(item[ps]) < 1:
						continue
					else:
						for store in item[ps]:
							store_name = store["STORE_NAME"]
							store_price = store["PRICE"]
							store_amount = store["AMOUNT"]
							total_amount += int(store["AMOUNT"])

							stores.append(
								{
									"store_name": store_name,
									"store_price": store_price,
									"store_amount": store_amount

								}
							)

			data_list.append(
				{
					"name": item_name,
					"price": item_price,
					"url": item_url,
					"img_url": item_image,
					"stores": stores,
					"total_amount": total_amount
				})
		print(f"[INFO] Обработал {page}/{pages_count}")

	cur_time = datetime.datetime.now().strftime("%d_%m_%Y_%H_%M")
	with open(f"data_{cur_time}.json", "a") as file:
		json.dump(data_list, file, indent=4, ensure_ascii=False)
	diff_time = datetime.datetime.now() - start_time
	print(diff_time)

def main():
	get_data()

if __name__ == '__main__':
		main()