import multiprocessing
import os
import threading
import time
from queue import Queue
from threading import Thread

from psutil import virtual_memory

from grandpa.utils.standard import print_warning, check_debug

class MultiprocessingManager:
    """
    Manages the multiprocessing of the framework.
    """
    def __init__(self):
        self.__task_queue = Queue()
        self.__worker_queue_list = []
        self.__init_workers()

    def add_worker_queue(self, worker_queue):
        """
        Registers a worker Queue.

        Args:
            worker_queue: Worker Queue Object to register.

        Returns:
            None
        """
        self.__worker_queue_list.append(worker_queue)

    def add_task(self, task):
        """
        Adds a task to be processed.

        Args:
            task: Task Object.

        Returns:
            None
        """
        self.__task_queue.put(task)

    def __init_workers(self):
        debug = os.environ["project_x_debug"] if "project_x_debug" in os.environ else False
        if debug == "true":
            worker_count = 1
        else:
            worker_count = multiprocessing.cpu_count() - 2
        for _ in range(worker_count):
            t = Thread(target=self.__work)
            t.daemon = True
            t.start()

    def __work(self):
        while True:
            try:
                if self.__task_queue.qsize() > 0:
                    task = self.__task_queue.get()
                    if task.lock.locked():
                        continue
                    else:
                        task.get_result()
                elif len(self.__worker_queue_list) > 0:
                    task_queue = self.__get_task_queue()
                    if task_queue is None:
                        time.sleep(0.01)
                    else:
                        task_queue.generate()
                else:
                    time.sleep(0.01)
            except Exception as e:
                if check_debug():
                    raise e
                else:
                    print_warning(str(e))

    def __get_task_queue(self):
        target_queue = None
        target_queue_size = float('inf')
        for t_queue in self.__worker_queue_list:
            if t_queue.qsize() / t_queue.target_size < target_queue_size and t_queue.qsize() < t_queue.target_size:
                target_queue_size = t_queue.qsize() / t_queue.target_size
                target_queue = t_queue
        return target_queue

class Task:
    """
    Task Class for Multiprocessing. Will execute once.
    """
    def __init__(self, multiprocessing_manager: MultiprocessingManager, target, *args, **kwargs):
        self.target = target
        self.args = args
        self.kwargs = kwargs
        self.lock = threading.Lock()
        self.result = NoData()
        multiprocessing_manager.add_task(self)

    def get_result(self):
        self.lock.acquire()
        if type(self.result) == NoData:
            self.result = self.target(*self.args, **self.kwargs)
        self.lock.release()
        return self.result

class WorkerQueue(Queue):
    """
    Class for registering a worker queue. Runs constantly until target_size is reached.
    """
    def __init__(self, multiprocessing_manager: MultiprocessingManager, target,
                 target_size: int = int(virtual_memory().total / 1000000000), split_results: bool = False,
                 batches_per_run: int = 1, *args, **kwargs):
        super().__init__()
        self.target = target
        self.args = args
        self.kwargs = kwargs
        self.target_size = target_size
        self.split_results = split_results
        self.batches_per_run = batches_per_run
        self.__pending_batches = 0
        self.static_workers = []
        self.__init_static_workers(1)
        multiprocessing_manager.add_worker_queue(self)

    def __init_static_workers(self, worker_num: int):
        debug = os.getenv("project_x_debug", False)
        if not debug == "true":
            for _ in range(worker_num):
                t = Thread(target=self.__worker_thread)
                t.daemon = True
                t.start()
                self.static_workers.append(t)

    def __worker_thread(self):
        if self.qsize() < self.target_size:
            self.generate()
        else:
            time.sleep(0.1)

    def get(self, block=True, timeout=None):
        debug = os.getenv("project_x_debug", False)
        if self.qsize() > 0:
            return super().get(block, timeout)
        else:
            try:
                self.generate()
            except Exception as e:
                if debug:
                    raise e
                else:
                    print(e)
            return self.get(block, timeout)

    def generate(self):
        self.__pending_batches += self.batches_per_run
        try:
            result = self.target(*self.args, **self.kwargs)
            if self.split_results:
                for r in result:
                    self.put(r)
            else:
                self.put(result)
        except Exception as e:
            if check_debug():
                raise e
            else:
                print_warning(str(e))
                return self.generate()
        self.__pending_batches -= self.batches_per_run

    def qsize_old(self) -> int:
        return super().qsize() + self.__pending_batches

class NoData:
    """
    Helper class. Symbolizes that a Task has not yet been completed.
    """
    pass


from PIL import Image, ImageFilter
import os

# Set maxsize of image(s)
maxsize = (100, 100)

# IMPORTS IMAGE THEN APPLIES GRAYSCALE AND SIZES IMAGE
def proccess_img():
	infile = "Chelsey1.png"
	f, e = os.path.splitext(infile)
	outfile = f + ".png"

	# If file is not the proper format then convert files
	if infile != outfile:
		try:
			Image.open(infile).save(outfile)
		except:
			raise IOError("cannot convert")

	try:
		img = Image.open(infile).convert('LA')

		# Print size before
		print(img.format, img.size, img.mode)
		img.thumbnail(maxsize, Image.ANTIALIAS)

		# Print size after
		print(img.format, img.size, img.mode)

		img.show()
		return img

	except:
		raise IOError("Unable to load image")

def pixel_intensities(img):
	#count = 0
	for pixel in img.getdata():
		print(pixel)
		#if count % 1000 == 0:
			#print("")
		#count += 1

if __name__ == '__main__':
	pixel_intensities(proccess_img())

# img.save('grayscale.png')
# img = img.filter(ImageFilter.SHARPEN)
# img = img.filter(ImageFilter.EDGE_ENHANCE)


from __future__ import print_function

import sqlite3
import numpy as np
from os.path import join
from pandas import read_csv
from perfect_match.data_access.batch_augmentation import BatchAugmentation

class DataAccess(BatchAugmentation):
    DB_FILE_NAME = "twins.db"

    TABLE_PAIRS = "pairs"

    GENDER_MALE = 0
    GENDER_FEMALE = 1

    def __init__(self, data_dir):
        self.data_dir = data_dir
        self.db = None
        self.connect()
        self.setup_schema()

    def get_split_indices(self):
        return None, None

    def connect(self):
        self.db = sqlite3.connect(join(self.data_dir, DataAccess.DB_FILE_NAME),
                                  check_same_thread=False,
                                  detect_types=sqlite3.PARSE_DECLTYPES)

        # Disable journaling.
        self.db.execute("PRAGMA journal_mode = OFF;")
        self.db.execute("PRAGMA page_size = 16384;")

    def setup_schema(self):
        self.setup_pairs()

        self.db.commit()

    def setup_pairs(self):
        columns = ""
        for idx, fields in enumerate(DataAccess.get_ordered_fields()):
            name = fields[1]
            if idx < 3:
                columns += name + "_0 INT,"
                columns += name + "_1 INT,"
            else:
                columns += name + " INT,"
        columns = columns[:-1]
        self.db.execute(("CREATE TABLE IF NOT EXISTS {table_name}"
                         "("
                         "id INT NOT NULL PRIMARY KEY, " + columns +
                         ");").format(table_name=DataAccess.TABLE_PAIRS))

    def insert_many(self, table_name, values):
        self.db.executemany("INSERT INTO {table_name} VALUES ({question_marks});"
                            .format(table_name=table_name,
                                    question_marks=",".join(["?"] * len(values[0]))),
                            values)

    def insert_clinical(self, values):
        self.insert_many(DataAccess.TABLE_PAIRS, values)

    def get_column(self, table_name, ids, column_name):
        tmp_name = "tmp_ids"
        self.create_temporary_table(tmp_name, ids)
        return_value = self.db.execute("SELECT {column_name} "
                                       "FROM {table_name} "
                                       "WHERE rowid IN (SELECT id FROM {tmp_table}) "
                                       "ORDER BY rowid;"
                                       .format(column_name=column_name,
                                               table_name=table_name,
                                               tmp_table=tmp_name)).fetchall()
        self.drop_temporary_table(tmp_name)
        return return_value

    def get_num_rows(self, table_name):
        # NOTE: This query assumes that there has never been any deletions in the time series table.
        return self.db.execute("SELECT MAX(_ROWID_) FROM {} LIMIT 1;".format(table_name)) \
            .fetchone()[0]

    def get_row(self, table_name, id, with_rowid=False):
        columns = "*"
        if with_rowid:
            columns = "rowid, " + columns

        query = "SELECT " \
                "{columns} " \
                "FROM {table_name} " \
                "WHERE rowid = ?;".format(table_name=table_name,
                                          columns=columns)
        return self.db.execute(query, (id,)).fetchone()

    def get_rows(self, train_ids):
        tmp_name = "tmp_pairs"
        self.create_temporary_table(tmp_name, map(lambda x: (x,), train_ids))

        pairs = self.db.execute("SELECT * "
                                "FROM {table_pairs} "
                                "WHERE rowid IN (SELECT id FROM {tmp_table});"
                                .format(table_pairs=DataAccess.TABLE_PAIRS,
                                        tmp_table=tmp_name)).fetchall()

        self.drop_temporary_table(tmp_name)

        input_data = np.array(pairs)
        ids, pair_data = input_data[:, 0], input_data[:, 7:]
        return input_data, ids, pair_data

    def get_row_by_id(self, table_name, id, with_rowid=False):
        columns = "*"
        if with_rowid:
            columns = "rowid, " + columns

        query = "SELECT " \
                "{columns} " \
                "FROM {table_name} " \
                "WHERE id = ?;".format(table_name=table_name,
                                       columns=columns)
        return self.db.execute(query, (id,)).fetchone()

    def get_rows_by_clinical_id(self, table_name, id):
        columns = "*"

        query = "SELECT " \
                "{columns} " \
                "FROM {table_name} " \
                "WHERE clinical_id = ?;".format(table_name=table_name,
                                                columns=columns)
        return self.db.execute(query, (id,)).fetchone()

    def get_labelled_patients(self):
        return np.arange(self.get_num_rows(DataAccess.TABLE_PAIRS)) + 1

    def create_temporary_table(self, table_name, values):
        self.db.execute("CREATE TEMP TABLE {table_name} (id INT);".format(table_name=table_name))
        if len(values) != 0:
            self.db.executemany("INSERT INTO {table_name} VALUES (?);".format(table_name=table_name), values)
        return table_name

    def drop_temporary_table(self, table_name):
        self.db.execute("drop table {tmp_table_name};".format(tmp_table_name=table_name))

    def get_pairs_dimension(self):
        pair = self.db.execute("SELECT * FROM {table_name} WHERE rowid = 1;"
                               .format(table_name=DataAccess.TABLE_PAIRS)).fetchone()
        return len(pair) - 7

    @staticmethod
    def get_ordered_fields(clean=False):
        if clean:
            factor_list = [8, 9]
            adequacy_missing = 4
        else:
            factor_list = 9
            adequacy_missing = None
        identity = lambda x: x
        minus_one = lambda x: x-1
        convert_gender = lambda x: DataAccess.GENDER_MALE if x == 1 else DataAccess.GENDER_FEMALE
        factor_fun = lambda x: 1 if x == 1 else 0
        divide_by = lambda val: lambda x: float(x) / float(val)

        return [
            ("DBIRWT", "birth_weight", None, identity, identity),
            ("CSEX", "child_sex", None, convert_gender, identity),
            ("AGED", "days_age_at_death", None, lambda x: 0 if np.isnan(x) else 1, identity),
            ("DTOTORD", "number_previous_births", 99, identity, divide_by(10)),
            ("DMAR", "marital_status", None, minus_one, identity),
            ("DMAGE", "mother_age", None, minus_one, divide_by(50)),
            ("DMEDUC", "mother_education", 99, minus_one, divide_by(17)),
            #("MRACE", "mother_race", None, minus_one),
            #("FRACE", "father_race", 99, minus_one),
            ("PLDEL", "place_of_delivery", 9, minus_one, identity),
            ("RESSTATB", "residence_state", None, minus_one, identity),
            # ("BRSTATE_REG", "residence_state_region", None),
            # ("DLIVORD", "number_previous_births_live", 99),
            ("GESTAT", "number_gestation_weeks", 99, identity, divide_by(47)),
            #("GESTAT10", "number_gestation_weeks_coded", 10),
            ("ADEQUACY", "adequacy_of_care", adequacy_missing, minus_one, identity),
            ("MPCB", "month_of_pregnancy_care_began", 99, minus_one, divide_by(9)),
            ("NPREVIST", "number_prenatal_visits", 99, identity, divide_by(49)),
            # ("DISLLB", "interval_since_last_live_birth", 999),
            ("ANEMIA", "anemia", factor_list, factor_fun, identity),
            ("CARDIAC", "cardiac", factor_list, factor_fun, identity),
            ("LUNG", "lung", factor_list, factor_fun, identity),
            ("DIABETES", "diabetes", factor_list, factor_fun, identity),
            ("HERPES", "herpes", factor_list, factor_fun, identity),
            ("HYDRA", "hydra", factor_list, factor_fun, identity),
            ("HEMO", "Hemoqlobinopathy", factor_list, factor_fun, identity),
            ("CHYPER", "hypertension_chronic", factor_list, factor_fun, identity),
            ("PHYPER", "hypertension_pregnancy", factor_list, factor_fun, identity),
            ("ECLAMP", "eclampsia", factor_list, factor_fun, identity),
            ("INCERVIX", "incompetent_cervix", factor_list, factor_fun, identity),
            ("PRE4000", "previous_infant_less_than_4000", factor_list, factor_fun, identity),
            ("PRETERM", "previous_preterm", factor_list, factor_fun, identity),
            ("RENAL", "renal", factor_list, factor_fun, identity),
            ("RH", "rh_sensitisation", factor_list, factor_fun, identity),
            ("UTERINE", "uterine_bleeding", factor_list, factor_fun, identity),
            ("OTHERMR", "other", factor_list, factor_fun, identity),
            # ("TOBACCO", "tobacco_use", 9),
            ("CIGAR", "num_cigarettes_per_day", 99, identity, divide_by(98)),  # < 98
            # ("ALCOHOL", "alcohol_use", 9),
            ("DRINK", "number_of_drinks", 99, identity, divide_by(98)),  # < 98
            ("WTGAIN", "num_pounds_gained", 99, identity, divide_by(98))  # < 98
        ]

    @staticmethod
    def get_pairs_data(filepath):
        pairs_data = read_csv(filepath)
        return pairs_data

    def get_labels(self, args, ids, benchmark):
        assignments = []
        for id in ids:
            pair = self.get_row(DataAccess.TABLE_PAIRS, id[0])
            assignment = benchmark.get_assignment(id, pair)[0]
            assignments.append(assignment)

        assignments = [t - 2 if t >= 2 else t for t in assignments]

        # get assignments from benchmark first - then select the correct "child_sex"
        sex_0 = np.squeeze(self.get_column(DataAccess.TABLE_PAIRS, ids, "child_sex_0"), axis=-1)
        sex_1 = np.squeeze(self.get_column(DataAccess.TABLE_PAIRS, ids, "child_sex_1"), axis=-1)
        dead_0 = np.squeeze(self.get_column(DataAccess.TABLE_PAIRS, ids, "days_age_at_death_0"), axis=-1)
        dead_1 = np.squeeze(self.get_column(DataAccess.TABLE_PAIRS, ids, "days_age_at_death_1"), axis=-1)
        sex = np.squeeze([np.stack([sex_0, sex_1]).T[idx, t] for idx, t in enumerate(assignments)])
        dead = np.squeeze([np.stack([dead_0, dead_1]).T[idx, t] for idx, t in enumerate(assignments)])

        num_labels = 2**2 - 1
        return sex * 2 ** 0 + dead * 2 ** 1, num_labels

    def get_entry_with_id(self, id, args):
        pair = self.get_row(DataAccess.TABLE_PAIRS, id)

        patient_id = pair[0]
        result = {"pair": pair}

        return patient_id, result

    def standardise_entry(self, entry):
        standardisers = map(lambda x: x[4], self.get_ordered_fields()[3:])
        for i in range(len(entry)):
            entry[i] = standardisers[i](entry[i])
        return entry

    def prepare_batch(self, args, batch_data, benchmark, is_train=False):
        pair_data = np.array(map(lambda x: x["pair"], batch_data))
        ids, input_data = pair_data[:, 0], pair_data[:, 7:]

        assignments = map(benchmark.get_assignment, ids, pair_data)
        treatment_data, batch_y = zip(*assignments)
        treatment_data = np.array(treatment_data)

        if args["with_propensity_batch"] and is_train:
            propensity_batch_probability = float(args["propensity_batch_probability"])
            num_randomised_neighbours = int(np.rint(args["num_randomised_neighbours"]))
            input_data, treatment_data, batch_y = self.enhance_batch_with_propensity_matches(benchmark,
                                                                                treatment_data,
                                                                                input_data,
                                                                                batch_y,
                                                                                propensity_batch_probability,
                                                                                num_randomised_neighbours)

        input_data = input_data.astype(np.float32)
        input_data = np.array(map(self.standardise_entry, input_data))

        batch_y = np.array(batch_y)
        batch_x = [
            input_data,
            treatment_data,
        ]
        return batch_x, batch_y


# -*- coding: utf-8 -*-
"""
Abstract Dataset Loader
=======================

Defines the abstract class implementing support for dataset loading:

-   :class:`colour_datasets.loaders.AbstractDatasetLoader`
"""

from __future__ import division, unicode_literals

from abc import ABCMeta, abstractmethod
from six import add_metaclass

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2019 - Colour Developers'
__license__ = 'New BSD License - https://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = ['AbstractDatasetLoader']

@add_metaclass(ABCMeta)
class AbstractDatasetLoader:
    """
    Defines the base class for a dataset loader.

    This is an :class:`ABCMeta` abstract class that must be inherited by
    sub-classes.

    The sub-classes are expected to implement the
    :meth:`colour_datasets.loaders.AbstractDatasetLoader.load` method that
    handles the syncing, parsing, conversion and return of the dataset content
    as a *Python* object.

    Attributes
    ----------
    ID
    record
    id
    content

    Methods
    -------
    load
    sync

    Parameters
    ----------
    record : Record
        Dataset record.
    """

    ID = None
    """
    Dataset record id, i.e. the *Zenodo* record number.

    ID : unicode
    """

    def __init__(self, record):
        self._record = record
        self._content = None

    @property
    def record(self):
        """
        Getter and setter property for the dataset record.

        Parameters
        ----------
        value : Record
            Value to set the dataset record with.

        Returns
        -------
        unicode
            Dataset record.
        """

        return self._record

    @property
    def id(self):
        """
        Getter and setter property for the dataset id.

        Parameters
        ----------
        value : unicode
            Value to set the dataset id with.

        Returns
        -------
        unicode
            Dataset id.
        """

        return self.__class__.ID

    @property
    def content(self):
        """
        Getter and setter property for the dataset content.

        Parameters
        ----------
        value : object
            Value to set the dataset content with.

        Returns
        -------
        unicode
           Dataset content.
        """

        return self._content

    @abstractmethod
    def load(self):
        """
        Syncs, parses, converts and returns the dataset content as a *Python*
        object.

        Returns
        -------
        object
            Dataset content as a *Python* object.

        Notes
        -----
        -   Sub-classes are required to call
            :meth:`colour_datasets.loaders.AbstractDatasetLoader.sync` method
            when they implement it, e.g.
            ``super(MyDatasetLoader, self).sync()``.
        """

        pass

    def sync(self):
        """
        Syncs the dataset content, i.e. checks whether it is synced and pulls
        it if required.
        """

        if not self.record.synced():
            self.record.pull()


#coding=utf-8
# File      :   cut_wav.py
# Time      :   2021/12/15 14:39:17
# Author    :   Jinghan Peng
# Desciption:   切割无效音频为小的chunk

import os, sys
from re import L
import wave
from pydub import AudioSegment
from pydub.utils import make_chunks
from tqdm import tqdm

from multiprocessing import Pool
import collections

def main():
    wav_dir = "/data1/pengjinghan/tsvad/invalid_sound/sum"

    out_wav_dir  = "/data1/pengjinghan/tsvad/invalid_sound/wav_chunk/wav"
    out_data_dir = "/data1/pengjinghan/tsvad/invalid_sound/wav_chunk/data"
    os.makedirs(out_wav_dir, exist_ok=True)
    os.makedirs(out_data_dir, exist_ok=True)

    wav_length = 16.02 # 单位: 秒，=16020秒
    chunk_length_ms    = int(wav_length*1000)
    chunk_length_frame = int(wav_length*100)

    """对音频进行切割chunk"""
    with open(os.path.join(out_data_dir, 'target_chunk.lst'), 'w') as wf:
        for file in os.listdir(wav_dir):
            wav_path = os.path.join(wav_dir, file) # 输入音频路径
            utt      = file.split(".wav")[0]
            audio    = AudioSegment.from_file(wav_path)    # 输入音频
            duration = audio.duration_seconds
            print(f"Processing {file}")
            print(f"{duration}s audio is going to be cut to {int(duration//wav_length)} chunk with chunk_size={wav_length}s")
            chunks   = make_chunks(audio, chunk_length_ms) # 切片

            """"对每个chunk"""
            for index, chunk in tqdm(enumerate(chunks)):
                if chunk.duration_seconds != wav_length: # 最后一个块可能长度不够而丢弃
                    continue

                """保存chunk音频"""
                chunk_name = utt+"-{0:08d}_{1:08d}.wav".format(index*chunk_length_ms, (index+1)*chunk_length_ms)
                chunk_path = os.path.join(out_wav_dir, chunk_name)
                chunk.export(chunk_path, format="wav") # 保存剪切后的样本音频
                print("Exporting", chunk_path)

                """记录chunk的target"""
                chunk_utt = chunk_name.split('.wav')[0]
                target_chunk_length_frame = chunk_length_frame - 2 # 特征要比音频时长小2秒
                target_line = ' '.join(['0' for i in range(target_chunk_length_frame)])

                wf.write(f"{chunk_utt} {target_line}\n")

if __name__ == "__main__":
    main()


import cv2
import numpy as np
import easyocr
import PIL.Image as pil_img

def Book_Image(Snap):

    pic = pil_img.open(Snap)
    img = np.array(pic).astype("uint8")
    picture = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    image = cv2.rotate(picture, cv2.cv2.ROTATE_90_COUNTERCLOCKWISE)
    reader = easyocr.Reader(['en'])
    result = reader.readtext(image, detail=0)

    Words = [re for re in result]
    text = " ".join(Words)
    return text


import yaml
import pandas as pd
import numpy as np

def build_cpu_and_mem_distribution_dataframes(trace_df : pd.DataFrame,
                            col_flavor_cpu : str = 'vmcorecount', col_flavor_mem : str = 'vmmemory',
                            col_vm_created : str = 'vmcreated', col_vm_deleted : str = 'vmdeleted',
                            timestamp_begin : int = None, timestamp_end : int = None, timestamp_step : int = 3600 ):

    if timestamp_begin is None: timestamp_begin = trace_df[col_vm_created].min()
    if timestamp_end is None: timestamp_end = trace_df[col_vm_created].max()

    keys_core, core_values_per_key = __init_values_per_key(trace_df, col_flavor_cpu)
    keys_mem, mem_values_per_key = __init_values_per_key(trace_df, col_flavor_mem)

    considered_timestamps = list()
    for timestamp in range(timestamp_begin, timestamp_end, timestamp_step):

        # Dataset containing all VM alive in current range (deleted after min, created before max)
        filtered_by_aliveness = trace_df.loc[(trace_df[col_vm_deleted] >= (timestamp)) & (trace_df[col_vm_created] <= (timestamp+timestamp_step) )]

        __add_to_result_dict_observed_freq(alive_df=filtered_by_aliveness, metric=col_flavor_cpu, metric_keys=keys_core, result_dict=core_values_per_key)
        __add_to_result_dict_observed_freq(alive_df=filtered_by_aliveness, metric=col_flavor_mem, metric_keys=keys_mem, result_dict=mem_values_per_key)
        considered_timestamps.append(timestamp)

    core_values_per_key['timestamp'] = considered_timestamps
    mem_values_per_key['timestamp'] = considered_timestamps

    return pd.DataFrame(core_values_per_key), pd.DataFrame(mem_values_per_key)

def get_cpu_and_mem_average_distribution(trace_df : pd.DataFrame,
                            col_flavor_cpu : str = 'vmcorecount', col_flavor_mem : str = 'vmmemory',
                            col_vm_created : str = 'vmcreated', col_vm_deleted : str = 'vmdeleted',
                            timestamp_begin : int = None, timestamp_end : int = None, timestamp_step : int = 3600 ):

    cpu_timestamped_df, mem_timestamped_df = build_cpu_and_mem_distribution_dataframes(trace_df=trace_df,
                                        col_flavor_cpu=col_flavor_cpu, col_flavor_mem=col_flavor_mem,
                                        col_vm_created=col_vm_created, col_vm_deleted=col_vm_deleted,
                                        timestamp_begin=timestamp_begin, timestamp_end=timestamp_end, timestamp_step=timestamp_step)

    cpu_grouped = cpu_timestamped_df.drop('timestamp', axis=1)
    cpu_grouped = cpu_grouped.mean().to_frame()

    cpu_keys = cpu_grouped.index.values
    cpu_vals = cpu_grouped.values.tolist()
    res_cpu = pd.DataFrame({col_flavor_cpu: [int(key) for key in cpu_keys], 'freq': [round(np.mean(value),2) for value in cpu_vals]})

    mem_grouped = mem_timestamped_df.drop('timestamp', axis=1)
    mem_grouped = mem_grouped.mean().to_frame()

    mem_keys = mem_grouped.index.values
    mem_vals = mem_grouped.values.tolist()
    res_mem = pd.DataFrame({col_flavor_mem: [float(key) for key in mem_keys], 'freq': [round(np.mean(value),2) for value in mem_vals]})

    return res_cpu.loc[res_cpu['freq'] > 0.01].reset_index(drop=True), res_mem.loc[res_mem['freq'] > 0.01].reset_index(drop=True)

def __init_values_per_key(trace_df : pd.DataFrame, column_name : str):
    keys = list(trace_df[column_name].unique())
    keys.sort()
    values_per_keys = dict()
    for key in keys: values_per_keys[str(key)] = list()
    return keys, values_per_keys

def __add_to_result_dict_observed_freq(alive_df : pd.DataFrame, metric : str, metric_keys : list, result_dict : dict):

    distribution_df = pd.DataFrame(alive_df.groupby(metric).size().rename('count')).reset_index()
    total = distribution_df['count'].sum()
    distribution_df['freq'] = round((distribution_df['count']/total),2)

    observed_metric = list(distribution_df[metric] )
    for key in metric_keys:
        if key not in observed_metric:
            result_dict[str(key)].append(0)
        else:
            result_dict[str(key)].append(float(distribution_df.loc[distribution_df[metric] == key]['freq'].iloc[0]))

def convert_distribution_to_scenario(cpu_distribution : pd.DataFrame,  mem_distribution : pd.DataFrame,
                                     col_freq_cpu : str = 'freq', col_flavor_cpu = 'vmcorecount',
                                     col_freq_mem : str = 'freq', col_flavor_mem = 'vmmemory',
                                     output_file : str = 'scenario-vm-distribution.yml'
                                    ):

    ordered_cpu_distribution = cpu_distribution.sort_values(by=[col_flavor_cpu]).set_index(col_flavor_cpu)
    ordered_mem_distribution = mem_distribution.sort_values(by=[col_flavor_mem]).set_index(col_flavor_mem)

    yml_dict = dict()
    yml_dict["vm_distribution"] = dict()
    yml_dict["vm_distribution"]["config_cpu"] = ordered_cpu_distribution.apply(
        lambda row : float(row[col_freq_cpu]), axis=1).to_dict()

    yml_dict["vm_distribution"]["config_mem"] = ordered_mem_distribution.apply(
        lambda row :float(row[col_freq_mem]), axis=1).to_dict()

    with open(output_file, 'w') as outfile:
        yaml.dump(yml_dict, outfile, default_flow_style=False)


import sqlite3

fname = input('Enter file name: ')
if(len(fname) < 1):
    fname = 'mbox.txt'

fh = open(fname)
emailList = list()
dic = dict()

for line in fh:
    if( line.startswith('From: ') ):
        pieces = line.split()
        email = pieces[1]

        secPc =  email.split('@')
        orgN = secPc[1]

        emailList.append(orgN)
    else:
        continue

# print(emailList)

#count email sending
for email in emailList:
    dic[email] = dic.get(email, 0) + 1

# print(dic)

conn = sqlite3.connect('assignmentdb.sqlite')
cur = conn.cursor()

cur.execute('''
DROP TABLE IF EXISTS Counts''')

cur.execute('''
    CREATE TABLE Counts (org TEXT, count INTEGER)
''')

## email insert
for email, count in dic.items():
    # print(email, count)
    cur.execute('''INSERT into Counts (org, count) values (?, ?)''', (email, count))
    conn.commit()

sqlstr = 'SELECT org, count FROM Counts ORDER BY count DESC'

print(cur.execute(sqlstr))
for row in cur.execute(sqlstr):
    print(str(row[0]), row[1])

#     email = pieces[1]
#     cur.execute('SELECT count from Counts where email = ?', (email,))

#     row = cur.fetchone()  # grab the first one
#     if row is None:
#         cur.execute('''INSERT into Counts (email, count)
#         values (?, 1)''', (email,))
#     else:
#         cur.execute(
#             'update Counts set count = count + 1 where email = ?', (email,))
#     conn.commit()

# sqlstr = 'SELECT email, count FROM Counts ORDER BY count DESC LIMIT 10'

# for row in cur.execute(sqlstr):
#     print(str(row[0]), row[1])


# -*- coding: utf-8 -*-
"""
Created on Thu Jul 13 23:46:18 2017

@author: Yangzj
"""

import requests
import re
import os
import time
#page = 1
i=0

user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
headers = { 'User-Agent' : user_agent }

def getHTML(url):
    try:
        r = requests.get(url, timeout = 30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "请求网站链接异常"

if __name__  == "__main__":

    t=str(time.strftime('%Y%m%d%H%M%S',time.localtime(time.time())))
    imgPath="C:\\img\\" + t
    print(imgPath)
    os.mkdir(imgPath)

    if not os.path.isdir(imgPath):
        print('sb')
        os.mkdir(imgPath)

    index=1

    url = 'http://www.141545.net/rt-13850-1-1.html'
    html = getHTML(url)

    pattern = re.compile('file="(.*?.jpg)"',re.S)
    imgurls = re.findall(pattern, html)

    for imgurl in imgurls:
       try:
           res = requests.get(imgurl,headers = headers)
       except:
           print('未下载成功：',imgurl)
           continue
       print(imgurl)
       filename = os.path.join(imgPath, str(index)+'.jpg')
       with open(filename, 'ab+') as f:
           f.write(res.content)
           print('下载完成\n')
           index += 1


# @Time : 2022-09-20 21:09
# @Author : Phalange
# @File : 128. 最长连续序列.py
# @Software: PyCharm
# C'est la vie,enjoy it! :D
from typing import List

class Solution:

    def __init__(self):
        self.nodes = dict()
        self.parents = dict()
        self.sizeMap = dict()

    def longestConsecutive(self, nums: List[int]) -> int:
        if nums == []:
            return 0
        n = len(nums)
        for i in range(n):
            self.nodes[nums[i]] = nums[i]
            self.parents[nums[i]] = nums[i]
            self.sizeMap[nums[i]] = 1

        # 连通起来
        for i in range(n):
            if nums[i]-1 in self.nodes:
                self.union(nums[i]-1,nums[i])

        return max(list(self.sizeMap.values()))

    def isSameSet(self,a,b):
        if a not in self.nodes or b not in self.nodes:
            return
        return self.findFather(a) == self.findFather(b)

    def findFather(self,cur):
        path = []
        while cur !=self.parents[cur]:
            path.append(cur)
            cur = self.parents[cur]

        while path:
            self.parents[path.pop()] = cur
        return cur

    def union(self,a,b):
        if a not in self.nodes or b not in self.nodes:
            return

        aHead = self.findFather(a)
        bHead = self.findFather(b)
        if aHead != bHead:
            aSize = self.sizeMap[aHead]
            bSize = self.sizeMap[bHead]
            big = aHead if aSize >=bSize else bHead
            small = bHead if aHead == big else aHead
            self.parents[small] = big
            self.sizeMap[big] = aSize + bSize
            self.sizeMap.pop(small)

print(Solution().longestConsecutive([1,3,2]))

哈希表/128. 最长连续序列.py


"""Deep Residual Network with Stochastic Depth."""
import cPickle as pickle
import datetime as dt
import gzip
import lasagne as nn
import numpy as np
import sys
import theano
import theano.tensor as T
import time

from Deep_Residual_Learning_CIFAR10 import load_data
from Deep_Residual_Learning_CIFAR10 import iterate_minibatches
from helpers import report
from helpers import ResBlockLayer

# Basic settings
sys.setrecursionlimit(2 ** 20)

TITLE = 'stochastic_depth'
OUTPUT_FILE = './output/{}.txt'.format(TITLE)

if __name__ == '__main__':
    report('\n\nSTART: {}'.format(dt.datetime.now()), OUTPUT_FILE)
    report('>>> {}'.format(OUTPUT_FILE), OUTPUT_FILE)

    # Model parameters
    try:
        N = int(sys.argv[1])
    except:
        N = 9 # default depth uses N = 9.

    epochs, batchsize = 500, 128
    report('Depth: N = {}, #Layers = {}.'.format(N, 6*N+2), OUTPUT_FILE)
    delta = 0.5 / (3 * N - 1)
    decay = [1.0 - i * delta for i in xrange(3*N)] # linear decay rule
    switches = [theano.shared(nn.utils.floatX(1.)) for i in xrange(3*N)]
    on, off = 1.0, 0.0 # nn.utils.floatX(1.), nn.utils.floatX(0.)

    # Use to turn on all blocks (during testing)
    def full_depth():
        global switches, on
        for c in switches:
            c.set_value(on)

    # Use to randomly on/off blocks based on decay rule (during training)
    def stochastic_depth():
        global switches, on, off, decay
        assert len(switches) == len(decay)
        drop = np.random.uniform(low=0, high=1, size=len(switches)) > decay
        for c, d in zip(switches, drop):
            c.set_value(off) if d else c.set_value(on)

    # Symbolic variables
    X, Y = T.tensor4('inputs'), T.ivector('targets')

    # Model architecture
    net = nn.layers.InputLayer(shape=(None, 3, 32, 32), input_var=X)

    # 16 x 32 x 32
    net = nn.layers.Conv2DLayer(
        incoming=net, num_filters=16, filter_size=(3,3), stride=(1,1),
        nonlinearity=nn.nonlinearities.rectify, pad='same',
        W=nn.init.HeNormal(gain='relu'), flip_filters=False)
    net = nn.layers.batch_norm(net)
    i = 0  # add i-th switch to i-th block

    for _ in range(N):
        net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=False)
        i += 1

    # 32 x 16 x 16
    net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=True)
    i += 1

    for _ in range(N-1):
        net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=False)
        i += 1

    # 64 x 8 x 8
    net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=True)
    i += 1

    for _ in range(N-1):
        net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=False)
        i += 1

    net = nn.layers.GlobalPoolLayer(net)
    net = nn.layers.DenseLayer(
        incoming=net, num_units=10, W=nn.init.HeNormal(),
        nonlinearity=nn.nonlinearities.softmax)
    report('Model OK...', OUTPUT_FILE)

    num_params = nn.layers.count_params(net, trainable=True)
    report("Number of parameters: {}".format(num_params), OUTPUT_FILE)

    # Training function
    output = nn.layers.get_output(net, deterministic=False)
    loss = nn.objectives.categorical_crossentropy(output, Y).mean()
    accuracy = T.mean(
        T.eq(T.argmax(output, axis=1), Y), dtype=theano.config.floatX)
    all_layers = nn.layers.get_all_layers(net)
    regularization = nn.regularization.regularize_layer_params(
        layer=all_layers, penalty=nn.regularization.l2)
    loss = loss + 1e-4 * regularization
    params = nn.layers.get_all_params(net, trainable=True)
    learning_rate = theano.shared(nn.utils.floatX(0.1))
    updates = nn.updates.momentum(
        loss_or_grads=loss, params=params,
        learning_rate=learning_rate, momentum=0.9)
    training_function = theano.function([X,Y],[loss,accuracy],updates=updates)
    report('Training function OK...', OUTPUT_FILE)

    # Test/validation function
    test_output = nn.layers.get_output(net, deterministic=True)
    test_loss = nn.objectives.categorical_crossentropy(test_output, Y).mean()
    test_accuracy = T.mean(
        T.eq(T.argmax(test_output, axis=1), Y), dtype=theano.config.floatX)
    test_function = theano.function([X,Y], [test_loss,test_accuracy])
    report('Validation function OK...', OUTPUT_FILE)

    # Load data
    data = load_data()
    X_train, Y_train = data['X_train'], data['Y_train']
    X_test, Y_test = data['X_test'], data['Y_test']
    report('Data OK...', OUTPUT_FILE)

    # Start training
    # `training loss, training accuracy, validation loss, validation accuracy`
    TL, TA, VL, VA = [], [], [], []
    report('Starting training...', OUTPUT_FILE)
    header = ['Epoch', 'TL', 'TA', 'VL', 'VA', 'Time']
    report('{:<10}{:<20}{:<20}{:<20}{:<20}{:<20}'.format(*header), OUTPUT_FILE)
    learning_rate_schedule = {250:0.01, 375:0.001}

    for e in xrange(epochs):
        if (e + 1) in learning_rate_schedule:
            learning_rate.set_value(learning_rate_schedule[e + 1])

        start_time = time.time()
        t_batches, v_batches = 0, 0
        tl, ta, vl, va = 0., 0., 0., 0.

        # Train with stochastic depth
        stochastic_depth()
        minibatches = iterate_minibatches(X_train, Y_train, batchsize,
            shuffle=True, augment=True)

        for data, target in minibatches:
            l, a = training_function(data, target)
            tl += l
            ta += a
            t_batches += 1

        tl /= t_batches
        ta /= t_batches
        TL.append(tl)
        TA.append(ta)

        # Test with full depth
        full_depth()
        minibatches = iterate_minibatches(X_test, Y_test, 500, shuffle=False,
            augment=False)

        for data, target in minibatches:
            l, a = test_function(data, target)
            vl += l
            va += a
            v_batches += 1

        vl /= v_batches
        va /= v_batches
        VL.append(vl)
        VA.append(va)

        row = [e + 1, tl, ta, vl, va, time.time() - start_time]
        report('{:<10}{:<20}{:<20}{:<20}{:<20}{:<20}'.format(*row),OUTPUT_FILE)

    report('Finished training...', OUTPUT_FILE)

    # Save training information
    f = gzip.open('./output/{}_info.pkl.gz'.format(TITLE), 'wb')
    info = {
        'training loss': TL,
        'training accuracy': TA,
        'validation loss': VL,
        'validation accuracy': VA
    }
    pickle.dump(info, f)
    f.close()
    report('Saved training information...', OUTPUT_FILE)

    # Save weights
    weights = nn.layers.get_all_params(net)
    weights = [np.array(w.get_value()) for w in weights]
    f = gzip.open('./output/{}_weights.pkl.gz'.format(TITLE), 'wb')
    pickle.dump(weights, f)
    f.close()

    report('Saved weights...', OUTPUT_FILE)
    report('END: {}'.format(dt.datetime.now()), OUTPUT_FILE)


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import plotly.graph_objs as go
import pandas as pd

df = pd.read_csv('./data/Stapleton.csv')

app = dash.Dash()

# https://dash.plot.ly/dash-core-components/dropdown
# We need to construct a dictionary of dropdown values for the years
year_options = []
for year in df['YEAR'].unique():
    year_options.append({'label':str(year),'value':year})

app.layout = html.Div([
    html.H3('Denver Max Daily Temp'),
    dcc.Graph(id='graph'),
    dcc.Dropdown(id='year-picker',options=year_options,value=df['YEAR'].min())
])

@app.callback(Output('graph', 'figure'),
              [Input('year-picker', 'value')])
def update_figure(selected_year):
    filtered_df = df[df['YEAR'] == selected_year]
    traces = []
    for month_num in filtered_df['MONTH'].unique():
        df_by_month = filtered_df[filtered_df['MONTH'] == month_num]
        traces.append(go.Scatter(
            x=df_by_month['DAY'],
            y=df_by_month['TMAX'],
            text=df_by_month['NAME'],
            mode='markers + lines',
            opacity=0.7,
            marker={'size': 5},
            name=month_num
        ))

    return {
        'data': traces,
        'layout': go.Layout(
            xaxis={'title': 'TIME'},
            yaxis={'title': 'TMAX'},
            hovermode='closest'
        )
    }

if __name__ == '__main__':
    app.run_server()


from django.shortcuts import render
from django.http import JsonResponse
from firstApp.models import Employee
from rest_framework.permissions import IsAuthenticated

# Create your views here.
def employeeView(request):
    emp = {
    'id':123,
    'name':'Pavan',
    'sal': 100000
    }

    data = Employee.objects.all();
    response = {'employees':list(data.values('name','sal'))}

    return JsonResponse(response)

    permission_classes = (IsAuthenticated,)


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 31 17:35:22 2022

@author: Gabriel
"""

from osgeo import gdal
import numpy as np
import cv2
import tkinter as tk
from tkinter import filedialog
import os

root = tk.Tk()
img_new = filedialog.askopenfilename(filetypes=(("tif files","*.tif"),("All files","*.*")))
img_old = filedialog.askopenfilename(filetypes=(("tif files","*.tif"),("All files","*.*")))
root.destroy()

saida = 'D:/TCC/IMAGENS/Classificadas/Reprocessada/Change_' + os.path.basename(img_old)

img_new = gdal.Open(img_new,gdal.GA_ReadOnly)
img_old = gdal.Open(img_old,gdal.GA_ReadOnly)

l,c = img_new.RasterYSize,img_new.RasterXSize

array_new = img_new.ReadAsArray()

array_old = img_old.ReadAsArray()

geo_transform = img_new.GetGeoTransform()
projection = img_new.GetProjectionRef()

#re_cls = np.zeros([l,c])

for i in range(0,l):
    for j in range(0,c):
        if array_new[i,j] == 2 and array_old[i,j] != 2:
            array_old[i,j] = 2
        if array_new[i,j] == 3 and array_old[i,j] != 3:
            array_old[i,j] = 3

array_old = array_old.astype('uint8')
kernel = np.ones([3,3])
array_old = cv2.morphologyEx(array_old, cv2.MORPH_CLOSE, kernel)

driver = gdal.GetDriverByName('GTiff')
rows, cols = array_old.shape
rasterDS = driver.Create(saida, cols, rows, 1, gdal.GDT_Int32)
rasterDS.SetProjection(projection)
rasterDS.SetGeoTransform(geo_transform)
band = rasterDS.GetRasterBand(1)
band.WriteArray(array_old)
rasterDS = None


# * ---------------- *
#
#   ** Deep Reinforcement Learning Nano Degree **
#   project: Navigation
#   author:  Matthias Schinacher
#
#   the script implements Q-learning with transitions- replay
#   optionally using priority replay
#
#   the model used for the Q- function approximation is a simple
#   neural network with 3 hidden layers and 2 'relu' activations in between;
#   note: there is no activation/ relu after the third linear function!
#
#   => thus the model has 2 adjustable size- parameters, as the number of
#      incoming values is determined by the sate- size (37) and the number
#      of outgoing values ("activations") is the number of actions (4)
#   => interpretation of the 4 resulting values is simply the Q- values
#      for the 4 actions (of the input-state given)
# * ---------------- *

# * ---------------- *
#    importing the packages we need
# * ---------------- *
import os.path
import sys
import copy
import re
import configparser
import pickle
from unityagents import UnityEnvironment
import numpy as np
import torch
import torch.nn.functional as fct

# * ---------------- *
#   command line arguments:
#    we expect exactly 2, the actual script name and the command-file-name
# * ---------------- *
if len(sys.argv) != 2:
    print('usage:')
    print('   python {} command-file-name'.format(sys.argv[0]))
    quit()

if not os.path.isfile(sys.argv[1]):
    print('usage:')
    print('   python {} command-file-name'.format(sys.argv[0]))
    print('[error] "{}" file not found or not a file!'.format(sys.argv[1]))
    quit()

# * ---------------- *
#   constants:
#    this code is only for the Banana- scenario, no generalization (yet)
# * ---------------- *
STATE_SIZE = int(37)
ACTION_SIZE = int(4)
fzero = float(0)
fone = float(1)

# * ---------------- *
#   the command-file uses the ConfigParser module, thus must be structured that way
#    => loading the config and setting the respective script values
# * ---------------- *
booleanpattern = re.compile('^\\s*(true|yes|1|on)\\s*$', re.IGNORECASE)

config = configparser.ConfigParser()
config.read(sys.argv[1])

# start the logfile
rlfn = 'run.log' # run-log-file-name
if 'global' in config and 'runlog' in config['global']:
    rlfn = config['global']['runlog']
print('!! using logfile "{}"\n'.format(rlfn))
rl = open(rlfn,'w')
rl.write('# ## configuration from "{}"\n'.format(sys.argv[1]))

if 'rand' in config and 'seed' in config['rand']:
    seed = int(config['rand']['seed'])
    rl.write('# [debug] using random seed: {}\n'.format(seed))
    np.random.seed(seed)
    torch.manual_seed(seed)

TRAIN = True   # default to training mode
if 'mode' in config and 'train' in config['mode']:
    train = config['mode']['train']
    TRAIN = True if booleanpattern.match(train) else False
    rl.write('# [debug] using mode.train: {} from "{}"\n'.format(TRAIN,train))
SHOW  = not TRAIN  # default for "show"- mode
if 'mode' in config and 'show' in config['mode']:
    show = config['mode']['show']
    SHOW = True if booleanpattern.match(show) else False
    rl.write('# [debug] using mode.show: {} from "{}"\n'.format(SHOW,show))

# * ---------------- *
#   hyper- parameters
# * ---------------- *
# defaults
EPISODES = int(1000)         # number of episodes (including warm-up)
WARMUP_EPISODES = int(10)    # number of warm-up episodes (training only)
EPSILON_EPISODES = int(100)  # number of episodes, over which we decrease epsilon (training only)
EPSILON_START = float(1)     # start-value for epsilon (training and show- mode!)
EPSILON_END = float(0.01)    # final value for epsilon (training only)
REPLAY_BUFFERSIZE = int(50000) # replay buffer/memory- size (training only)
REPLAY_BATCHSIZE = int(300)  # batch size for replay (training only)
REPLAY_STEPS = int(1)        # replay transisitions- batch each x steps (training only)
PRIO_REPLAY = False          # priority replay flag (training only)
Q_RESET_STEPS = int(50)      # steps, after which to reset Q_ to Q  (training only)
GAMMA = float(0.99)          # gamma- parameter (training only)
LEARNING_RATE = float(0.001) # (training only)

# overwrite defaults
if 'hyperparameters' in config:
    hp = config['hyperparameters']
    EPISODES          = int(hp['episodes'])          if 'episodes'          in hp else EPISODES
    WARMUP_EPISODES   = int(hp['warmup_episodes'])   if 'warmup_episodes'   in hp else WARMUP_EPISODES
    EPSILON_EPISODES  = int(hp['epsilon_episodes'])  if 'epsilon_episodes'  in hp else EPSILON_EPISODES
    EPSILON_START     = float(hp['epsilon_start'])   if 'epsilon_start'     in hp else EPSILON_START
    EPSILON_END       = float(hp['epsilon_end'])     if 'epsilon_end'       in hp else EPSILON_END
    REPLAY_BUFFERSIZE = int(hp['replay_buffersize']) if 'replay_buffersize' in hp else REPLAY_BUFFERSIZE
    REPLAY_BATCHSIZE  = int(hp['replay_batchsize'])  if 'replay_batchsize'  in hp else REPLAY_BATCHSIZE
    REPLAY_STEPS      = int(hp['replay_steps'])      if 'replay_steps'      in hp else REPLAY_STEPS
    if 'prio_replay' in hp:
        PRIO_REPLAY   = True if booleanpattern.match(hp['prio_replay']) else False
    Q_RESET_STEPS     = int(hp['q_reset_steps'])     if 'q_reset_steps'     in hp else Q_RESET_STEPS
    GAMMA             = float(hp['gamma'])           if 'gamma'             in hp else GAMMA
    LEARNING_RATE     = float(hp['learning_rate'])   if 'learning_rate'     in hp else LEARNING_RATE

# model- defaults (only if model is not loaded from file)
MODEL_H1 = int(10)     # hidden layer size 1
MODEL_H2 = int(10)     # hidden layer size 2

# filenames for loading the model and buffer/memory of transistions
load_file = 'Q.model' if not TRAIN else None # only default when not training
load_transitions_file = None
# filenames for saving the model (and the "best model") and buffer/memory of transistions
save_file = 'Q.out.model' if TRAIN else None # only default when training
save_best_file = 'Q.out.best.model' if TRAIN else None # only default when training
save_transitions_file = None

# overwrite defaults
if 'model' in config:
    m = config['model']
    MODEL_H1 = int(m['h1'])  if 'h1' in m else MODEL_H1
    MODEL_H2 = int(m['h2'])  if 'h2' in m else MODEL_H2
    load_file = m['load_file']                         if 'load_file'             in m else load_file
    load_transitions_file = m['load_transitions_file'] if 'load_transitions_file' in m else load_transitions_file
    save_file = m['save_file']                         if 'save_file'             in m else save_file
    save_best_file = m['save_best_file']               if 'save_best_file'        in m else save_best_file
    save_transitions_file = m['save_transitions_file'] if 'save_transitions_file' in m else save_transitions_file

# consistency check, reset values not consistent
if EPSILON_EPISODES <= 0:
    EPSILON_EPISODES = 0
    EPSILON_START = EPSILON_END
    DELTA_EPSILON = 0.0
# computed
else:
    DELTA_EPSILON = (EPSILON_START - EPSILON_END)/float(EPSILON_EPISODES +1)

# * ---------------- *
#   writing the used config to the logfile
# * ---------------- *
rl.write('# TRAIN (mode):      {}\n'.format(TRAIN))
rl.write('# SHOW (mode):       {}\n\n'.format(SHOW))
rl.write('# EPSILON_EPISODES:  {}\n'.format(EPSILON_EPISODES))
rl.write('# EPISODES:          {}\n'.format(EPISODES))
rl.write('# WARMUP_EPISODES:   {}\n'.format(WARMUP_EPISODES))
rl.write('# EPSILON_START:     {}\n'.format(EPSILON_START))
rl.write('# EPSILON_END:       {}\n'.format(EPSILON_END))
rl.write('# DELTA_EPSILON:     {}\n'.format(DELTA_EPSILON))
rl.write('# REPLAY_BUFFERSIZE: {}\n'.format(REPLAY_BUFFERSIZE))
rl.write('# REPLAY_BATCHSIZE:  {}\n'.format(REPLAY_BATCHSIZE))
rl.write('# REPLAY_STEPS:      {}\n'.format(REPLAY_STEPS))
rl.write('# PRIO_REPLAY:       {}\n'.format(PRIO_REPLAY))
rl.write('# Q_RESET_STEPS:     {}\n'.format(Q_RESET_STEPS))
rl.write('# GAMMA:             {}\n'.format(GAMMA))
rl.write('# LEARNING_RATE:     {}\n'.format(LEARNING_RATE))
rl.write('#   -- model\n')
rl.write('# H1: {}\n'.format(MODEL_H1))
rl.write('# H2: {}\n'.format(MODEL_H2))
rl.write('# load_file: {}\n'.format(load_file))
rl.write('# load_transitions_file: {}\n'.format(load_transitions_file))
rl.write('# save_file: {}\n'.format(save_file))
rl.write('# save_best_file: {}\n'.format(save_best_file))
rl.write('# save_transitions_file: {}\n'.format(save_transitions_file))
rl.flush()

# * ---------------- *
#   torch:
#    local computer was a laptop with no CUDA available
#    => feel free to change this, if you have a machine (with GPU)
# * ---------------- *
dtype = torch.float64
torch.set_default_dtype(dtype)
device = torch.device("cpu")
# device = torch.device("cuda:0") # Uncomment this to run on GPU

# * ---------------- *
#   buildung and initializing the torch- models for Q and Q_
# * ---------------- *
if load_file and os.path.isfile(load_file):
    modelQ = torch.load(load_file)
    modelQ_ = torch.load(load_file) # instead of zeros, we initialize with the same model

    rl.write('# [info] loaded model(s) from "{}"\n'.format(load_file))
    rl.write('# [info]   => H1, H2 parameters not used (nn-sizes from file model are implicitly used)\n')
    rl.flush()
elif not TRAIN:
    rl.write('# [error] not training, but model not loaded from file ("{}").\n'.format(load_file))
    rl.close()
    quit()
else:
    L1 = torch.nn.Linear(STATE_SIZE, MODEL_H1)
    torch.nn.init.uniform_(L1.weight) # uniform [0,1) for the actual Q function
    torch.nn.init.uniform_(L1.bias)
    L2 = torch.nn.Linear(MODEL_H1, MODEL_H2)
    torch.nn.init.uniform_(L2.weight)
    torch.nn.init.uniform_(L2.bias)
    L3 = torch.nn.Linear(MODEL_H2, ACTION_SIZE)
    torch.nn.init.uniform_(L3.weight)
    torch.nn.init.uniform_(L3.bias)

    modelQ = torch.nn.Sequential(
        L1
        ,torch.nn.ReLU()
        ,L2
        ,torch.nn.ReLU()
        ,L3
    #    ,torch.nn.LogSoftmax()
    )

    L1_ = torch.nn.Linear(STATE_SIZE, MODEL_H1)
    torch.nn.init.constant_(L1_.weight,fzero) # zeros, for the fixed Q
    torch.nn.init.constant_(L1_.bias,fzero)
    L2_ = torch.nn.Linear(MODEL_H1, MODEL_H2)
    torch.nn.init.constant_(L2_.weight,fzero)
    torch.nn.init.constant_(L2_.bias,fzero)
    L3_ = torch.nn.Linear(MODEL_H2, ACTION_SIZE)
    torch.nn.init.constant_(L3_.weight,fzero)
    torch.nn.init.constant_(L3_.bias,fzero)

    modelQ_ = torch.nn.Sequential(
        L1_
        ,torch.nn.ReLU()
        ,L2_
        ,torch.nn.ReLU()
        ,L3_
    #    ,torch.nn.LogSoftmax()
    )

    L1.weight.requires_grad_(requires_grad=True)
    L2.weight.requires_grad_(requires_grad=True)
    L3.weight.requires_grad_(requires_grad=True)
    L1.bias.requires_grad_(requires_grad=True)
    L2.bias.requires_grad_(requires_grad=True)
    L3.bias.requires_grad_(requires_grad=True)

    L1_.weight.requires_grad_(requires_grad=False)
    L2_.weight.requires_grad_(requires_grad=False)
    L3_.weight.requires_grad_(requires_grad=False)
    L1_.bias.requires_grad_(requires_grad=False)
    L2_.bias.requires_grad_(requires_grad=False)
    L3_.bias.requires_grad_(requires_grad=False)

if TRAIN:
    optimizer = torch.optim.Adam(modelQ.parameters(),lr=LEARNING_RATE)

# * ---------------- *
#   loading the Banana environment, loading the default brain (external)
# * ---------------- *
env = UnityEnvironment(file_name="Banana_Linux/Banana.x86_64")
brain_name = env.brain_names[0]
brain = env.brains[brain_name]

# * ---------------- *
#   the actual algorithm
# * ---------------- *

if TRAIN:
    # a very simple replay memory, a list (of tuples)
    #   - is assumed to never shrink
    #   - only inserts at given index, if next index would be > size, start at 0
    #         => list entries 0..size-1 are occupied
    replay_memory = []   # actual replay memory
    p_array       = []   # priority array
    if PRIO_REPLAY:
        index_array   = [i for i in range(REPLAY_BUFFERSIZE)]
        min_p         = abs(0.00000001) # value added to |delta| for numeric stability
    rm_size = 0          # number of entries in replay memory
    rm_next = 0          # next index to use for insert

    # we always laod/save transitions together with the priority- p_array
    #  ... even if we do not or had not used priority replay
    if load_transitions_file and os.path.isfile(load_transitions_file):
        with open(load_transitions_file, 'rb') as f:
            ( tmpm, tmpp ) = pickle.load(f)

            replay_memory = tmpm if REPLAY_BUFFERSIZE >= len(tmpm) else tmpm[0:REPLAY_BUFFERSIZE]
            rm_size = len(replay_memory)
            rm_next = rm_size if rm_size < REPLAY_BUFFERSIZE else 0

            if PRIO_REPLAY:
                # we need to make certain, that p_array has the same length
                #  as the replay-memory (we pad with the average if necc.)
                p_array = tmpp if tmpp else []
                if len(p_array) > rm_size:
                    p_array = p_array[0:rm_size]
                elif len(p_array) == 0:
                    if rm_size > 0:
                        p_array = [float(1.0) for i in range(rm_size)]
                elif len(p_array) < rm_size:
                    ptmpv = float(sum(p_array))/float(len(p_array))
                    tmplen = len(p_array)
                    for i in range(tmplen,rm_size):
                        p_array.append(ptmpv)
            else:
                p_array = [] # discard if we don't use prio-replay

# score buffer, for the last 100 scores
score_buffer = []
max_score = float('-inf')
best_model = None

epsilon = EPSILON_START
q_steps = 0
r_steps = 0

rl.write('#\n# Episode Score average(last-100-Scores) Steps Epsilon\n')

for episode in range(1,EPISODES+1):
    train_mode = not SHOW
    env_info = env.reset(train_mode=train_mode)[brain_name] # reset the environment
    state = env_info.vector_observations[0]            # get the start state
    score = 0                                          # initialize the episode- score
    step  = 0                                          # step within episodes
    if TRAIN and episode > WARMUP_EPISODES and (epsilon - DELTA_EPSILON + 1e-10) >= EPSILON_END:
        epsilon -= DELTA_EPSILON
    while True:
        step += 1
        rs = np.random.random_sample()
        if TRAIN and episode <= WARMUP_EPISODES:
            action = np.random.randint(ACTION_SIZE)
        elif (epsilon >= fone or rs < epsilon):
            action = np.random.randint(ACTION_SIZE)
        else:
            pred_a = modelQ(torch.tensor(state))
            action = int(torch.argmax(pred_a))

        env_info = env.step(action)[brain_name]        # send the action to the environment
        next_state = env_info.vector_observations[0]   # get the next state
        reward = env_info.rewards[0]                   # get the reward
        done = env_info.local_done[0]                  # see if episode has finished

        #print("Episode: {}; Step: {}; Reward: {}".format(episode,step,reward))

        if TRAIN:
            if PRIO_REPLAY:
                p = abs(float( reward + GAMMA * max(modelQ(torch.tensor(next_state))) - modelQ(torch.tensor(state))[action] )) + min_p
            # store transition in replay memory
            transition = (state,action,reward,next_state,done)
            if rm_size < REPLAY_BUFFERSIZE:
                replay_memory.append(transition)
                if PRIO_REPLAY:
                    p_array.append(p)
                rm_size += 1
            else:
                replay_memory[rm_next] = transition
                if PRIO_REPLAY:
                    p_array[rm_next] = p
            rm_next += 1
            if rm_next >= REPLAY_BUFFERSIZE:
                rm_next = 0

            if rm_size >= REPLAY_BATCHSIZE and episode > WARMUP_EPISODES:
                r_steps += 1
                if r_steps >= REPLAY_STEPS:
                    r_steps = 0

                    # learn with random sampled transitions from the past
                    if PRIO_REPLAY:
                        p_sum = float(sum(p_array))
                        tmp = float(1)/p_sum
                        P = np.array(p_array) * tmp
                        #print('[DEBUG] p_array: ',p_array)
                        #print('[DEBUG] P: ',P)
                        if rm_size < REPLAY_BUFFERSIZE:
                            batch_idx = np.random.choice(index_array[0:rm_size],size=REPLAY_BATCHSIZE,p=P)
                        else:
                            batch_idx = np.random.choice(index_array,size=REPLAY_BATCHSIZE,p=P)
                    else:
                        batch_idx = np.random.randint(rm_size, size=REPLAY_BATCHSIZE)
                    #print('[DEBUG] batch_idx: ',batch_idx)

                    listt  = [replay_memory[idx] for idx in batch_idx] # transitions
                    lists  = torch.tensor([s for s,a,r,ns,d in listt],dtype=torch.float64) # states
                    lista  = torch.tensor([a for s,a,r,ns,d in listt],dtype=torch.int64) # actions
                    listr  = torch.tensor([[r] for s,a,r,ns,d in listt],dtype=torch.float64) # rewards
                    listns = torch.tensor([ns for s,a,r,ns,d in listt],dtype=torch.float64) # next states
                    listd  = torch.tensor([[d] for s,a,r,ns,d in listt],dtype=torch.int64) # "done"- flags
                    tmpzeroes = torch.tensor([0 for i in range(0,len(batch_idx))],dtype=torch.int64)
                    tmpa   = torch.stack([lista] + [tmpzeroes for i in range(1,ACTION_SIZE)])
                    tmpa   = torch.t(tmpa)

                    pred_v = torch.index_select(modelQ(lists).gather(1, tmpa), 1, torch.tensor([0]))

                    pred_v_, _ = torch.max(modelQ_(listns),dim=1,keepdim=True)
                    pred_v_ =  listr  + pred_v_ * GAMMA * torch.tensor((1 - listd),dtype=torch.float64)

                    #quit()

                    loss = fct.mse_loss(pred_v,pred_v_)

                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()

        score += reward                                # update the score
        state = next_state                             # roll over the state to next time step

        if TRAIN:
            q_steps += 1
            if q_steps >= Q_RESET_STEPS:
                modelQ_.load_state_dict(modelQ.state_dict())
                q_steps = 0

        if done:                                       # exit loop if episode finished
            break

    if max_score < score:
        max_score = score
        best_model = copy.deepcopy(modelQ)

    score_buffer.append(score)
    while len(score_buffer) > 100:
        score_buffer.pop(0)
    l100_score = float(sum(score_buffer))/float(len(score_buffer)) if len(score_buffer) >= 100 else float(0)

    rl.write('{} {} {} {} {}\n'.format(episode,score,l100_score,step,epsilon))
    rl.flush()
    print("Episode: {}; Score: {} ({}); #Steps: {}; Epsilon: {}".format(episode,score,l100_score,step,epsilon))

env.close()

if TRAIN:
    if save_file:
        rl.write('# .. writing final model to "{}"\n'.format(save_file))
        torch.save(modelQ,save_file)
    if save_best_file:
        rl.write('# .. writing best model to "{}"\n'.format(save_best_file))
        torch.save(best_model,save_best_file)

    if save_transitions_file:
        rl.write('# .. saving transisitions to "{}"\n'.format(save_transitions_file))
        with open(save_transitions_file, 'wb') as f:
            pickledata = ( replay_memory,p_array )
            pickle.dump(pickledata, f, pickle.HIGHEST_PROTOCOL)

rl.close()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

headers  = {
    'Connection': 'keep-alive',
    'Accept': 'application/json, text/plain, */*',
    'x-nba-stats-token': 'true',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
    'x-nba-stats-origin': 'stats',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-Mode': 'cors',
    'Referer': 'https://stats.nba.com/',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en-US,en;q=0.9',
}
column_names = ["PLAYER_ID",
"PLAYER_NAME",
"NICKNAME",
"TEAM_ID",
"TEAM_ABBREVIATION",
"AGE",
"GP",
"W",
"L",
"W_PCT",
"MIN",
"FGM",
"FGA",
"FG_PCT",
"FG3M",
"FG3A",
"FG3_PCT",
"FTM",
"FTA",
"FT_PCT",
"OREB",
"DREB",
"REB",
"AST",
"TOV",
"STL",
"BLK",
"BLKA",
"PF",
"PFD",
"PTS",
"PLUS_MINUS",
"NBA_FANTASY_PTS",
"DD2",
"TD3",
"WNBA_FANTASY_PTS",
"GP_RANK",
"W_RANK",
"L_RANK",
"W_PCT_RANK",
"MIN_RANK",
"FGM_RANK",
"FGA_RANK",
"FG_PCT_RANK",
"FG3M_RANK",
"FG3A_RANK",
"FG3_PCT_RANK",
"FTM_RANK",
"FTA_RANK",
"FT_PCT_RANK",
"OREB_RANK",
"DREB_RANK",
"REB_RANK",
"AST_RANK",
"TOV_RANK",
"STL_RANK",
"BLK_RANK",
"BLKA_RANK",
"PF_RANK",
"PFD_RANK",
"PTS_RANK",
"PLUS_MINUS_RANK",
"NBA_FANTASY_PTS_RANK",
"DD2_RANK",
"TD3_RANK",
"WNBA_FANTASY_PTS_RANK",
"CFID",
"CFPARAMS"]

cols = ["PLAYER_NAME","W_PCT", "MIN", "GP","FG_PCT_RANK","PTS_RANK", "NBA_FANTASY_PTS_RANK","W_PCT_RANK", "PLUS_MINUS_RANK","REB_RANK","AST_RANK","TOV_RANK","STL_RANK","BLK_RANK","FTA_RANK", "Year"]

# In[1]:

import requests
import pandas as pd
from bs4 import BeautifulSoup

# In[ ]:

d = [*range(96,100,1)]
e = [*range(0,10,1)]
f = [*range(10,23,1)]
for i in range(len(e)):
    e[i] = '%02d' % i

for i in range(len(d)):
    d[i] = str(d[i])

for i in range(len(f)):
    f[i] = str(f[i])

years = d + e + f

for i in range(len(years)-1):
    years[i] = str(years[i]+"-"+years[i+1])

    if years[i][0] == '8' or years[i][0] == '9':
        years[i] = str('19'+years[i])

for i in range(4,len(years)-1,1):
    years[i] = str('20'+years[i])

years.pop()
print(years)

# In[ ]:

urlyearslist = []
i = 0
while i < len(years):
    playerinfourl = 'https://stats.nba.com/stats/leaguedashplayerstats?College=&Conference=&Country=&DateFrom=&DateTo=&Division=&DraftPick=&DraftYear=&GameScope=&GameSegment=&Height=&LastNGames=0&LeagueID=00&Location=&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=PerGame&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=' + years[i] + '&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&TwoWay=0&VsConference=&VsDivision=&Weight='
    urlyearslist.append(str(playerinfourl))
    i = i+1

# In[9]:

years_list = [*range(1997,2023,1)]
print(len(years_list))

# In[ ]:

data_list = []

j = 0
while j < len(urlyearslist):
    tempurl = urlyearslist[j]

    response = requests.get(url=tempurl,headers=headers).json()
    player_info = response['resultSets'][0]['rowSet']

    year_data = pd.DataFrame(player_info, columns = column_names)
    #newdf = year_data[cols]
    year_data["Year"] = years_list[j]

    data_list.append(year_data)

    j = j+1

player_stats_all_years = pd.concat(data_list)

# In[ ]:

player_stats_all_years

# In[ ]:

player_stats_all_years.to_csv("full_player_stats.csv")

# In[ ]:

data_list = []

for year in years_list:
    with open("years_html/{}.html".format(year),encoding="utf-8") as f:
        page = f.read()

    soup = BeautifulSoup(page, "html.parser")
    soup.find('tr', class_="over_header").decompose()
    mvp_table = soup.find(id="mvp")
    mvp = pd.read_html(str(mvp_table))[0]
    mvp["Year"] = year
    data_list.append(mvp)

# In[ ]:

mvp_all_years = pd.concat(data_list)
mvp_all_years.to_csv("mvp_all_years.csv")

# In[ ]:

mvps_df = pd.read_csv("mvp_all_years.csv")

# In[ ]:

mvp_columns = ["Player","Year","Pts Won", "Pts Max", "Share"]
mvps_df = mvps_df[mvp_columns]
mvps_df.rename(columns = {'Player':'PLAYER_NAME'}, inplace = True)
mvps_df["PLAYER_NAME"] = mvps_df["PLAYER_NAME"].str.replace("ć","c", regex=False)
mvps_df["PLAYER_NAME"] = mvps_df["PLAYER_NAME"].str.replace("č","c", regex=False)
mvps_df["PLAYER_NAME"] = mvps_df["PLAYER_NAME"].str.replace("ó","o", regex=False)
mvps_df.head()

# In[ ]:

all_players_df = pd.read_csv("full_player_stats.csv")
#del all_players_df["Unnamed: 0"]

a = mvps_df.loc[(mvps_df['PLAYER_NAME'] == 'Steve Smith')]
a

# In[ ]:

mvps_df = mvps_df[mvps_df['PLAYER_NAME'] != 'Steve Smith']

# In[ ]:

players_and_mvps = all_players_df.merge(mvps_df, how="outer", on=["PLAYER_NAME", "Year"])

# In[ ]:

replace_nan = ["Pts Won", "Pts Max", "Share"]
players_and_mvps[replace_nan] = players_and_mvps[replace_nan].fillna(0)
players_and_mvps.head(10)

# In[ ]:

cols = ["PLAYER_NAME","W_PCT", "MIN", "GP","FG_PCT_RANK","PTS_RANK", "NBA_FANTASY_PTS_RANK","W_PCT_RANK", "PLUS_MINUS_RANK","REB_RANK","AST_RANK","TOV_RANK","STL_RANK","BLK_RANK","FTA_RANK", "Year", "Pts Won", "Pts Max", "Share"]

# In[ ]:

players_and_mvps = players_and_mvps[cols]

# In[ ]:

players_and_mvps

# In[ ]:

players_and_mvps = players_and_mvps[players_and_mvps['PLAYER_NAME'].notna()]
pd.isnull(players_and_mvps).sum()

# In[ ]:

players_and_mvps.to_csv("ml_players_mvp_stats.csv")

# In[ ]:

players_and_mvps.corr()["Share"].sort_values(ascending=False)

# In[2]:

input_data = pd.read_csv("ml_players_mvp_stats.csv")
input_data.columns
del input_data["Unnamed: 0"]

# In[3]:

ml_years = list(range(1997,2023))
all_data = []
for year in ml_years:
    test = input_data[input_data["Year"] == year]
    all_data.append(test)
print(all_data)

# In[4]:

m = 0
while m<len(ml_years):
    if m == 23:
        all_data[m] = all_data[m][all_data[m]['GP']>40]
        all_data[m] = all_data[m][all_data[m]['MIN']>25]
        m = m+1
        continue

    else:
        all_data[m] = all_data[m][all_data[m]['GP']>55]
        all_data[m] = all_data[m][all_data[m]['MIN']>25]

    m = m+1

# In[5]:

input_data = pd.concat(all_data)

# In[6]:

features = ['W_PCT', 'MIN', 'GP', 'FG_PCT_RANK', 'PTS_RANK',
       'NBA_FANTASY_PTS_RANK', 'W_PCT_RANK', 'PLUS_MINUS_RANK', 'REB_RANK',
       'AST_RANK', 'TOV_RANK', 'STL_RANK', 'BLK_RANK', 'FTA_RANK']

# In[23]:

one_year_train = input_data[input_data["Year"]< 2022]
one_year_test = input_data[input_data["Year"] == 2022]

# In[24]:

from sklearn.linear_model import Ridge
sample_reg = Ridge(alpha=.1)
sample_reg.fit(one_year_train[features], one_year_train["Share"])

# In[25]:

sample_predictions = sample_reg.predict(one_year_test[features])
sample_predictions = pd.DataFrame(sample_predictions, columns = ["predictions"], index=one_year_test.index)
sample_playerprediction = pd.concat([one_year_test[["PLAYER_NAME","Share"]],sample_predictions],axis=1)

# In[26]:

sample_playerprediction = sample_playerprediction.sort_values("Share", ascending=False)
sample_playerprediction["rank"] = list(range(1,sample_playerprediction.shape[0]+1))
sample_playerprediction.head(10)

# In[27]:

sample_playerprediction = sample_playerprediction.sort_values("predictions", ascending=False)
sample_playerprediction["predicted_rank"] = list(range(1,sample_playerprediction.shape[0]+1))
sample_playerprediction.head(10)
#ridge regression training on 25 years and testing on 2022 did not show the correct prediction

# In[4]:

ml_years = list(range(1997,2023))
ml_years

# In[28]:

all_predictions = []
for year in ml_years[5:]:
    train = input_data[input_data["Year"] < year]
    test = input_data[input_data["Year"] == year]
    sample_reg.fit(train[features], train["Share"])
    predictions = sample_reg.predict(test[features])
    predictions = pd.DataFrame(predictions, columns = ["predictions"], index=test.index)
    combination = pd.concat([test[["PLAYER_NAME","Share"]],predictions],axis=1)
    all_predictions.append(combination)

# In[8]:

def add_ranks(combination):
    combination = combination.sort_values("Share", ascending=False)
    combination["rank"] = list(range(1,combination.shape[0]+1))
    combination = combination.sort_values("predictions", ascending=False)
    combination["predicted_rank"] = list(range(1,combination.shape[0]+1))
    return combination

# In[9]:

def ml_test(stats,model,years,predictors):
    all_predictions = []
    for year in years:

        train = stats[stats["Year"] < year]
        test = stats[stats["Year"] == year]
        model.fit(train[predictors], train["Share"])
        predictions = model.predict(test[predictors])
        predictions = pd.DataFrame(predictions, columns = ["predictions"], index=test.index)
        combination = pd.concat([test[["PLAYER_NAME","Share"]],predictions],axis=1)
        combination = add_ranks(combination)
        all_predictions.append(combination)
    return all_predictions

# In[31]:

get_ipython().system(' pip install sklearn')

# In[10]:

def scores(predictions):
    correct = 0
    total = 0
    for year in predictions:
        row_1=year.iloc[0]
        if row_1['rank'] == 1 and row_1['predicted_rank'] == 1:
            correct += 1
        total += 1
    score = correct/total
    return score

# In[34]:

alpha_list = [x/10 for x in range(0, 101, 1)]
def ridge_alpha_tuning(stats,alpha_list,years,predictors):
    alpha_tuning_predictions = []
    alpha_scores = pd.DataFrame(columns = ['Alpha', 'Score'])
    for alpha in alpha_list:
        model = Ridge(alpha=alpha)
        for year in years:
            train = stats[stats["Year"] < year]
            test = stats[stats["Year"] == year]
            model.fit(train[predictors], train["Share"])
            predictions = model.predict(test[predictors])
            predictions = pd.DataFrame(predictions, columns = ["predictions"], index=test.index)
            combination = pd.concat([test[["PLAYER_NAME","Share"]],predictions],axis=1)
            combination = add_ranks(combination)
            alpha_tuning_predictions.append(combination)
        score = scores(alpha_tuning_predictions)

        alpha_scores = alpha_scores.append({'Alpha' : alpha, 'Score' : score}, ignore_index = True)
    return alpha_scores

ridge_alpha_tuning_test = ridge_alpha_tuning(input_data,alpha_list,ml_years[5:], features)
print(ridge_alpha_tuning_test)

# In[35]:

print(ridge_alpha_tuning_test.sort_values("Score", ascending=False))
#ridge regression training on 5 years and testing on 21 years produced a best accuracy of 0.52 with small alphas

# In[36]:

ridge_alpha_tuning_test_ten = ridge_alpha_tuning(input_data,alpha_list,ml_years[10:], features)
print(ridge_alpha_tuning_test_ten)
#ridge regression training on 10 years and testing on 16 years produced a best accuracy of 0.63 with small alphas

# In[33]:

from sklearn.linear_model import LinearRegression
linear_reg = LinearRegression()
linear_reg_predictions = ml_test(input_data, linear_reg,ml_years[5:], features)
linear_reg_score = scores(linear_reg_predictions)
print(linear_reg_score)
#Linear regression training on 5 years and testing on 21 years produced an accuracy of 0.52 same as ridge regression

# In[38]:

linear_reg_predictions_ten = ml_test(input_data, linear_reg,ml_years[10:], features)
linear_reg_score_ten = scores(linear_reg_predictions_ten)
print(linear_reg_score_ten)
#Linear regression training on 10 years and testing on 16 years produced an accuracy of 0.625 same as ridge regression

# In[39]:

from sklearn.ensemble import RandomForestRegressor
rf = RandomForestRegressor(n_estimators = 50, random_state=1, min_samples_split=4)
rf_all_predictions = ml_test(input_data, rf,ml_years[5:], features)

rf_score = scores(rf_all_predictions)
print(rf_score)
print(rf_all_predictions)

# random forest regression with the above parameters training on 5 years and testing on 10 years produce an accuracy of 0.67

# In[46]:

rf_all_predictions_ten = ml_test(input_data, rf,ml_years[10:], features)

rf_score_ten = scores(rf_all_predictions_ten)
print(rf_score_ten)
# random forest regression with the above parameters training on 10 years and testing on 21 years produce an accuracy of 0.69

# In[40]:

from sklearn.model_selection import RandomizedSearchCV
import numpy as np

n_estimators = [int(x) for x in range(50,1050,50)]

max_features = ['auto', 'sqrt']

max_depth = [int(x) for x in range(10, 110, 10)]
max_depth.append(None)
print(max_depth)
min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]

random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}

# In[41]:

rf_random = RandomizedSearchCV(estimator=rf, param_distributions=random_grid,
                              n_iter = 100, scoring='neg_mean_absolute_error',
                              cv = 3, verbose=2, random_state=42, n_jobs=-1,
                              return_train_score=True)

rf_random.fit(train[features], train["Share"])
rf_random.best_params_

# In[43]:

rf_parameter_tuned = RandomForestRegressor(n_estimators = 100, min_samples_split = 2, min_samples_leaf = 2, max_features = 'auto', max_depth = 50, bootstrap = True)
rf_parameter_tuned_predictions = ml_test(input_data, rf_parameter_tuned,ml_years[5:], features)

rf_parameter_tuned_score = scores(rf_parameter_tuned_predictions)
print(rf_parameter_tuned_score)

#Hyper parameter tuned random forest regressor training for 5 years and testing for 21 makes accurary 67% - no improvement in accuracy

# In[44]:

from sklearn.ensemble import RandomForestRegressor
rf_parameter_tuned = RandomForestRegressor(n_estimators = 100, min_samples_split = 2, min_samples_leaf = 2, max_features = 'auto', max_depth = 50, bootstrap = True)
rf_parameter_tuned_predictions_ten = ml_test(input_data, rf_parameter_tuned, ml_years[10:], features)

rf_parameter_tuned_score_ten = scores(rf_parameter_tuned_predictions_ten)
print(rf_parameter_tuned_score_ten)

#Hyper parameter tuned random forest regressor training for 10 years and testing for 16 makes accurary 75%

# In[11]:

from sklearn.svm import SVR
svm_classifier = SVR()
svm_all_predictions = ml_test(input_data, svm_classifier,ml_years[5:], features)

svm_score = scores(svm_all_predictions)
print(svm_score)
print(svm_all_predictions)
# Support vector regression testing on 5 years and testing on 21 gives an accuracy of 0.52

# In[48]:

svm_all_predictions_ten = ml_test(input_data, svm_classifier,ml_years[10:], features)

svm_score_ten = scores(svm_all_predictions_ten)
print(svm_score_ten)

# Support vector regression testing on 10 years and testing on 16 gives an accuracy of 0.63

# In[ ]:

from sklearn.model_selection import GridSearchCV
param_grid = {'C': [0.1,1, 20, 50], 'epsilon': [1,0.1,0.01,0.001],'kernel': ['rbf', 'poly', 'sigmoid']}
grid = GridSearchCV(svm_classifier,param_grid, return_train_score=True)

grid.fit(train[features], train["Share"])
grid.best_params_

# In[12]:

svm_tuned_regressor = SVR(C=50,epsilon=.001,kernel="rbf")
svm_tuned_all_predictions = ml_test(input_data, svm_tuned_regressor,ml_years[5:], features)

svm_tuned_score = scores(svm_tuned_all_predictions)
print(svm_tuned_score)
print(svm_tuned_all_predictions)
# Hyperparameter Tuned support vector regression testing on 5 years and testing on 21 gives an accuracy of 0.57

# In[13]:

svm_tuned_all_predictions_ten = ml_test(input_data, svm_tuned_regressor,ml_years[10:], features)

svm_tuned_score_ten = scores(svm_tuned_all_predictions_ten)
print(svm_tuned_score_ten)
# Hyperparameter Tuned support vector regression testing on 5 years and testing on 21 gives an accuracy of 0.56
# this is interesting

# In[14]:

from sklearn.tree import DecisionTreeRegressor

decision_tree = DecisionTreeRegressor()
decision_tree_predictions = ml_test(input_data, decision_tree,ml_years[5:], features)

decision_tree_score = scores(decision_tree_predictions)
print(decision_tree_score)
print(decision_tree_predictions)
# Decision Tree regression training on 5 years testing on 21 produces accuracy of 0.47

# In[16]:

decision_tree_predictions_ten = ml_test(input_data, decision_tree,ml_years[10:], features)

decision_tree_score_ten = scores(decision_tree_predictions_ten)
print(decision_tree_score_ten)
# Decision Tree regression training on 10 years testing on 16 produces accuracy of 0.44
# Lower then training on 5 years

# In[17]:

from sklearn.neighbors import KNeighborsRegressor
k_neighbors = KNeighborsRegressor()

k_neighbors_predictions = ml_test(input_data, k_neighbors,ml_years[5:], features)

k_neighbors_score = scores(k_neighbors_predictions)
print(k_neighbors_score)
print(k_neighbors_predictions)
# k_nearest neighbors regression training for 5 years and testing for 21 produces 0.38 accuracy

# In[18]:

k_neighbors_predictions_ten = ml_test(input_data, k_neighbors,ml_years[10:], features)

k_neighbors_score_ten = scores(k_neighbors_predictions_ten)
print(k_neighbors_score_ten)
# k_nearest neighbors regression training for 10 years and testing for 16 produces 0.44 accuracy

# In[ ]:

#Highest accuracy model is hyper parameter tuned random forest regressor training for 10 years and testing for 16 with accurary 75%

NBA MVP Machine Learning Regression Model.py


import numpy as np
from sklearn.neighbors import NearestNeighbors
import numpy as np

def calc_SL(W,n_objs=3):
    num = min(n_objs,len(W))

    neigh = NearestNeighbors(num)
    neigh.fit(W)

    neighbors = neigh.kneighbors(W, num)[0].tolist()
    SL = [np.prod(item[1:num]) for item in neighbors]
    return SL

def delete_vector_random(X, Y, W, z, Archive, delete_ratio = 0.01,lower_pop=10):
    arg_edges = (W == 1).any(axis=1)
    W_edges = W[arg_edges]
    X_edges = X[arg_edges]
    Y_edges = Y[arg_edges]
    if len(W)<=lower_pop:
        return X, Y, W, max(2, len(W)//10)

    W_ = W[~arg_edges]
    X_ = X[~arg_edges]
    Y_ = Y[~arg_edges]

    num  = int(len(W_) - len(W_)*delete_ratio)
    arg_remain = np.random.choice(range(len(W_)),num,replace=False)
    W_ = W_[arg_remain]
    X_ = X_[arg_remain]
    Y_ = Y_[arg_remain]

    W_ = np.append(W_, W_edges,axis=0)
    X_ = np.append(X_, X_edges,axis=0)
    Y_ = np.append(Y_, Y_edges,axis=0)
    T = max(2, len(W_)//10)

    return X_, Y_, W_, T

# def delete_vector(X, Y, W, z, Archive, delete_ratio = 0.01,lower_pop=10):
#     arg_edges = (W == 1).any(axis=1)
#     W_edges = W[arg_edges]
#     X_edges = X[arg_edges]
#     Y_edges = Y[arg_edges]
#     if len(W)<=lower_pop:
#         return X, Y, W, max(2, len(W)//10)

#     W_ = W[~arg_edges]
#     X_ = X[~arg_edges]
#     Y_ = Y[~arg_edges]

#     SL = calc_SL(W_)
#     num  = int(len(W_) - len(W_)*delete_ratio)
#     L = np.argsort(SL)[::-1][-num:]

#     arg_remain = np.array([False if i in L else True for i in range(len(W_))])

#     W_ = W_[arg_remain]
#     X_ = X_[arg_remain]
#     Y_ = Y_[arg_remain]

#     W_ = np.append(W_, W_edges,axis=0)
#     X_ = np.append(X_, X_edges,axis=0)
#     Y_ = np.append(Y_, Y_edges,axis=0)
#     T = max(2, len(W_)//10)

#     return X_, Y_, W_, T

def delete_vector_AWA(X, Y, W, z, Archive, update_ratio = 0.05,first_pop=105):
    arg_edges = (W == 1).any(axis=1)
    W_edges = W[arg_edges]
    X_edges = X[arg_edges]
    Y_edges = Y[arg_edges]

    W_ = W[~arg_edges]
    X_ = X[~arg_edges]
    Y_ = Y[~arg_edges]

    nus = max(int(first_pop*update_ratio),1)
    L = []
    for i in range(nus):
        SL = calc_SL(W_)
        arg = np.argmin(SL)
        index = np.ones(len(W_),dtype=bool)
        index[arg] = False
        W_ = W_[index]
        X_ = X_[index]
        Y_ = Y_[index]

    W_ = np.append(W_, W_edges,axis=0)
    X_ = np.append(X_, X_edges,axis=0)
    Y_ = np.append(Y_, Y_edges,axis=0)
    T = max(2, len(W_)//10)

    return X_, Y_, W_, T


import numpy as np
import pytest
import torch

from tsbatteries import preprocessing
from tsbatteries.preprocessing.pipeline import (PipelineCompiler,
                                                PipelineDataset)
from tsbatteries.tests.helpers import make_classification_problem

@pytest.fixture()
def static_pipeline():
    pipeline = preprocessing.Pipeline(
        [
            ("negative_impute", preprocessing.NegativeFilter()),
            ("stdsc", preprocessing.TensorScaler(method="mms")),
        ]
    )
    return pipeline

@pytest.fixture()
def temporal_pipeline():
    pipeline = preprocessing.Pipeline(
        [
            ("pad", preprocessing.PadRaggedTensors()),
            ("negative_impute", preprocessing.NegativeFilter()),
            ("stdsc", preprocessing.TensorScaler(method="stdsc")),
            ("interpolation", preprocessing.Interpolation(method="linear")),
            ("backfill", preprocessing.ForwardFill(backwards=True)),
        ]
    )
    return pipeline

@pytest.fixture()
def label_pipeline():
    pipeline = preprocessing.LabelProcessor(problem="oneshot")
    return pipeline

def test_pipeline(temporal_pipeline):
    # Create random tensor data
    data = [torch.randn(5, 2) for _ in range(5)]
    assert torch.isnan(temporal_pipeline.fit_transform(data)).sum() == 0

@pytest.mark.parametrize(
    "static_dim, batch_size",
    [
        (None, None),
        (None, 32),
        (5, None),
        (5, 32),
    ],
)
def test_preprocessing_pipeline(
    static_pipeline, temporal_pipeline, label_pipeline, static_dim, batch_size
):
    # Check the preprocessing pipeline can be run effectively with and without static data
    train_data, _ = make_classification_problem(static_dim=5)
    static_data, temporal_data, labels = train_data
    temporal_data = np.array([x for x in temporal_data], dtype=object)
    if static_dim is None:
        static_data = None
        static_pipeline = None

    # Check fit transform works with full data
    dataset = PipelineDataset(static_data, temporal_data, labels)
    main_pipeline = PipelineCompiler(
        static_pipeline, temporal_pipeline, label_pipeline, batch_size=batch_size
    )
    output = main_pipeline.fit_transform(dataset)

    # Check fit transform works with and without static
    assert len(output) == 3


import pytest

from sabnzbd.newsunpack import *

class TestNewsUnpack:
    @pytest.mark.parametrize(
        "test_input, expected_output",
        [
            (["cmd1", 9, "cmd3"], '"cmd1" "9" "cmd3"'),  # sending all commands as valid string
            (["", "cmd1", "5"], '"" "cmd1" "5"'),  # sending blank string
            (["cmd1", None, "cmd3", "tail -f"], '"cmd1" "" "cmd3" "tail -f"'),  # sending None in command
            (["cmd1", 0, "ps ux"], '"cmd1" "" "ps ux"'),  # sending 0
        ],
    )
    def test_list_to_cmd(self, test_input, expected_output):
        """ Test to convert list to a cmd.exe-compatible command string """

        res = list2cmdline(test_input)
        # Make sure the output is cmd.exe-compatible
        assert res == expected_output

    def test_is_sfv_file(self):
        assert is_sfv_file("tests/data/good_sfv_unicode.sfv")
        assert is_sfv_file("tests/data/one_line.sfv")
        assert not is_sfv_file("tests/data/only_comments.sfv")
        assert not is_sfv_file("tests/data/random.bin")


from typing import List
class Solution:
    def maxArea(self, height: List[int]) -> int:
        left=0
        right = len(height)-1
        max_area = 0
        while left < right:
            min_height = min(height[left], height[right])
            area = (right - left) * min_height
            max_area = max(area, max_area)
            if min_height == height[left]:
                left += 1
            else:
                right -= 1
        return max_area


# sample 300 points from torus S1 x S1 and 100 points from circle S1 (belonging to the first S1)
# Compute distance among the points

import math
import numpy as np
import h5py

def angle_distance(phi, theta):
    s = min(phi, theta)
    l = max(phi, theta)
    distance = min(l-s, s + 2 * math.pi - l)

    return distance

def compute_distances(X):
    n_points = X.shape[0]
    d = np.zeros((n_points, n_points))
    for i in range(n_points):
        for j in range(i+1, n_points):
            d_phi = angle_distance(X[i,0], X[j,0])
            d_theta = angle_distance(X[i,1], X[j,1])
            distance = math.sqrt(d_phi**2 + d_theta **2)
            d[i][j] = distance
            d[j][i] = distance
    return d

def main():
    n_torus = 300
    n_circle = 100
    # generate 300 points on S1 x S1
    torus = np.random.uniform(0, 2*math.pi, (300, 2))
    # columns 0, 1, 2: phi, psi, theta

    # generate 100 points from S1
    circle = np.zeros((20, 2))
    for i in range(20):
        circle[i,0] = np.random.uniform(0, 2*math.pi)

        # add some noise
        circle[i,1] = np.random.normal(math.pi, 0.2)

    data = np.concatenate((torus, circle), axis=0)
    distance = compute_distances(data)

    # break into 4 distance matrices
    # Define submatrices
    D_torus = distance[:n_torus, :n_torus]
    D_circle = distance[n_torus:, n_torus:]
    D_torus_circle = distance[:n_torus, n_torus:]
        # rows (landmarks): torus
        # columns (witness) : circle
    D_circle_torus = distance[n_torus:, :n_torus];
        # rows (landmarks): circle
        # columns (witness) : torus
    np.savetxt("distance.csv", distance)
    np.savetxt("distance_torus.csv", D_torus)
    np.savetxt("distance_circle.csv", D_circle)
    np.savetxt("distance_torus_circle.csv", D_torus_circle)
    np.savetxt("distance_circle_torus.csv", D_circle_torus)

    # save coordinates for visualization comparison
    hf = h5py.File("coords.h5", "w")
    hf.create_dataset("torus", data = np.transpose(torus))
    hf.create_dataset("circle", data = np.transpose(circle))
    hf.close()
if __name__ == '__main__':
    main()