# %%
"""
Transport distributions using dense solvers
===========================================

In this example, we sample 2 toy distributions and compute
a dense fugw alignment between them.
Dense alignments are typically used when both aligned distributions
have less than 10k points.
"""

# sphinx_gallery_thumbnail_number = 4
import matplotlib.pyplot as plt
import numpy as np
import torch

from fugw.mappings import FUGW
from fugw.utils import _init_mock_distribution
from matplotlib.collections import LineCollection

# %%
torch.manual_seed(0)

n_points_source = 50
n_points_target = 40
n_features_train = 2
n_features_test = 2

# %%
# Let us generate random training data for the source and target distributions
_, source_features_train, source_geometry, source_embeddings = (
    _init_mock_distribution(n_features_train, n_points_source)
)
_, target_features_train, target_geometry, target_embeddings = (
    _init_mock_distribution(n_features_train, n_points_target)
)
source_features_train.shape

# %%
# We can visualize the generated features:
fig = plt.figure(figsize=(4, 4))
ax = fig.add_subplot()
ax.set_title("Source and target features")
ax.set_aspect("equal", "datalim")
ax.scatter(source_features_train[0], source_features_train[1], label="Source")
ax.scatter(target_features_train[0], target_features_train[1], label="Target")
ax.legend()
plt.show()

# %%
# And embeddings:
fig = plt.figure()
ax = fig.add_subplot(projection="3d")
ax.set_title("Source and target embeddings (ie geometries)")
ax.scatter(
    source_embeddings[:, 0],
    source_embeddings[:, 1],
    source_embeddings[:, 2],
    s=15,
    label="Source",
)
ax.scatter(
    target_embeddings[:, 0],
    target_embeddings[:, 1],
    target_embeddings[:, 2],
    s=15,
    label="Target",
)
ax.legend()
plt.show()

# %%
# Features and geometries should be normalized before calling the solver
source_features_train_normalized = source_features_train / torch.linalg.norm(
    source_features_train, dim=1
).reshape(-1, 1)
target_features_train_normalized = target_features_train / torch.linalg.norm(
    target_features_train, dim=1
).reshape(-1, 1)

source_geometry_normalized = source_geometry / source_geometry.max()
target_geometry_normalized = target_geometry / target_geometry.max()

# %%
# Let us define the optimization problem to solve
alpha = 0.5
rho = 1000
eps = 1e-4
mapping = FUGW(alpha=alpha, rho=rho, eps=eps)

# %%
# Now, we fit a transport plan between source and target distributions
# using a sinkhorn solver
_ = mapping.fit(
    source_features_train_normalized,
    target_features_train_normalized,
    source_geometry=source_geometry_normalized,
    target_geometry=target_geometry_normalized,
    solver="sinkhorn",
    verbose=True,
)

# %%
# The transport plan can be accessed after the model has been fitted
pi = mapping.pi
print(f"Transport plan's total mass: {pi.sum():.5f}")

# %%
# Here is the evolution of the FUGW loss during training,
# as well as the contribution of each loss term:

fig, ax = plt.subplots(figsize=(10, 4))
ax.set_title("Mapping training loss")
ax.set_ylabel("Loss")
ax.set_xlabel("BCD step")
ax.stackplot(
    mapping.loss_steps,
    [
        (1 - alpha) * np.array(mapping.loss["wasserstein"]),
        alpha * np.array(mapping.loss["gromov_wasserstein"]),
        rho * np.array(mapping.loss["marginal_constraint_dim1"]),
        rho * np.array(mapping.loss["marginal_constraint_dim2"]),
        eps * np.array(mapping.loss["regularization"]),
    ],
    labels=[
        "wasserstein",
        "gromov_wasserstein",
        "marginal_constraint_dim1",
        "marginal_constraint_dim2",
        "regularization",
    ],
    alpha=0.8,
)
ax.legend()
plt.show()

# %%
# Using the computed mapping
# --------------------------
# The computed mapping is stored in ``mapping.pi`` as a ``torch.Tensor``.
# In this example, the transport plan is small enough that we can display
# it altogether.

fig, ax = plt.subplots(figsize=(4, 4))
ax.set_title("Transport plan")
ax.set_xlabel("target vertices")
ax.set_ylabel("source vertices")
im = plt.imshow(pi, cmap="viridis")
plt.colorbar(im, ax=ax, shrink=0.8)
plt.show()

# %%
# The previous figure of the transport plan tells us it is very sparse
# and not very regularized.
# Another informative way to look at the plan consists in checking
# which points of the source and target distributions
# were matched together in the feature space.

fig = plt.figure(figsize=(4, 4))
ax = fig.add_subplot()
ax.set_aspect("equal", "datalim")
ax.set_title("Mapping\ndisplayed in feature space")

# Draw lines between matched points
indices = torch.cartesian_prod(
    torch.arange(n_points_source), torch.arange(n_points_target)
)
segments = torch.stack(
    [
        source_features_train[:, indices[:, 0]],
        target_features_train[:, indices[:, 1]],
    ]
).permute(2, 0, 1)
pi_normalized = pi / pi.sum(dim=1).reshape(-1, 1)
line_segments = LineCollection(
    segments, alpha=pi_normalized.flatten(), colors="black", lw=1, zorder=1
)
ax.add_collection(line_segments)

# Draw distributions
ax.scatter(source_features_train[0], source_features_train[1], label="Source")
ax.scatter(target_features_train[0], target_features_train[1], label="Target")

ax.legend()
plt.show()

# %%
# Finally, the fitted model can transport unseen data
# between source and target
source_features_test = torch.rand(n_features_test, n_points_source)
target_features_test = torch.rand(n_features_test, n_points_target)
transformed_data = mapping.transform(source_features_test)
transformed_data.shape

# %%
assert transformed_data.shape == target_features_test.shape

# %%


from matplotlib import pyplot as plt
import numpy as np

import figlatex
import afterpulse_tile21
import textbox

lengths = {
    5.5: 128,
    7.5: 64,
    9.5: 64
}

################

figs = []

for i, (vov, length) in enumerate(lengths.items()):

    ap21 = afterpulse_tile21.AfterPulseTile21(vov)

    fig = plt.figure(num=f'figpe2-{i}', clear=True, figsize=[8, 4])

    ap21.sim.hist('mainheight', f'~saturated & ~closept & (mainpos >= 0) & (length == {length})', yscale='log', fig=fig, nbins=1000, selection=False)
    ax, = fig.get_axes()
    ax.set_xlabel('Laser peak height')
    textbox.textbox(ax, f'{vov} VoV\nFilter {length} ns', fontsize='medium', loc='center right')
    ilen = np.searchsorted(ap21.sim.filtlengths, length)
    boundaries = ap21.sim.computenpeboundaries(ilen)
    afterpulse_tile21.vlines(ax, boundaries, linestyle=':')

    figs.append(fig)

for fig in figs:
    fig.tight_layout()
    fig.show()

figlatex.save(figs)


# SPDX-License-Identifier: GPL-2.0-only
"""
A module with utilities for storing information related to a specific user.
"""

import time
import collections
import logging
import itertools
import os
import pwd
import usage
import cinfo
import statuses
from cfgparser import cfg, shared

def get_whitelist(status_group):
    """
    Returns the whitelist for the status group plus the global whitelist as a
    set.
    """
    status_group_prop = statuses.lookup_status_prop(status_group)
    whlist = set(cfg.processes.whitelist)
    whlist.update(status_group_prop.whitelist)

    if cfg.processes.whitelist_other_processes:
        whlist.add(shared.other_processes_label)

    whlist_files = [
        cfg.processes.whitelist_file, status_group_prop.whitelist_file
    ]
    for wfile in whlist_files:
        if wfile and os.path.isfile(wfile):
            with open(wfile, "r") as f:
                whlist.update(item.strip() for item in f.readlines())
    return whlist

proc_owner_whitelist = set(cfg.processes.proc_owner_whitelist)
whitelist = {}
for status_group in cfg.status.order + cfg.status.penalty.order:
    whitelist[status_group] = get_whitelist(status_group)

class User:
    """
    Contains information related to an user.

    Properties:
    uid: int
        The user's uid.
    gids: [int, ]
        A list of gids that the user belongs to.
    cgroup: cinfo.UserSlice()
        A cgroup obj representing the user.
    history: collections.deque(dict, )
        A list of history events ordered chronologically (i.e. most recent is
        first). History events are formatted as:
            {"time": float,
             "mem": float,
             "cpu": float,
             "pids": {int (pid): pidinfo.StaticProcess()}}
    badness_history: collections.deque(dict, )
        A list of badness history events ordered chronologically (i.e. most
        recent is first). Badness events are formatted as:
            {"timestamp": int,
             "delta_badness": {"mem": float, "cpu": float},
             "badness": {"mem": float, "cpu": float}}
    badness_timestamp: int
        A epoch timestamp for when the user's badness score goes above zero.
        This is reset after the user's badness score goes to 0.
    status: statuses.Status()
        The status of the user. See statuses.get_status() for format.
    cpu_usage: float
        The average cpu usage of the user over the arbiter interval (as a
        percent of a single core), averaged from collector events during the
        arbiter interval.
    mem_usage: float
        The average mem usage of the user over the arbiter interval (as a
        percentage of the entire machine), averaged from collector events
        during the arbiter interval.
    cpu_quota: float
        The user's cpu quota (as a percent of a single core) based on their
        current status.
    mem_quota: float
        The user's memory quota (as a percentage of the entire machine) based
        on their current status.
    """
    __slots__ = ["uid", "gids", "cgroup", "username", "uid_name", "history",
                 "badness_history", "badness_timestamp", "status",
                 "cpu_usage", "mem_usage", "cpu_quota", "mem_quota"]

    def __init__(self, uid):
        """
        Initializes a user.

        uid: int
            The user's uid
        """
        self.uid = uid
        self.gids = statuses.query_gids(self.uid)
        self.cgroup = cinfo.UserSlice(self.uid)
        self.username = "?"
        try:
            self.username = pwd.getpwuid(uid).pw_name
        except KeyError:
            pass
        self.uid_name = "{} ({})".format(self.uid, self.username)
        self.status = statuses.get_status(uid)
        self.history = collections.deque(maxlen=cfg.badness.max_history_kept)
        self.badness_history = collections.deque(maxlen=cfg.badness.max_history_kept)
        self.badness_timestamp = 0  # epoch when badness started increasing
        self.set_badness({"cpu": 0.0, "mem": 0.0}, int(time.time()))

    def set_badness(self, badness, record_time):
        """
        Sets and clears the badness history with the given badness.

        badness: dict
            The new first badness.
        record_time: float, int
            Time in epoch that the badness was calculated.
        """
        self.badness_history.clear()
        self.badness_history.appendleft({
            "timestamp": record_time,
            "delta_badness": {"cpu": 0.0, "mem": 0.0},
            "badness": badness
        })
        if sum(badness.values()) == 0:
            self.badness_timestamp = record_time

    def add_badness(self, badness, delta_badness, record_time):
        """
        Imports new badness and prepends the new badness into the
        self.badness_history accordingly.

        badness: dict
            A new badness dictionary.
        delta_badness: dict
            The change in badness dictionary.
        record_time: float, int
            Time in epoch that the badness was calculated.
        """
        self.badness_history.appendleft({
            "timestamp": record_time,
            "delta_badness": delta_badness,
            "badness": badness
        })

        # Set badness_timestamp when user starts gaining badness or loses it
        if not self.badness_timestamp and sum(badness.values()) != 0:
            self.badness_timestamp = record_time
        elif self.badness_timestamp and sum(badness.values()) == 0:
            self.badness_timestamp = 0

    def update_properties(self):
        """
        Sets properties of the user.
        """
        self.status = statuses.get_status(self.uid)
        avg_cpu, avg_mem = self.avg_gen_usage()
        self.cpu_usage = avg_cpu
        self.mem_usage = avg_mem
        cpu_quota, mem_quota = statuses.lookup_quotas(self.uid, self.status.current)
        self.cpu_quota = cpu_quota
        self.mem_quota = mem_quota
        self.gids = statuses.query_gids(self.uid)

    def whitelisted_processes(self, processes):
        """
        Filters the given processes and into list of whitelisted processes.

        processes: iter
            A iterable of processes.
        """
        return [
            proc for proc in processes
            if proc.name.rstrip("*") in whitelist[self.status.current] or
               proc.owner in proc_owner_whitelist
        ]

    def mark_whitelisted_processes(self, processes):
        """
        Marks the given StaticProcess()s with a asterisk at the end of their
        name if the process is whitelisted either by the global whitelist,
        status whitelist or pid owner whitelist.
        """
        whitelisted_procs = self.whitelisted_processes(processes)
        for proc in whitelisted_procs:
            proc.name += "*"

    def avg_gen_usage(self):
        """
        Returns the current average cgroup usage between the arbiter
        intervals.
        """
        updates = cfg.general.history_per_refresh
        cpu_usages = [
            event["cpu"]
            for event in itertools.islice(self.history, 0, updates)
        ]
        mem_usages = [
            event["mem"]
            for event in itertools.islice(self.history, 0, updates)
        ]
        return (
            sum(cpu_usages) / len(cpu_usages) if cpu_usages else 0.0,
            sum(mem_usages) / len(mem_usages) if mem_usages else 0.0
        )

    def avg_proc_usage(self, whitelisted=False):
        """
        Returns the current average total process usage between the arbiter
        intervals.

        whitelisted: bool
            Whether to only count whitelisted processes.
        """
        updates = cfg.general.history_per_refresh
        total_procs = []
        for event in itertools.islice(self.history, 0, updates):
            if whitelisted:
                procs = self.whitelisted_processes(event["pids"].values())
            else:
                procs = list(event["pids"].values())
            total_procs.append(sum(procs))

        if not total_procs:
            return 0.0, 0.0

        avg_proc = usage.average(*total_procs)
        return avg_proc.usage["cpu"], avg_proc.usage["mem"]

    def new(self):
        """
        Returns whether the user is new (as in the obj created).
        """
        return len(self.badness_history) <= 1


# ドイツの言語クラスをインポート
from spacy.lang.de import German

# nlpオブジェクトを作成
nlp = German()

# テキストを処理（ドイツ語で「よろしく！」の意味）
doc = nlp("Liebe Grüße!")

# docのテキストをプリント
print(doc.text)


#!/usr/bin/env python3

import os
import sys
import re
import numpy as np
import json
import gzip

from pathlib import Path, PurePath
sys.path.append(str(PurePath(PurePath(Path(__file__).absolute()).parent).parent))

from TestHarness import Utils, Account
from dataclasses import dataclass, asdict, field
from platform import release, system
from datetime import datetime
from typing import List
from pathlib import Path

Print = Utils.Print
errorExit = Utils.errorExit
cmdError = Utils.cmdError

COMPLETEPRODUCTIONWINDOWSIZE = 12

@dataclass
class ArtifactPaths:
    nodeosLogPath: Path = Path("")
    trxGenLogDirPath: Path = Path("")
    blockTrxDataPath: Path = Path("")
    blockDataPath: Path = Path("")
    transactionMetricsDataPath: Path = Path("")

@dataclass
class TpsTestConfig:
    targetTps: int = 0
    testDurationSec: int = 0
    tpsLimitPerGenerator: int = 0
    numBlocksToPrune: int = 0
    numTrxGensUsed: int = 0
    targetTpsPerGenList: List[int] = field(default_factory=list)
    quiet: bool = False

@dataclass
class stats():
    min: int = 0
    max: int = 0
    avg: float = 0
    sigma: float = 0
    emptyBlocks: int = 0
    numBlocks: int = 0

@dataclass
class basicStats():
    min: float = 0
    max: float = 0
    avg: float = 0
    sigma: float = 0
    samples: int = 0

@dataclass
class trxData():
    blockNum: int = 0
    cpuUsageUs: int = 0
    netUsageUs: int = 0
    blockTime: datetime = None
    latency: float = 0
    _sentTimestamp: str = ""
    _calcdTimeEpoch: float = 0

    @property
    def sentTimestamp(self):
        return self._sentTimestamp

    @property
    def calcdTimeEpoch(self):
        return self._calcdTimeEpoch

    @sentTimestamp.setter
    def sentTimestamp(self, sentTime: str):
        self._sentTimestamp = sentTime
        # When we no longer support Python 3.6, would be great to update to use this
        # self._calcdTimeEpoch = datetime.fromisoformat(sentTime).timestamp()
        self._calcdTimeEpoch = datetime.strptime(sentTime, "%Y-%m-%dT%H:%M:%S.%f").timestamp()

    @sentTimestamp.deleter
    def sentTimestamp(self):
        self._sentTimestamp = ""
        self._calcdTimeEpoch = 0

@dataclass
class blkData():
    blockId: int = 0
    producer: str = ""
    status: str = ""
    _timestamp: str = field(init=True, repr=True, default='')
    _calcdTimeEpoch: float = 0

    def __post_init__(self):
        self.timestamp = self._timestamp

    @property
    def timestamp(self):
        return self._timestamp

    @property
    def calcdTimeEpoch(self):
        return self._calcdTimeEpoch

    @timestamp.setter
    def timestamp(self, time: str):
        self._timestamp = time[:-1]
        # When we no longer support Python 3.6, would be great to update to use this
        # self._calcdTimeEpoch = datetime.fromisoformat(time[:-1]).timestamp()
        #Note block timestamp formatted like: '2022-09-30T16:48:13.500Z', but 'Z' is not part of python's recognized iso format, so strip it off the end
        self._calcdTimeEpoch = datetime.strptime(time[:-1], "%Y-%m-%dT%H:%M:%S.%f").timestamp()

    @timestamp.deleter
    def timestamp(self):
        self._timestamp = ""
        self._calcdTimeEpoch = 0

@dataclass
class productionWindow():
    producer: str = ""
    startBlock: int = 0
    endBlock: int = 0
    blockCount: int = 0

@dataclass
class productionWindows():
    totalWindows: int = 0
    averageWindowSize: float = 0
    missedWindows: int = 0

@dataclass
class chainBlocksGuide():
    firstBlockNum: int = 0
    lastBlockNum: int = 0
    totalBlocks: int = 0
    testStartBlockNum: int = 0
    testEndBlockNum: int = 0
    setupBlocksCnt: int = 0
    tearDownBlocksCnt: int = 0
    leadingEmptyBlocksCnt: int = 0
    trailingEmptyBlocksCnt: int = 0
    configAddlDropCnt: int = 0
    testAnalysisBlockCnt: int = 0

@dataclass
class blockData():
    partialBlockId: str = ""
    blockNum: int = 0
    transactions: int = 0
    net: int = 0
    cpu: int = 0
    elapsed: int = 0
    time: int = 0
    latency: int = 0

class chainData():
    def __init__(self):
        self.blockLog = []
        self.startBlock = None
        self.ceaseBlock = None
        self.totalTransactions = 0
        self.totalNet = 0
        self.totalCpu = 0
        self.totalElapsed = 0
        self.totalTime = 0
        self.totalLatency = 0
        self.droppedBlocks = {}
        self.forkedBlocks = []
    def __eq__(self, other):
        return self.startBlock == other.startBlock and\
         self.ceaseBlock == other.ceaseBlock and\
         self.totalTransactions == other.totalTransactions and\
         self.totalNet == other.totalNet and\
         self.totalCpu == other.totalCpu and\
         self.totalElapsed == other.totalElapsed and\
         self.totalTime == other.totalTime and\
         self.totalLatency == other.totalLatency
    def updateTotal(self, transactions, net, cpu, elapsed, time, latency):
        self.totalTransactions += transactions
        self.totalNet += net
        self.totalCpu += cpu
        self.totalElapsed += elapsed
        self.totalTime += time
        self.totalLatency += latency
    def __str__(self):
        return (f"Starting block: {self.startBlock}\nEnding block:{self.ceaseBlock}\nChain transactions: {self.totalTransactions}\n"
         f"Chain cpu: {self.totalCpu}\nChain net: {(self.totalNet / (self.ceaseBlock - self.startBlock + 1))}\nChain elapsed: {self.totalElapsed}\n"
         f"Chain time: {self.totalTime}\nChain latency: {self.totalLatency}")
    def assertEquality(self, other):
        assert self == other, f"Error: Actual log:\n{self}\ndid not match expected log:\n{other}"

def selectedOpen(path):
    return gzip.open if path.suffix == '.gz' else open

def scrapeLog(data: chainData, path):
    #node_00/stderr.txt
    selectedopen = selectedOpen(path)
    with selectedopen(path, 'rt') as f:
        line = f.read()
        blockResult = re.findall(r'Received block ([0-9a-fA-F]*).* #(\d+) .*trxs: (\d+)(.*)', line)
        if data.startBlock is None:
            data.startBlock = 2
        if data.ceaseBlock is None:
            data.ceaseBlock = len(blockResult) + 1
        for value in blockResult:
            v3Logging = re.findall(r'net: (\d+), cpu: (\d+), elapsed: (\d+), time: (\d+), latency: (-?\d+) ms', value[3])
            if v3Logging:
                data.blockLog.append(blockData(value[0], int(value[1]), int(value[2]), int(v3Logging[0][0]), int(v3Logging[0][1]), int(v3Logging[0][2]), int(v3Logging[0][3]), int(v3Logging[0][4])))
                if int(value[1]) in range(data.startBlock, data.ceaseBlock + 1):
                    data.updateTotal(int(value[2]), int(v3Logging[0][0]), int(v3Logging[0][1]), int(v3Logging[0][2]), int(v3Logging[0][3]), int(v3Logging[0][4]))
            else:
                v2Logging = re.findall(r'latency: (-?\d+) ms', value[3])
                if v2Logging:
                    data.blockLog.append(blockData(value[0], int(value[1]), int(value[2]), 0, 0, 0, 0, int(v2Logging[0])))
                    if int(value[1]) in range(data.startBlock, data.ceaseBlock + 1):
                        data.updateTotal(int(value[2]), 0, 0, 0, 0, int(v2Logging[0]))
                else:
                    print("Error: Unknown log format")
        droppedBlocks = re.findall(r'dropped incoming block #(\d+) id: ([0-9a-fA-F]+)', line)
        for block in droppedBlocks:
            data.droppedBlocks[block[0]] = block[1]
        forks = re.findall(r'switching forks from ([0-9a-fA-F]+) \(block number (\d+)\) to ([0-9a-fA-F]+) \(block number (\d+)\)', line)
        for fork in forks:
            data.forkedBlocks.append(int(fork[1]) - int(fork[3]) + 1)

def scrapeTrxGenLog(trxSent, path):
    #trxGenLogs/trx_data_output_*.txt
    selectedopen = selectedOpen(path)
    with selectedopen(path, 'rt') as f:
        trxSent.update(dict([(x[0], x[1]) for x in (line.rstrip('\n').split(',') for line in f)]))

def scrapeBlockTrxDataLog(trxDict, path, nodeosVers):
    #blockTrxData.txt
    selectedopen = selectedOpen(path)
    with selectedopen(path, 'rt') as f:
        if nodeosVers == "v2":
            trxDict.update(dict([(x[0], trxData(blockNum=x[1], cpuUsageUs=x[2], netUsageUs=x[3])) for x in (line.rstrip('\n').split(',') for line in f)]))
        else:
            trxDict.update(dict([(x[0], trxData(blockNum=x[1], blockTime=x[2], cpuUsageUs=x[3], netUsageUs=x[4])) for x in (line.rstrip('\n').split(',') for line in f)]))

def scrapeBlockDataLog(blockDict, path):
    #blockData.txt
    selectedopen = selectedOpen(path)
    with selectedopen(path, 'rt') as f:
        blockDict.update(dict([(x[0], blkData(x[1], x[2], x[3], x[4])) for x in (line.rstrip('\n').split(',') for line in f)]))

def scrapeTrxGenTrxSentDataLogs(trxSent, trxGenLogDirPath, quiet):
    filesScraped = []
    for fileName in trxGenLogDirPath.glob("trx_data_output_*.txt"):
        filesScraped.append(fileName)
        scrapeTrxGenLog(trxSent, fileName)

    if not quiet:
        print(f"Transaction Log Files Scraped: {filesScraped}")

def populateTrxSentTimestamp(trxSent: dict, trxDict: dict, notFound):
    for sentTrxId in trxSent.keys():
        if sentTrxId in trxDict.keys():
            trxDict[sentTrxId].sentTimestamp = trxSent[sentTrxId]
        else:
            notFound.append(sentTrxId)

def populateTrxLatencies(blockDict: dict, trxDict: dict):
    for trxId, data in trxDict.items():
        if data.calcdTimeEpoch != 0:
            trxDict[trxId].latency = blockDict[data.blockNum].calcdTimeEpoch - data.calcdTimeEpoch

def writeTransactionMetrics(trxDict: dict, path):
    with open(path, 'wt') as transactionMetricsFile:
        transactionMetricsFile.write("TransactionId,BlockNumber,BlockTime,CpuUsageUs,NetUsageUs,Latency,SentTimestamp,CalcdTimeEpoch\n")
        for trxId, data in trxDict.items():
            transactionMetricsFile.write(f"{trxId},{data.blockNum},{data.blockTime},{data.cpuUsageUs},{data.netUsageUs},{data.latency},{data._sentTimestamp},{data._calcdTimeEpoch}\n")

def getProductionWindows(prodDict: dict, blockDict: dict, data: chainData):
    prod = ""
    count = 0
    blocksFromCurProd = 0
    numProdWindows = 0
    for k, v in blockDict.items():
        count += 1
        if prod == "":
            prod = v.producer
        if prod != v.producer or count+data.startBlock == data.ceaseBlock:
            prodDict[str(numProdWindows)] = productionWindow(prod, count-blocksFromCurProd+data.startBlock-1, count+data.startBlock-2, blocksFromCurProd)
            prod = v.producer
            blocksFromCurProd = 1
            numProdWindows += 1
        else:
            blocksFromCurProd += 1
    return prodDict

def calcProductionWindows(prodDict: dict):
    prodWindows = productionWindows()
    totalBlocksForAverage = 0
    for i, (k, v) in enumerate(prodDict.items()):
        if v.blockCount == COMPLETEPRODUCTIONWINDOWSIZE:
            prodWindows.totalWindows += 1
            totalBlocksForAverage += v.blockCount
        else:
            #First and last production windows are possibly incomplete but
            #should not count against total or missed windows
            if i != 0 and i != len(prodDict)-1:
                prodWindows.missedWindows += 1
                totalBlocksForAverage += v.blockCount
                prodWindows.totalWindows += 1
    if prodWindows.totalWindows > 0:
        prodWindows.averageWindowSize = totalBlocksForAverage / prodWindows.totalWindows
    return prodWindows

def calcChainGuide(data: chainData, numAddlBlocksToDrop=0) -> chainBlocksGuide:
    """Calculates guide to understanding key points/blocks in chain data. In particular, test scenario phases like setup, teardown, etc.

    This includes breaking out 3 distinct ranges of blocks from the total block data log:
    1) Blocks during test scenario setup and tear down
    2) Empty blocks during test scenario ramp up and ramp down
    3) Additional blocks - potentially partially full blocks while test scenario ramps up to steady state

    Keyword arguments:
    data -- the chainData for the test run.  Includes blockLog, startBlock, and ceaseBlock
    numAddlBlocksToDrop -- num potentially non-empty blocks to ignore at beginning and end of test for steady state purposes

    Returns:
    chain guide describing key blocks and counts of blocks to describe test scenario
    """
    firstBN = data.blockLog[0].blockNum
    lastBN = data.blockLog[-1].blockNum
    total = len(data.blockLog)
    testStartBN = data.startBlock
    testEndBN = data.ceaseBlock

    setupCnt = 0
    if data.startBlock is not None:
        setupCnt = data.startBlock - firstBN

    tearDownCnt = 0
    if data.ceaseBlock is not None:
        tearDownCnt = lastBN - data.ceaseBlock

    leadingEmpty = 0
    for le in range(setupCnt, total - tearDownCnt - 1):
        if data.blockLog[le].transactions == 0:
            leadingEmpty += 1
        else:
            break

    trailingEmpty = 0
    for te in range(total - tearDownCnt - 1, setupCnt + leadingEmpty, -1):
        if data.blockLog[te].transactions == 0:
            trailingEmpty += 1
        else:
            break

    testAnalysisBCnt = total - setupCnt - tearDownCnt - leadingEmpty - trailingEmpty - ( 2 * numAddlBlocksToDrop )
    testAnalysisBCnt = 0 if testAnalysisBCnt < 0 else testAnalysisBCnt

    return chainBlocksGuide(firstBN, lastBN, total, testStartBN, testEndBN, setupCnt, tearDownCnt, leadingEmpty, trailingEmpty, numAddlBlocksToDrop, testAnalysisBCnt)

def pruneToSteadyState(data: chainData, guide: chainBlocksGuide):
    """Prunes the block data log down to range of blocks when steady state has been reached.

    This includes pruning out 3 distinct ranges of blocks from the total block data log:
    1) Blocks during test scenario setup and tear down
    2) Empty blocks during test scenario ramp up and ramp down
    3) Additional blocks - potentially partially full blocks while test scenario ramps up to steady state

    Keyword arguments:
    data -- the chainData for the test run.  Includes blockLog, startBlock, and ceaseBlock
    guide -- chain guiderails calculated over chain data to guide interpretation of whole run's block data

    Returns:
    pruned list of blockData representing steady state operation
    """

    return data.blockLog[guide.setupBlocksCnt + guide.leadingEmptyBlocksCnt + guide.configAddlDropCnt:-(guide.tearDownBlocksCnt + guide.trailingEmptyBlocksCnt + guide.configAddlDropCnt)]

def scoreTransfersPerSecond(data: chainData, guide: chainBlocksGuide) -> stats:
    """Analyzes a test scenario's steady state block data for statistics around transfers per second over every two-consecutive-block window"""
    prunedBlockDataLog = pruneToSteadyState(data, guide)

    blocksToAnalyze = len(prunedBlockDataLog)
    if blocksToAnalyze == 0:
        return stats()
    elif blocksToAnalyze == 1:
        onlyBlockTrxs = prunedBlockDataLog[0].transactions
        return stats(onlyBlockTrxs, onlyBlockTrxs, onlyBlockTrxs, 0, int(onlyBlockTrxs == 0), 1)
    else:
        # Calculate the num trxs in each two-consecutive-block window and count any empty blocks in range.
        # for instance: given 4 blocks [1, 2, 3, 4], the two-consecutive-block windows analyzed would be [(1,2),(2,3),(3,4)]
        consecBlkTrxsAndEmptyCnt = [(first.transactions + second.transactions, int(first.transactions == 0)) for first, second in zip(prunedBlockDataLog, prunedBlockDataLog[1:])]

        npCBTAEC = np.array(consecBlkTrxsAndEmptyCnt, dtype=np.uint)

        # Note: numpy array slicing in use -> [:,0] -> from all elements return index 0
        return stats(int(np.min(npCBTAEC[:,0])), int(np.max(npCBTAEC[:,0])), float(np.average(npCBTAEC[:,0])), float(np.std(npCBTAEC[:,0])), int(np.sum(npCBTAEC[:,1])), len(prunedBlockDataLog))

def calcBlockSizeStats(data: chainData, guide : chainBlocksGuide) -> stats:
    """Analyzes a test scenario's steady state block data for block size statistics during the test window"""
    prunedBlockDataLog = pruneToSteadyState(data, guide)

    blocksToAnalyze = len(prunedBlockDataLog)
    if blocksToAnalyze == 0:
        return stats()
    elif blocksToAnalyze == 1:
        onlyBlockNetSize = prunedBlockDataLog[0].net
        return stats(onlyBlockNetSize, onlyBlockNetSize, onlyBlockNetSize, 0, int(onlyBlockNetSize == 0), 1)
    else:
        blockSizeList = [(blk.net, int(blk.net == 0)) for blk in prunedBlockDataLog]

        npBlkSizeList = np.array(blockSizeList, dtype=np.uint)

        # Note: numpy array slicing in use -> [:,0] -> from all elements return index 0
        return stats(int(np.min(npBlkSizeList[:,0])), int(np.max(npBlkSizeList[:,0])), float(np.average(npBlkSizeList[:,0])), float(np.std(npBlkSizeList[:,0])), int(np.sum(npBlkSizeList[:,1])), len(prunedBlockDataLog))

def calcTrxLatencyCpuNetStats(trxDict : dict, blockDict: dict):
    """Analyzes a test scenario's steady state block data for transaction latency statistics during the test window

    Keyword arguments:
    trxDict -- the dictionary mapping trx id to trxData, wherein the trx sent timestamp has been populated from the trx generator at moment of send
    blockDict -- the dictionary of block number to blockData, wherein the block production timestamp is recorded

    Returns:
    transaction latency stats as a basicStats object
    """
    trxLatencyCpuNetList = [(data.latency, data.cpuUsageUs, data.netUsageUs) for trxId, data in trxDict.items() if data.calcdTimeEpoch != 0]

    npLatencyCpuNetList = np.array(trxLatencyCpuNetList, dtype=float)

    return basicStats(float(np.min(npLatencyCpuNetList[:,0])), float(np.max(npLatencyCpuNetList[:,0])), float(np.average(npLatencyCpuNetList[:,0])), float(np.std(npLatencyCpuNetList[:,0])), len(npLatencyCpuNetList)), \
           basicStats(float(np.min(npLatencyCpuNetList[:,1])), float(np.max(npLatencyCpuNetList[:,1])), float(np.average(npLatencyCpuNetList[:,1])), float(np.std(npLatencyCpuNetList[:,1])), len(npLatencyCpuNetList)), \
           basicStats(float(np.min(npLatencyCpuNetList[:,2])), float(np.max(npLatencyCpuNetList[:,2])), float(np.average(npLatencyCpuNetList[:,2])), float(np.std(npLatencyCpuNetList[:,2])), len(npLatencyCpuNetList))

def createReport(guide: chainBlocksGuide, tpsTestConfig: TpsTestConfig, tpsStats: stats, blockSizeStats: stats, trxLatencyStats: basicStats, trxCpuStats: basicStats,
                 trxNetStats: basicStats, forkedBlocks, droppedBlocks, prodWindows: productionWindows, notFound: dict, testStart: datetime, testFinish: datetime,
                 argsDict: dict, completedRun: bool, nodeosVers: str) -> dict:
    report = {}
    report['completedRun'] = completedRun
    report['testStart'] = testStart
    report['testFinish'] = testFinish
    report['Analysis'] = {}
    report['Analysis']['BlockSize'] = asdict(blockSizeStats)
    report['Analysis']['BlocksGuide'] = asdict(guide)
    report['Analysis']['TPS'] = asdict(tpsStats)
    report['Analysis']['TPS']['configTps'] = tpsTestConfig.targetTps
    report['Analysis']['TPS']['configTestDuration'] = tpsTestConfig.testDurationSec
    report['Analysis']['TPS']['tpsPerGenerator'] = tpsTestConfig.targetTpsPerGenList
    report['Analysis']['TPS']['generatorCount'] = tpsTestConfig.numTrxGensUsed
    report['Analysis']['TrxCPU'] = asdict(trxCpuStats)
    report['Analysis']['TrxLatency'] = asdict(trxLatencyStats)
    report['Analysis']['TrxNet'] = asdict(trxNetStats)
    report['Analysis']['DroppedBlocks'] = droppedBlocks
    report['Analysis']['DroppedBlocksCount'] = len(droppedBlocks)
    report['Analysis']['DroppedTransactions'] = len(notFound)
    report['Analysis']['ProductionWindowsTotal'] = prodWindows.totalWindows
    report['Analysis']['ProductionWindowsAverageSize'] = prodWindows.averageWindowSize
    report['Analysis']['ProductionWindowsMissed'] = prodWindows.missedWindows
    report['Analysis']['ForkedBlocks'] = forkedBlocks
    report['Analysis']['ForksCount'] = len(forkedBlocks)
    report['args'] =  argsDict
    report['env'] = {'system': system(), 'os': os.name, 'release': release(), 'logical_cpu_count': os.cpu_count()}
    report['nodeosVersion'] = nodeosVers
    return report

class LogReaderEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.isoformat()
        if isinstance(obj, PurePath):
            return str(obj)
        if obj is None:
            return "Unknown"
        if isinstance(obj, Path):
            return str(obj)
        if isinstance(obj, Account):
            return str(obj)
        defaultStr = ""
        try:
            defaultStr = json.JSONEncoder.default(self, obj)
        except TypeError as err:
            defaultStr = f"ERROR: {str(err)}"
        return defaultStr

def reportAsJSON(report: dict) -> json:
    return json.dumps(report, indent=2, cls=LogReaderEncoder)

def calcAndReport(data: chainData, tpsTestConfig: TpsTestConfig, artifacts: ArtifactPaths, argsDict: dict, testStart: datetime=None, completedRun: bool=True, nodeosVers: str="") -> dict:
    scrapeLog(data, artifacts.nodeosLogPath)

    trxSent = {}
    scrapeTrxGenTrxSentDataLogs(trxSent, artifacts.trxGenLogDirPath, tpsTestConfig.quiet)

    trxDict = {}
    scrapeBlockTrxDataLog(trxDict, artifacts.blockTrxDataPath, nodeosVers)

    blockDict = {}
    scrapeBlockDataLog(blockDict, artifacts.blockDataPath)

    notFound = []
    populateTrxSentTimestamp(trxSent, trxDict, notFound)

    prodDict = {}
    getProductionWindows(prodDict, blockDict, data)

    if len(notFound) > 0:
        print(f"Transactions logged as sent but NOT FOUND in block!! lost {len(notFound)} out of {len(trxSent)}")
        if argsDict.get("printMissingTransactions"):
            print(notFound)

    populateTrxLatencies(blockDict, trxDict)
    writeTransactionMetrics(trxDict, artifacts.transactionMetricsDataPath)
    guide = calcChainGuide(data, tpsTestConfig.numBlocksToPrune)
    trxLatencyStats, trxCpuStats, trxNetStats = calcTrxLatencyCpuNetStats(trxDict, blockDict)
    tpsStats = scoreTransfersPerSecond(data, guide)
    blkSizeStats = calcBlockSizeStats(data, guide)
    prodWindows = calcProductionWindows(prodDict)

    if not tpsTestConfig.quiet:
        print(f"Blocks Guide: {guide}\nTPS: {tpsStats}\nBlock Size: {blkSizeStats}\nTrx Latency: {trxLatencyStats}\nTrx CPU: {trxCpuStats}\nTrx Net: {trxNetStats}")

    start = None
    finish = None
    if testStart is not None:
        start = testStart
        finish = datetime.utcnow()

    report = createReport(guide=guide, tpsTestConfig=tpsTestConfig, tpsStats=tpsStats, blockSizeStats=blkSizeStats, trxLatencyStats=trxLatencyStats,
                          trxCpuStats=trxCpuStats, trxNetStats=trxNetStats, forkedBlocks=data.forkedBlocks, droppedBlocks=data.droppedBlocks,
                          prodWindows=prodWindows, notFound=notFound, testStart=start, testFinish=finish, argsDict=argsDict, completedRun=completedRun,
                          nodeosVers=nodeosVers)
    return report

def exportReportAsJSON(report: json, exportPath):
    with open(exportPath, 'wt') as f:
        f.write(report)


from typing import List, Union
from .payloadObject import PayLoadObject, PayLoadStatus
from ...organizer.source import SourceObject
from gailbot.core.utils.general import (
    get_extension,
    copy)
from gailbot.core.utils.logger import makelogger
from gailbot.core.utils.media import AudioHandler
import os
from gailbot.workspace.manager import WorkspaceManager
from gailbot.configs import service_config_loader

SUPPORTED_AUDIO = service_config_loader().engines.audio_supported_format
MERGED_FILE_NAME = "merged"

logger = makelogger("audioPayload")

def load_audio_payload(
    source: SourceObject, ws_manager: WorkspaceManager) -> Union[bool, List[PayLoadObject]]:
    """ given a source object, convert it into an audio payload if the source

    Args:
        source (SourceObject): an instance of SourceObject that stores the
        datafile and setting of the transcription

    Returns:
        Union[bool, List[PayLoadObject]]: return the converted payload if the
        conversion is successful, return false other wise
    """
    if not source.setting:
        return False
    if not AudioPayload.is_supported(source.source_path()):
        return False
    try:
        return [AudioPayload(source, ws_manager)]
    except Exception as e:
        logger.error(source.__class__)
        logger.error(e, exc_info=e)
        return False

class AudioPayload(PayLoadObject):
    """
    Class for audio payload
    """
    def __init__(self, source: SourceObject, workspace: WorkspaceManager) -> None:
        super().__init__(source, workspace)

    @staticmethod
    def is_supported(file_path: str) -> bool:
        """
        Determines if a given file path has a supported file extension
        """
        logger.info(file_path)
        return get_extension(file_path) in SUPPORTED_AUDIO

    def _set_initial_status(self) -> None:
        """
        Sets the initial status of the payload object to initialized
        """
        self.status = PayLoadStatus.INITIALIZED

    def _copy_file(self) -> None:
        """
        Copies file to workspace
        """
        extension = get_extension(self.original_source)
        tgt_path =os.path.join(
            self.workspace.data_copy, f"{self.name}.{extension}")
        copy(self.original_source, tgt_path)
        self.data_files = [tgt_path]

    def _merge_audio(self):
        try:
            handler = AudioHandler()
            merged_path = handler.overlay_audios(self.data_files, self.out_dir.media_file, MERGED_FILE_NAME)
            self.merged_audio = merged_path
            assert merged_path
        except Exception as e:
            logger.error(e, exc_info=e)

    @staticmethod
    def supported_format() -> List[str]:
        """
        Contains and accesses a list of the supported formats
        """
        return SUPPORTED_AUDIO

    def __repr__(self) -> str:
        return "Audio payload"


from numpy import *
from UDFManager import *
import matplotlib.pyplot as plt

number=range(1,11)
number_face=[]
calc_time0=[]
calc_time1=[]
for i in number:
	udf=UDFManager("./line%02d_o.udf"%(i))
	f=udf.size("input.face[]")
	t=udf.get("output.time")
	number_face.append(f)
	calc_time0.append(t)
for i in number:
	udf=UDFManager("./line%02d_fo.udf"%(i))
	f=udf.size("input.face[]")
	t=udf.get("output.time")
	calc_time1.append(t)
fig = plt.figure(figsize=(12.0, 9.0))
ax=fig.add_subplot()
ax.plot(number_face,calc_time0,marker='o',markersize=5,linestyle='-',color='blue')
ax.plot(number_face,calc_time1,marker='o',markersize=5,linestyle='-',color='red')
ax.set_xlabel("number of faces")
ax.set_ylabel("calculation time")
plt.tight_layout()
plt.savefig("calculation_time.png")
plt.show()


import math

import numpy as np
import pandas as pd
import scipy.interpolate
import plotly.express as px
import plotly.graph_objects as go
import matplotlib.pyplot as plt
from matplotlib import animation
from colorcet import glasbey
import sklearn.datasets
import umap
import umap.plot
import umap.utils as utils
import umap.aligned_umap
from utils import load_embedded_data
from utils.embedding import SentenceTransformerBackend
from utils.topics import FrankenTopic, UMAPArgs, VectorizerArgs, TSNEArgs, KMeansArgs, HDBSCANArgs

TILE_SIZE = 5

def axis_bounds(embedding):
    left, right = embedding.T[0].min(), embedding.T[0].max()
    bottom, top = embedding.T[1].min(), embedding.T[1].max()
    adj_h, adj_v = (right - left) * 0.1, (top - bottom) * 0.1
    return [left - adj_h, right + adj_h, bottom - adj_v, top + adj_v]

def plot_series(mapper_):
    N_COLS = 3
    num_rows = math.ceil(len(mapper_.embeddings_) / N_COLS)

    fig, axs = plt.subplots(num_rows, N_COLS, figsize=(TILE_SIZE * N_COLS, TILE_SIZE * num_rows))
    ax_bound = axis_bounds(np.vstack(mapper_.embeddings_))
    for i, ax in enumerate(axs.flatten()):
        if i >= len(mapper_.embeddings_):  # skip last one if uneven number
            continue
        # current_target = ordered_target[150 * i:min(ordered_target.shape[0], 150 * i + 400)]
        len_prev = 0 if i == 0 else len(mapper_.embeddings_[i - 1])
        cmap = plt.get_cmap('tab10').colors
        ax.scatter(*mapper_.embeddings_[i].T,
                   c=[cmap[0] if j < len_prev else cmap[1] for j in range(len(mapper_.embeddings_[i]))],
                   s=2, cmap="tab10")  # c=current_target,
        ax.axis(ax_bound)
        ax.set(xticks=[], yticks=[])

    plt.tight_layout()

def plot_animation(mapper_: umap.AlignedUMAP, frames_per_slice=20):
    n_slices = len(mapper_.embeddings_)
    ax_bound = axis_bounds(np.vstack(mapper_.embeddings_))

    all_frame_layouts = []

    for slice_i in range(n_slices - 1):
        slice_j = slice_i + 1
        layout_i = mapper_.embeddings_[slice_i]
        layout_j = mapper_.embeddings_[slice_j]
        mapping = mapper_.dict_relations_[slice_i]

        # mask for all the things included in the next slice
        mask = np.array([idx in mapping for idx in range(len(layout_i))], dtype=bool)

        from_layout = np.vstack([layout_i[list(mapping.keys())],
                                 layout_i[~mask]])
        to_layout = np.vstack([layout_j[list(mapping.values())],
                               layout_i[~mask]])

        z = np.linspace(0, 1.0, frames_per_slice)
        xs = scipy.interpolate.interp1d(np.linspace(0, 1.0, 2),
                                        np.vstack([from_layout[:, 0], to_layout[:, 0]]).T)(z)
        ys = scipy.interpolate.interp1d(np.linspace(0, 1.0, 2),
                                        np.vstack([from_layout[:, 1], to_layout[:, 1]]).T)(z)

        frame_layouts = np.array([xs, ys]).T
        all_frame_layouts.append(frame_layouts)

    all_frame_layouts.append(np.array([mapper_.embeddings_[-1] for _ in range(frames_per_slice)]))

    fig = plt.figure(figsize=(TILE_SIZE, TILE_SIZE), dpi=150)
    ax = fig.add_subplot(1, 1, 1)

    scat = ax.scatter([], [], s=2)
    text = ax.text(ax_bound[0] + 0.5, ax_bound[2] + 0.5, '')
    ax.axis(ax_bound)
    ax.set(xticks=[], yticks=[])
    plt.tight_layout()

    num_frames = frames_per_slice * len(all_frame_layouts)

    def animate(i):
        slice_k = i // frames_per_slice
        slice_l = i % frames_per_slice
        scat.set_offsets(all_frame_layouts[slice_k][slice_l])
        text.set_text(f'Frame {i} ({slice_k}–{slice_l})')
        return scat

    anim = animation.FuncAnimation(
        fig,
        init_func=None,
        func=animate,
        frames=num_frames,
        interval=40)
    anim.save('data/alumap.gif', writer="pillow")
    plt.close(anim._fig)

if __name__ == '__main__':
    limit = 10000

    embedding_backend = SentenceTransformerBackend
    embedding_model = 'paraphrase-multilingual-MiniLM-L12-v2'

    tweets, embeddings = load_embedded_data(cache_dir='data/',
                                            db_file='data/identifier.sqlite',
                                            limit=limit,
                                            remove_urls=True,
                                            remove_nonals=True,
                                            remove_hashtags=True,
                                            remove_mentions=True,
                                            backend=embedding_backend,
                                            model=embedding_model)

    print('Sorting...')
    # data = list(sorted(zip(tweets.tweets, embeddings), key=lambda x: x[0].date))
    data = list(zip(tweets.tweets, embeddings))
    N_SLICES = 6
    SLICE_SIZE = math.ceil(len(data) / N_SLICES)

    print('Slicing...')
    # two modes:
    #  a) data always lives on
    #  b) data decays (only some overlap between slices)
    MODE = 'a'
    if MODE == 'a':
        slices = [data[:(i + 1) * SLICE_SIZE] for i in range(N_SLICES)]
        sliced_embeddings = [np.array([e for _, e in s]) for s in slices]
    else:
        # FIXME: no overlap yet
        slices = [data[i * SLICE_SIZE:(i + 1) * SLICE_SIZE] for i in range(N_SLICES)]
        sliced_embeddings = [np.array([e for _, e in s]) for s in slices]

    print('Building relations dict')
    relation_dicts = [{i: i for i in range(len(sliced_embeddings[si]))}
                      for si in range(N_SLICES - 1)]

    print('Running UMAP...')
    alumap = umap.AlignedUMAP(verbose=True,
                              alignment_window_size=2,
                              alignment_regularisation=0.001,
                              n_neighbors=50,
                              min_dist=0.5,
                              metric='cosine')
    mapper = alumap.fit(sliced_embeddings, relations=relation_dicts)

    print('Plotting...')
    plot_series(mapper)
    plt.show()
    plt.close()

    plot_animation(mapper, frames_per_slice=30)


import gym
import numpy as np
import random
import matplotlib.pyplot as plt

env = gym.make('MountainCar-v0')

# Initialize q-table values to 0
discretization = 15
pos_discretization = 15
Q = np.zeros((pos_discretization,discretization, env.action_space.n))    # (states_space, action_space)

# Initialize hyperparameters
gamma = 0.98         # Discount factor used to balance immediate and future reward.
lr = 0.1            # learning rate
epsilon = 0.9         # rate of exploration where 1 is completely random and 0 is deterministic
epsilon_decay = 0.02
# Initialize list for plotting running mean
rewards = []
N = 100     # Number of episodes for the running mean

def running_mean(x, N):
    """function for creating list with running mean"""
    cumsum = np.cumsum(np.insert(x, 0, 0))
    return (cumsum[N:] - cumsum[:-N]) / float(N)

def get_state(cart_position, tip_speed):
    """Returns index of the given speed state in the Q-table"""
    positions = np.linspace(-1.2,0.6,num=pos_discretization)
    states = np.linspace(-0.07,0.07,num=discretization)   # Discretized table of speeds.
    for rounded_cart_position in positions:
        if rounded_cart_position >= cart_position:
            cart_index = np.where(positions == rounded_cart_position)[0][0]
            for rounded_tip_speed in states:
                if rounded_tip_speed >= tip_speed:
                    index = np.where(states == rounded_tip_speed)
                    index = index[0][0]
                    return cart_index, index
    print("NONE",cart_position,cart_index)
    print(str(index))
    return None

def better_action(cart_position, tip_speed):
    """Returns index of best action according to Q-table for a given tip speed"""
    possible_actions_in_state = Q[get_state(cart_position, tip_speed)]
    action_of_choice = np.argmax(possible_actions_in_state)
    return action_of_choice

def update_q(state, action, new_state, cart_pos, new_cart_pos, reward=1):
    Q[cart_pos, state, action] = Q[cart_pos, state, action] + lr * (reward + gamma * np.max(Q[new_cart_pos, new_state, :]) - Q[cart_pos, state, action])

for i_episode in range(5000):
    observation = env.reset()
    episode_reward = 0

    if 4900 < i_episode :
        # Drop exploration
        epsilon = 0

    if i_episode%100 == 0 and i_episode != 0:
        #Give some feedback for improvement of the algorithm
        print(i_episode)
        print("best run was", max(rewards[-99:]))

    for timestep in range(500):
        if i_episode > 4900:
            env.render()
            pass

        #if len(rewards) > 1:

            #if episode_reward >= max(rewards) and episode_reward > -150:
                #Implementing dynamic epsilon so that exploration decays over time.
             #   epsilon -= 0.006

        tip_speed = observation[1]
        cart_position = observation[0]

        if random.uniform(0, 1) < epsilon:      # Chooses random action often if epsilon is high
            action = env.action_space.sample()

        else: #Chooses action based on previous exploration
            action = better_action(cart_position, tip_speed)

        # Gather information about changes in environment
        cart_state, state = get_state(cart_position,tip_speed)
        observation, reward, done, info = env.step(action)
        new_tip_speed = observation[1]
        new_cart_pos = observation[0]
        new_cart_state, new_state = get_state(new_cart_pos, new_tip_speed)

        # Update Q-table
        update_q(state, action, new_state, cart_state, new_cart_state, reward)

        episode_reward += reward

        if done:
            #print("Episode finished after {} timesteps".format(timestep+1))
            break
    if i_episode > 0 & i_episode < 4900:
        epsilon = epsilon - epsilon/i_episode
    rewards.append(episode_reward)
print(Q)
print(epsilon)
env.render()
env.close()

print(rewards)
plt.plot(running_mean(rewards,N))
plt.show()


import matplotlib.pyplot as plt
import numpy as np

delay = 0.000001

def displaytrajectory(particles, bestP, gt, plot_all=True, disp=True):
    prts = np.array([[p.x, p.y, p.weight] for p in particles])
    prts[:, 2] /= prts[:, 2].sum()

    if plot_all:
        plt.scatter(prts[:, 0], prts[:, 1], c=prts[:, 2], cmap='gray', s=10, edgecolors='grey')

    plt.plot(gt[0], gt[1], 'g*')
    plt.plot(bestP.x, bestP.y, 'c*')
    # locate the legend at top left corner
    plt.legend((['GT', 'Estimation']), loc=2)

    if disp:
        plt.pause(delay)


from typing import Callable, Union, List
import pandas as pd

def groupby_two_df(
    df1: pd.DataFrame,
    df2: pd.DataFrame,
    f: Callable,
    df1_group_colname: str = "group",
    df2_group_colname: str = "group",
    **kwargs
) -> Union[pd.Series, pd.DataFrame]:
    """Apply split apply combine to two pandas DataFrames

    Args:
        df1 (pd.DataFrame): First DataFrame
        df2 (pd.DataFrame): Second DataFrame
        f (Callable): Callable that takes two DataFrames as input
        df1_group_colname (str, optional): Label of the group column in df1. Defaults to "group".
        df2_group_colname (str, optional): Label of the group column in df2. Defaults to "group".

    Returns:
        Union[pd.Series, pd.DataFrame]: The results of the group by
    """
    groups_df1 = set(
        df1[df1_group_colname].unique()
    ).copy()  # so not updating the original dataframes
    groups_df2 = set(
        df2[df2_group_colname].unique()
    ).copy()  # so not updating the original dataframes
    groups_shared = groups_df1.intersection(groups_df2)
    results: List[pd.DataFrame] = []
    for group in groups_shared:
        df1_sub = df1.loc[df1[df1_group_colname] == group].copy()
        df2_sub = df2.loc[df2[df2_group_colname] == group].copy()
        group_result = f(df1_sub, df2_sub, **kwargs)
        group_result["group"] = group
        results.append(group_result)
    return pd.concat(results)


from sys import argv
import csv
import os
from typing import Union, Literal, Iterable
import re

_files = {}

# These are the labels we want to use for our classification. The labels are taken from the original data set.
_desired_labels = [
    ["Name_Calling,Labeling"],
    ["Exaggeration,Minimisation"],
    ["Doubt", "Appeal_to_fear-prejudice"],
    ["Black-and-White_Fallacy"],
    ["Straw_Men", "Whataboutism", "Red_Herring"],
    ["Bandwagon"],
    ["Flag-Waving"],
    ["Repetition", "Slogans"]
]

def main(args):
    """
    Main function of the program. Takes a path as an argument and extracts data from all tsv files in that directory.
    :param args: path to the directory containing the tsv files
    :return: None
    """
    if len(args) != 2:
        print("usage: python data_extractor.py <path>")
        exit(1)

    doc_data: list = []

    dir_name = args[1]

    try:
        # get all tsv files in the given directory
        for file in os.listdir(dir_name):
            if file.endswith(".tsv"):
                # extract id from file name. using this re is dangerous if names are not consistent
                doc_data.extend(process_file(dir_name, file))
    except NotADirectoryError or NotImplementedError as e:
        print("Invalid path. Make sure it is a valid directory.")
        print(e)
        exit(1)
    except FileNotFoundError as e:
        print("File not found. Make sure the path is correct and that the file exists.")
        print(e)
        exit(1)

    # write data to file
    with open("data.csv", "w") as f:
        writer = csv.writer(f)
        writer.writerow(["document_id", "label", "text"])
        writer.writerows(doc_data)

    # collect labels
    # collect_labels(doc_data)

    # use desired labels to create a "collapsed" dataset where each line has the text and then several 0 or 1 flags for
    # each label group
    collapsed_data = {}
    for _, label, text in doc_data:
        if text not in collapsed_data:
            collapsed_data[text] = [0] * len(_desired_labels)
        for i, label_group in enumerate(_desired_labels):
            if label in label_group:
                collapsed_data[text][i] = 1

    with open("data_collapsed.csv", "w") as f:
        writer = csv.writer(f)
        writer.writerow(["text"] + [str(label_group) for label_group in _desired_labels])
        writer.writerows([[text] + flags for text, flags in collapsed_data.items()])

def process_file(dir_name: str, file: str) -> list:
    """
    Extracts data from given file.
    :param file: name of the file from which to extract data
    :return: processed data as a list of lists. This can be considered tabulated data, as in a csv file.
    """
    ret = []

    with open(dir_name + file, "r") as f:
        print("Processing " + file)
        reader = csv.reader(f, delimiter="\t" if ".tsv" in file else ",")
        for document_id, label, start, end, *_ in reader:
            ret.append((document_id, label, get_text(dir_name, document_id, int(start), int(end))))

    return ret

def get_text(dir_name: str, document_id: str, start: int, end: int) -> str:
    if document_id not in _files:
        full_name = f"{dir_name + ('/' if not dir_name.endswith('/') else '')}article{document_id}.txt"
        # just open 2 for now so we can try the other encoding on error
        with open(full_name, "r", encoding="utf-8") as f_utf_8, open(full_name, "r", encoding="utf-16") as f_utf_16:
            try:
                _files[document_id] = f_utf_8.read()
            except UnicodeDecodeError:
                _files[document_id] = f_utf_16.read()

    return read_file(_files[document_id], start, end)

def read_file(content: str, start: int, end: int):
    # Define a regex pattern for sentence-ending punctuation
    sentence_end_pattern = re.compile(r'[.?!]\s*|\n')

    # Find the start and end indices of the sentence containing the specified span
    start_sentence_index = max(0, max(
        [match.end() for match in sentence_end_pattern.finditer(content, 0, start)] or [0]))

    end_sentence_index = sentence_end_pattern.search(content, end)

    if end_sentence_index:
        end_sentence_index = end_sentence_index.end()
    else:
        # If the end of the text is reached, consider the rest of the content as the sentence
        end_sentence_index = len(content)

    # Extract the sentence or set of sentences
    sentence = content[start_sentence_index:end_sentence_index].strip()
    sentence = re.sub(r'\s+', ' ', sentence).strip()

    # replace quotes at the beginning and end with angled quotes
    sentence = re.sub(r'^"', '“', sentence)
    sentence = re.sub(r'"$', '”', sentence)

    return sentence

def collect_labels(csv_iterable: Iterable) -> set[str]:
    """
    Collects all labels from the data and writes them to a file.
    :return: None
    """
    labels = set()

    for _, label, _ in csv_iterable:
        labels.add(label)

    with open("labels.txt", "w") as f:
        f.write("\n".join(labels))

    return labels

if __name__ == "__main__":
    main(argv)


#!/usr/bin/python3
# -*- coding: utf-8 -*-
# Рекомендуется Python 3.x

"""
Я старался как следует прокомментировать основные моменты, чтобы в случае необходимости
в моём коде мог разобраться сотрудник, незнакомый с Python
© Илья <WST>
"""

# Импортируем важные пакеты
import time, sys, threading, requests, signal

# Время начала работы скрипта
execution_start = time.time()

# Расставим затычки-мьютексы
in_mutex = threading.Lock()
out_mutex = threading.Lock()

# Прикинемся браузером
request_headers = {
	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/49.0.2623.108 Chrome/49.0.2623.108 Safari/537.36',
}

# Счётчик обработанных ссылок (для отображения прогресса)
counter = 0

# http://stackoverflow.com/questions/22492484/how-do-i-get-the-ip-address-from-a-http-request-using-the-requests-library
try:
	from requests.packages.urllib3.connectionpool import HTTPConnectionPool

	# Отключим ругань на невалидный сертификат
	from requests.packages.urllib3.exceptions import InsecureRequestWarning
	requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
except:
	print("Sadly, your version of Requests is too old.\nTry using the version from OTS repo: http://doc.mkpnet.ru/admin/deb/index.html")
	sys.exit(-1)

# Новый метод, который мы обезьянним методом воткнём вместо старого
def _make_request(self, conn, method, url, **kwargs):
	response = self._old_make_request(conn, method, url, **kwargs)
	sock = getattr(conn, 'sock', False)
	if sock:
		setattr(response, 'peer', sock.getpeername())
	else:
		setattr(response, 'peer', None)
	return response

# Осуществляем подмену
HTTPConnectionPool._old_make_request = HTTPConnectionPool._make_request
HTTPConnectionPool._make_request = _make_request

# Наш воркер
class Worker(threading.Thread):
	def __init__(self, thread_id, in_data, out_data, trace):
		threading.Thread.__init__(self),
		self.thread_id = thread_id
		self.in_data = in_data
		self.out_data = out_data
		self.timeout = 3
		self.total_count = len(in_data)
		self.trace = trace

	def select_unprocessed(self):
		with in_mutex:
			try:
				result = self.in_data.pop()
			except:
				result = None
			return result

	def report_progress(self, item):
		global counter
		counter += 1
		print(u"(%d of %d) [%s] %s" % (counter, self.total_count, item[2], item[1]))

	def process_item(self, item):
		global request_headers
		item[4] = int(time.time())

		try:
			response = requests.get(item[1], timeout = self.timeout, stream = True, headers = request_headers)
			content = response.raw.read(100000, decode_content = True)

			if b'eais.rkn.gov.ru' in content:
				item[2] = 'blocked'
			else:
				peer = response.raw._original_response.peer
				if peer is not None:
					address = peer[0]
					if address.startswith('127') or address.startswith('192.168') or address.startswith('10.10') or address == '::1' or address is None:
						item[2] = 'local-ip'
						item[3] = ''
					else:
						item[2] = 'available'
						#item[3] = content
						item[3] = ''
				else:
					item[2] = 'available'
					#item[3] = content
					item[3] = ''
		except Exception as e:
			item[2] = 'failure'

		with out_mutex:
			if self.trace:
				self.report_progress(item)
			self.out_data.append(item)

	def set_timeout(self, new_timeout):
		self.timeout = new_timeout

	def run(self):
		while True:
			item = self.select_unprocessed()
			if item is None:
				break
			else:
				self.process_item(item)

try:
	import mysql.connector as MySQLdb
except:
	print("mysql.connector is required, other drivers are not supported")
	sys.exit(-1)

# Импортируем конфигурацию
import config

# Профилирование
import resource

# Получение ID проверялки из базы данных
def get_instance_id(api = False):
	timestamp = int(time.time())
	instance_id = 0
	cursor.execute("SELECT checker_id, checker_force_scan, checker_state, checker_last_scan_time, checker_enabled, checker_threads FROM roskom_checkers WHERE REPLACE(checker_ip, '127.0.0.1', 'localhost') = (SELECT SUBSTRING_INDEX(host, ':', 1) FROM information_schema.processlist WHERE ID = connection_id()) FOR UPDATE")
	rows = cursor.fetchall()
	if len(rows) != 1:
		print("This checker instance is not registered within lanbill database. Register this checker first.")
		sys.exit(-1)
	else:
		force_scan = int(rows[0][1])
		checker_state = rows[0][2]
		checker_last_scan_time = int(rows[0][3])
		checker_enabled = rows[0][4]
		checker_threads = int(rows[0][5])

		if checker_threads != 0:
			config.THREADS = checker_threads

		if checker_enabled != 'yes':
			print("Checker is turned off in the settings section")
			sys.exit(0)

		if api:
			if force_scan == 0:
				sys.exit(0)
		else:
			if force_scan == 1:
				print("There is an unfinished scheduled scan")
				sys.exit(0)

		if checker_state == 'scanning':
			# В этом месте существует вероятность, что по факту проверка не выполняется, т.е скрипт упал или что-то такое, возможно, стоит ругаться об этом в лог, если последняя проверка была слишком давно
			# Форсируем проверку, если есть подозрение на такую ситуацию
			if checker_last_scan_time > (timestamp - 3600):
				print("Already scanning")
				sys.exit(0)

		instance_id = int(rows[0][0])
		cursor.execute("UPDATE roskom_checkers SET checker_state = 'scanning', checker_force_scan = 0, checker_last_scan_time = %s WHERE checker_id = %s", (timestamp, instance_id))

	return instance_id

# Соединение с БД
def connect_db():
	try:
		db = MySQLdb.connect(charset = 'utf8', use_unicode = True, **config.DATABASE)
		return db
	except Exception as e:
		print("MySQL connection failure\n%s" % str(e))
		print("Edit /opt/roskomcheck/config.py to configure access to lanbill database")
		sys.exit(-1)

def log_message(message, level = 'info'):
	timestamp = int(time.time())
	cursor.execute("INSERT INTO roskom_log (log_time, log_message, log_type, log_script_name) VALUES (%s, %s, %s, 'roskomcheck.py')", (timestamp, message, level))
	db.commit()

trace = True
api = False
if len(sys.argv) >= 2:
	if sys.argv[1] == 'cron':
		trace = False

	if sys.argv[1] == 'api':
		api = True

def signal_handler(signal, frame):
	timestamp = int(time.time())
	message = "instance #%d aborted by signal" % (instance_id,)
	db = connect_db()
	cursor = db.cursor()
	cursor.execute("UPDATE roskom_checkers SET checker_state = 'idle', checker_force_scan = 0 WHERE checker_id = %s", (instance_id,))
	cursor.execute("INSERT INTO roskom_log (log_time, log_message, log_type, log_script_name) VALUES (%s, %s, %s, 'roskomcheck.py')", (timestamp, message, 'info'))
	db.commit()
	print('Exitting requested')
	db.close()
	exit(0)

signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGQUIT, signal_handler)

# Установим соединение с БД
db = connect_db()
#db.begin()
cursor = db.cursor(buffered = True)
# Проверим, зарегистрирован ли наш экземпляр в админке. Если нет, нам незачем работать
instance_id = get_instance_id(api)
db.commit()

log_message("instance #%d started using %d threads" % (instance_id, config.THREADS))

# Получим список URL-ок, доступность которых нам требуется проверить
cursor.execute("SELECT url_text FROM roskom_url")
in_data = [[0, i[0], 'unknown', '', 0] for i in cursor.fetchall()]
out_data = []

# Можно отсоединиться от БД на время анализа
cursor.close()
db.close()

# Инициализируем наши рабочие потоки
threads = {}
for i in range(config.THREADS):
	threads[i] = Worker(i, in_data, out_data, trace)
	threads[i].set_timeout(config.HTTP_TIMEOUT)
	threads[i].setDaemon(True)

# Разветвляемся
for index, thread in threads.items():
	thread.start()

# Соединяемся
for index, thread in threads.items():
	thread.join()

# На этом этапе у нас сформирована статистика в массиве out_data, получим данные для внесения в БД
timestamp = int(time.time())
total = len(out_data)
available = [i for i in out_data if i[2] == 'available']
unavailable = [i for i in out_data if i[2] in ['blocked', 'failure', 'local-ip']]

# Предварительная оценка ресурсов для записи в лог
stat = resource.getrusage(resource.RUSAGE_SELF)

# Время окончания работы скрипта, не считая финальную транзакцию
execution_end = time.time()
execution_time = execution_end - execution_start
execution_minutes = int(execution_time / 60)
execution_seconds = (execution_time - (execution_minutes * 60))

# Установим соединение с БД для сохранения результатов
db = connect_db()
cursor = db.cursor()
cursor.execute("INSERT INTO roskom_checker_scans (scan_checker_id, scan_when, scan_urls_total, scan_urls_available, scan_urls_unavailable, scan_time) VALUES (%s, %s, %s, %s, %s, %s)", (instance_id, timestamp, total, len(available), len(unavailable), int(execution_time)))
cursor.execute("UPDATE roskom_checkers SET checker_last_scan_time = %s, checker_last_scan_id = %s, checker_state = 'idle', checker_force_scan = 0 WHERE checker_id = %s", (timestamp, cursor.lastrowid, instance_id))
cursor.execute("DELETE FROM roskom_available_links WHERE link_checker_id = %s", (instance_id,))

for link in available:
	# Внесём ссылку в список доступных при последней проверке
	cursor.execute("INSERT INTO roskom_available_links (link_url, link_body, link_checker_id) VALUES (%s, %s, %s)", (link[1], link[3], instance_id))

	# Обновим статистику в roskom_url_stat
	cursor.execute("UPDATE roskom_url_stat SET us_ptime = %s, us_checker_id = %s, us_pcount = us_pcount + 1 WHERE us_hash = MD5(%s)", (link[4], instance_id, link[1]))

log_message("instance #%d finished, taking %d kb RES and %dm %.2fs" % (instance_id, stat.ru_maxrss, execution_minutes, execution_seconds))
db.commit()
cursor.close()
db.close()

if trace:
	# Выведем статистику проверки
	print("\n Scan report")
	print(" Total: %d\n Unavailable: %d\n Available: %d\n" % (total, len(unavailable), len(available)))

if config.ENABLE_LOG:
	with open(config.LOG, "w") as handle:
		handle.write("\n".join([i[1] for i in available]))
		handle.write("\n")


from config import db, ma
from marshmallow import fields

class ArtObject(db.Model):
    __tablename__ = 'artobject'
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String(128))
    description = db.Column(db.String(128))
    coordinate = db.relationship('Coordinate', uselist=False)

class Coordinate(db.Model):
    __tablename__ = 'coordinate'
    artobject_id = db.Column(db.Integer, db.ForeignKey('artobject.id'),
                             nullable=False, primary_key=True)
    latitude = db.Column(db.Float)
    longitude = db.Column(db.Float)
    altitude = db.Column(db.Float)

class Feedback(db.Model):
    __tablename__ = 'feedback'
    id = db.Column(db.Integer, primary_key=True)

class CoordinateSchema(ma.ModelSchema):
    class Meta:
        model = Coordinate
        sqla_session = db.session

class ArtObjectSchema(ma.ModelSchema):
    class Meta:
        model = ArtObject
        sqla_session = db.session
    coordinate = fields.Nested(CoordinateSchema, many=False)

class FeedbackSchema(ma.ModelSchema):
    class Meta:
        model = Feedback
        sqla_session = db.session

general_schema = ArtObjectSchema()
summary_schema = ArtObjectSchema(only=('id', 'title', 'description', 'coordinate'), many=True)
description_schema = ArtObjectSchema(only=('id', 'description',))
coordinate_schema = ArtObjectSchema(only=('id', 'coordinate',))


import sklearn
from sklearn.ensemble import RandomForestClassifier
from sklearn import naive_bayes, svm, linear_model
from sklearn import neighbors
from sklearn.metrics import accuracy_score

# Use five basic machine learning prediction models to predict:
def train_models(model_name, X_train, y_train):
    if model_name == "Random Forest":
        model = RandomForestClassifier(n_estimators = 200, random_state=0)
    elif model_name == "Naive Bayes":
        model = naive_bayes.MultinomialNB()
    elif model_name == "Logistic Regression":
        model = linear_model.LogisticRegression(penalty = 'l2')
    elif model_name == "SVM":
        model = svm.SVC(kernel = 'linear')
    elif model_name == "KNN":
        model = neighbors.KNeighborsClassifier(n_neighbors=5)

    model.fit(X_train, y_train)
    return model

# Use the model to predict and gives accuracy:s
def make_prediction(model, X_test, y_test):
    predictions = model.predict(X_test)
    accuracy = accuracy_score(predictions, y_test)
    return predictions, accuracy


from split_settings.tools import include
from dotenv import load_dotenv
load_dotenv()

include(
    'components/auth_password_validators.py',
    'components/database.py',
    'components/installed_apps.py',
    'components/middleware.py',
    'components/tempates.py',
)

from pathlib import Path
import os

DEBUG = os.environ.get('DEBUG', False) == 'True'
ALLOWED_HOSTS = ['127.0.0.1']

BASE_DIR = Path(__file__).resolve().parent.parent
SECRET_KEY = 'django-insecure-zemvma(i#(znnxfs_@o%+5(540z)^o(liq5w!t@y%60w2m3507'
ROOT_URLCONF = 'kino.urls'
WSGI_APPLICATION = 'kino.wsgi.application'

LANGUAGE_CODE = 'ru-RU'
LOCALE_PATHS = ['movies/locale']
TIME_ZONE = 'UTC'

USE_I18N = True
USE_TZ = True

STATIC_URL = 'static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

#this is for debuger
INTERNAL_IPS = [
    "127.0.0.1",
]


#
# fixImageRotation.py - A script to fix image rotation based on EXIF data
#
# (C) 2023 Marcel Verpaalen
#
# Licensed under the GNU GPL v3.0 license
#

import os

import pyexiv2
from PIL import Image, ImageOps

def fix_exif_data(imagefile):
    """
    Fixes the EXIF data of an image file to set the orientation to 1.

    Args:
        imagefile (str): The file path of the image to fix.

    Returns:
        None
    """
    with open(imagefile, "rb+") as f:
        with pyexiv2.ImageData(f.read()) as img:
            exif = img.read_exif()
            print(exif)

            img.modify_exif({"Exif.Image.Orientation": 1})
            # Empty the original file
            f.seek(0)
            f.truncate()
            # Get the bytes data of the image and save it to the file
            f.write(img.get_bytes())

def get_orientation_description(orientation):
    """
    Returns a string description of the given image orientation code.

    Parameters:
    orientation (int): An integer representing the image orientation code.

    Returns:
    str: A string description of the given image orientation code.
    """
    orientation_descriptions = {
        1: "Normal",
        2: "Flipped horizontally",
        3: "Upside down",
        4: "Flipped vertically",
        5: "Rotated 90 degrees counterclockwise and flipped horizontally",
        6: "Rotated 90 degrees counterclockwise",
        7: "Rotated 90 degrees counterclockwise and flipped vertically",
        8: "Rotated 90 degrees clockwise",
    }
    return orientation_descriptions.get(orientation, "Unknown")

def fix_image_rotation(image_path):
    """
    Rotates an image based on its EXIF orientation tag and saves it with orientation tag set to 1.

    Args:
        image_path (str): The file path of the image to be rotated.

    Returns:
        None
    """
    img = Image.open(image_path)

    exif = img.getexif()

    if exif:
        # 274 corresponds to the 'Orientation' tag, default to 1 if not found
        orientation = exif.get(274, 1)
        orientation_description = get_orientation_description(orientation)

        # Get image dimensions (width and height)
        width, height = img.size

        print(f"Original Orientation: {orientation} - {orientation_description}")
        print(f"Dimensions: {width} x {height}")

        if orientation > 1:
            image = ImageOps.exif_transpose(img)

            # Remove the EXIF orientation tag (set it to 1) --> Does not work, not saved
            exif[274] = 1  # 274 corresponds to the 'Orientation' tag
            image.save(image_path, exif=img.info["exif"])
            fix_exif_data(image_path)
            print("Rotated to Orientation 1")
        print()
    else:
        print("No EXIF data found.\n")

def rotate_images(folder_path):
    """
    Rotates all JPEG images in the specified folder to the correct orientation.

    Args:
        folder_path (str): The path to the folder containing the images to be rotated.
    """
    files = os.listdir(folder_path)
    files.sort()
    for file in files:
        if file.endswith(".jpg"):
            file_path = os.path.join(folder_path, file)
            fix_image_rotation(file_path)

# rotate_images("C:\\TMP\Photo")
rotate_images("D:\\")


import cv2
import settings
import mediapipe as mp
import numpy as np
from numpy import core
from tensorflow.keras.models import load_model

mp_hands = mp.solutions.hands
camera = settings.camera
model_name = settings.model_name
labels = settings.labels

# Mediapipe Setup from https://google.github.io/mediapipe/solutions/hands.html

def process_mediapipe(frame, hands):
    image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(image)
    return results.multi_hand_landmarks

def image_capture(model, hands):
    cap = cv2.VideoCapture(camera)
    while True:
        ret, frame = cap.read()
        landmarks = process_mediapipe(frame, hands)
        output = 'no hand detected'
        if landmarks:
            landmarks = unpack_landmarks(landmarks)
            coordinates = vectorize_landmarks(landmarks)
            prediction, confidence = process_flainet(coordinates, model)
            output = f'{prediction}: {confidence}'

        # Flip the image horizontally for a selfie-view display.
        image = frame.copy()
        image = cv2.flip(image, 1)
        cv2.putText(image,
                    output,
                    (30, 30),
                    cv2.FONT_HERSHEY_SIMPLEX,
                    .6,
                    (0, 255, 0),
                    1)

        cv2.imshow('Flainet Demo', image)

        if cv2.waitKey(5) & 0xFF == 27:
            break
    cap.release()

def unpack_landmarks(landmarks):
    lm = []
    for landmark in landmarks[0].landmark:
        vec = [landmark.x, landmark.y, landmark.z]
        lm.append(vec)
    return lm

def vectorize_landmarks(landmarks):
    coordinates = np.array(landmarks)
    return coordinates

def process_flainet(coordinates, model):
    examples = np.array([coordinates])
    predictions = model.predict(examples)
    prediction_classes = np.argmax(predictions, axis=-1)
    confidence = np.amax(predictions, axis=-1)
    return labels[prediction_classes[0]], confidence[0]

def main():
    hands = mp_hands.Hands(
        static_image_mode=False,
        model_complexity=1,
        max_num_hands=1,
        min_detection_confidence=0.7,
        min_tracking_confidence=0.8)
    model = load_model(f'../{model_name}.h5')
    image_capture(model, hands)

if __name__ == '__main__':
    main()


from cities.models import City, Country, Region
from django.contrib.gis.geos import Point
from django.test import TestCase

from airports.management.commands.airports import create_airport, get_location_info

class AirportTests(TestCase):
    def setUp(self):
        self.country = Country(
            slug="United-States",
            name="United States",
            code="US",
            code3="USA",
            population=310232863,
            area=9629091,
            currency="USD",
            currency_name="Dollar",
            currency_symbol="$",
            language_codes="en-US,es-US,haw,fr",
            phone="1",
            tld="us",
            capital="Washington",
        )
        self.country.save()
        self.region = Region(
            slug="California_US.CA",
            name="California",
            name_std="California",
            code="CA",
            country=self.country,
        )
        self.region.save()
        self.city = City(
            slug="5323401-Alpine",
            name="Alpine",
            name_std="Alpine",
            country=self.country,
            region=self.region,
            location=Point(-116.76641, 32.83505),
            population=14236,
            elevation=559,
            kind="PPL",
            timezone="America/Los_Angeles",
        )
        self.city.save()
        self.id = 8227
        self.airport_defaults = dict(
            name="On the Rocks Airport",
            municipality="Alpine",
            iata=None,
            icao="1CA6",
            local_code="1CA6",
            ident="1CA6",
            altitude=0.0,
            longitude=-116.7229995727539,
            latitude=32.76509857177734,
            country=self.country,
            region=self.region,
            city=self.city,
            type="small_airport",
        )

    def test_create_airport(self):
        country, region, city = get_location_info("Alpine", None, None, -116.7229995727539, 32.76509857177734)
        self.assertIsNotNone(country)
        self.assertIsNotNone(region)
        self.assertIsNotNone(city)
        airport = create_airport(id=self.id, **self.airport_defaults)
        self.assertIsNotNone(airport)

    def test_create_airport_no_name(self):
        country, region, city = get_location_info("", None, None, -116.7229995727539, 32.76509857177734)
        self.assertIsNotNone(country)
        self.assertIsNotNone(region)
        self.assertIsNotNone(city)
        defaults = self.airport_defaults
        defaults["name"] = ""
        airport = create_airport(id=self.id, **defaults)
        self.assertIsNotNone(airport)
        self.assertEqual(airport.name, "Alpine")

    def test_create_airport_no_municipality(self):
        country, region, city = get_location_info("", None, None, -116.7229995727539, 32.76509857177734)
        self.assertIsNotNone(country)
        self.assertIsNotNone(region)
        self.assertIsNotNone(city)
        defaults = self.airport_defaults
        defaults["name"] = ""
        defaults["municipality"] = ""
        airport = create_airport(id=self.id, **defaults)
        self.assertIsNotNone(airport)
        self.assertEqual(airport.name, "Alpine")


import os
from joblib import dump, load
import numpy as np
from sklearn.preprocessing import StandardScaler
from audio_feature_extraction import get_clip_mel_spectrogram
from constants import MOODS_TO_QUADRANTS, DATA_TYPE, MEL_RESIZED_SHAPE, TOTAL_NUM_CLIPS
from pathlib import Path

def save_training_data(dataset_dir: Path) -> np.ndarray:
    """
    Extracts mel spectrograms from all audio samples and saves them into a destination .gz file.

    :param: dataset_dir: the source directory containing the data
    :return: mel spectrograms of all audio samples
    """

    mels = np.empty((TOTAL_NUM_CLIPS,) + MEL_RESIZED_SHAPE, dtype=DATA_TYPE)

    index = 0

    for i, path in enumerate(dataset_dir.iterdir()):
        if os.path.isdir(path):
            for audiofile in path.iterdir():
                mel = get_clip_mel_spectrogram(audiofile)

                print('Finished with', audiofile, 'index is', index)

                mels[index] = mel

                index += 1

    print('Saving data...')

    dump(mels, 'turkish_mels.gz', compress=True)

    return mels

def save_labels(dataset_dir: Path) -> np.array:
    """
    Extracts data labels from the given dataset and saves them into a .gz file.

    :param: dataset_dir: the source directory containing the data
    :return: labels for all audio samples
    """

    labels = np.zeros(TOTAL_NUM_CLIPS, dtype=np.int32)

    i = 0

    for item in os.listdir(dataset_dir):
        if os.path.isdir(dataset_dir / item):
            sub_dir_length = len(os.listdir(dataset_dir / item))

            label = MOODS_TO_QUADRANTS[item]

            labels[i:i+sub_dir_length] = np.repeat(label, sub_dir_length)

            print('label is', label)

            i += sub_dir_length

    dump(labels, 'turkish_labels.gz', compress=True)

    return labels

def normalize_data(data_file: Path) -> np.ndarray:
    """
    Normalizes the mel spectrograms data using StandardScaler(), squeezing the data into the range [-1, 1]
    and ensuring that it has 0 mean and 1 standard deviation.

    :param: data_file: a path to the file containing the mel spectrograms data
    :return: the normalized data
    """

    data = load(data_file)

    scaler = StandardScaler()

    flattened = data.flatten()

    normalized_data = scaler.fit_transform(flattened.reshape(-1, 1)).reshape((data.shape + (1,)))

    dump(normalized_data, 'turkish_normalized_mels.gz', compress=True)

    return normalized_data


import pandas as pd
import numpy as np
from sklearn import linear_model
# reading the csbv file
df = pd.read_csv("homeprices.csv")
# calculating the median of the column
median = int(df.bedroom.median())
# fill the empty rows in the column
df.bedroom = df.bedroom.fillna(median)

reg = linear_model.LinearRegression()
reg.fit(df[['area','bedroom','age']],df.price)
# f = m1*area+m2*bedroom+m3*age+b
# m is coef
# b is intercept
print(reg.coef_)
print(reg.intercept_)

prediction = reg.predict([[4563,3,45]])
print(prediction)


from datetime import date, datetime, timedelta
from time import daylight, sleep

def getNumberOfWeeksTillNow(year,month,day):

    d1=date(year,month,day)
    d2=date.today()

    monday1 = (d1 - timedelta(days=d1.weekday()))
    monday2 = (d2 - timedelta(days=d2.weekday()))
    return (monday2 - monday1).days / 7


import json
from os import makedirs
from os.path import join, exists

import deep_surfel as dsurf
import torch
from torch.utils.data import Dataset, IterableDataset

from ._util import get_frame_ids, find_closest_frames, stack_frames
from .frame import Frame

class OneSceneIterableDataset(IterableDataset):
    def __init__(self, config, dir_path, scene, frame_ids, closest_frame_ids, mode):
        super(OneSceneIterableDataset).__init__()

        self.mode = mode
        self.config = config
        self.dir_path = dir_path
        self.frame_ids = frame_ids
        self.closest_frame_ids = closest_frame_ids
        self.scene = scene
        self.n_frames = len(frame_ids)
        self.offset = 0

    def __iter__(self):
        # worker_info = torch.utils.data.get_worker_info()
        for frame_id, closest_frame_id in zip(self.frame_ids, self.closest_frame_ids):
            frame = Frame(self.config, self.dir_path, self.scene, frame_id, self.mode)
            closest_frame = Frame(self.config, self.dir_path, self.scene, closest_frame_id, self.mode)
            yield frame(), closest_frame()

    @staticmethod
    def worker_init_fn(worker_id):
        worker_info = torch.utils.data.get_worker_info()
        dataset = worker_info.dataset  # the dataset copy in this worker process
        indices = list(range(worker_id, dataset.n_frames, worker_info.num_workers))
        dataset.frame_ids = [dataset.frame_ids[i] for i in indices]

    @staticmethod
    def collate_fn(batch):
        closest_frames_batch = [b[1] for b in batch]
        batch = [b[0] for b in batch]

        keys = batch[0].keys()
        for i in range(len(batch)):
            for key in keys:
                if key != 'frame_id':
                    batch[i][key] = batch[i][key].unsqueeze(0)
                    closest_frames_batch[i][key] = closest_frames_batch[i][key].unsqueeze(0)
        return batch, closest_frames_batch

class GeneralizationDataset(Dataset):

    def __init__(self, config, data_root, scene_ids, frame_ids):
        self.mode = 'train'
        self.config = config
        self.data_root = data_root
        self.scene_ids = scene_ids
        self.frame_ids = frame_ids
        self.n_samples = len(scene_ids)
        self.surfel_channels = config.surfel_channels

        self.batch_size = config.batch_size
        self.cache_dir = config.tmp_cache_dir
        self.current_batch_scene_ids = set()
        self.device = self.config.device
        # self.worker_id = 0

    def __len__(self):
        return self.n_samples

    def load_scene(self, scene_id):
        # worker_info = torch.utils.data.get_worker_info()
        scene_name = f'scene_{self.config.geometry_resolution}_{self.config.patch_resolution}.dsurf'

        if not exists(join(self.cache_dir, scene_id, 'cache.json')):  # first time
            makedirs(join(self.cache_dir, scene_id))
            dir_path = join(self.data_root, scene_id, self.mode)
            frame_ids = get_frame_ids(dir_path)
            closest_frame_ids = find_closest_frames(dir_path, frame_ids, self.config.dont_toggle_yz)
            with open(join(self.cache_dir, scene_id, 'cache.json'), 'w') as f:
                json.dump({'frame_ids': frame_ids, 'closest_frame_ids': closest_frame_ids, 'iterator': 0}, f)

            scene = dsurf.load(join(self.data_root, scene_id, scene_name))
            if scene.channels != self.surfel_channels:
                scene.reset(self.surfel_channels)
        else:
            scene = dsurf.load(join(self.cache_dir, scene_id, scene_name))
        scene.scene_id = scene_id

        return scene

    def load_frame(self, scene):
        dir_path = join(self.data_root, scene.scene_id, self.mode)
        with open(join(self.cache_dir, scene.scene_id, 'cache.json')) as f:
            cache = json.load(f)
            frame = Frame(self.config, dir_path, scene, cache['frame_ids'][cache['iterator']], self.mode)()
            closest_frame = Frame(self.config, dir_path, scene, cache['closest_frame_ids'][cache['iterator']],
                                  self.mode)()
            for key in frame.keys():
                if key != 'frame_id':
                    frame[key] = frame[key].unsqueeze(0)
                    closest_frame[key] = closest_frame[key].unsqueeze(0)

        return frame, closest_frame

    def __getitem__(self, idx):
        # print(f'{torch.utils.data.get_worker_info().id} loads {self.scene_ids[idx]}')
        print(f'loads {self.scene_ids[idx]}')
        scene = self.load_scene(self.scene_ids[idx])
        frame, closest_frame = self.load_frame(scene)
        return scene, frame, closest_frame

    @staticmethod
    def collate_fn(batch):
        scene = dsurf.stack([b[0] for b in batch])
        frame = stack_frames([b[1] for b in batch])
        closest_frame = stack_frames([b[2] for b in batch])

        return scene, frame, closest_frame


#Functions for library
import datetime

todaysDate = str(datetime.date.today())
now =datetime.datetime.now()
time = str(now.strftime("%Y-%m-%d %H: %M: %S"))
x = 0

def read():
    file = open(".//resource//stock.txt","r")
    for line in file:
        print(line.strip("\n"))
    file.close()

def storedList():
    books = []
    file = open(".//resource//stock.txt","r")
    for line in file:
        books.append(line.strip("\n").split(","))
    file.close()
    return books

def chkStock(bookname,quantity):
    books = storedList()
    if int(books[bookname][2]) > quantity-1:
            return True

def loan(bookname,quantity):
    books = storedList()
    q = int(books[bookname][2])
    r = q - quantity
    books[bookname][2] = r

    file = open(".//resource//stock.txt","w")
    for i in books:
        file.writelines(str(i[0]) +","+ str(i[1])+ "," + str(i[2]) + ","+ str(i[3])+"\n")
    file.close()

def reciept(name,bookname,quantity,rate,uniqueid):
    books = storedList()
    book = books[bookname][0]

    file = open(name+str(uniqueid)+".txt","w")
    file.write("-----------------------------------------|LOAN|---------------------------------------------" +"\n")
    file.write("Time: " + time + "\n")
    file.write("--------------------------------------------------------------------------------------------"+"\n")
    file.write("Name: " + name +"\n")
    file.write("Borrowed: ")
    file.write(str(book) + ", " + str(quantity) + "," + rate+"\n")
    file.write("Amount: $" + str(transcation(bookname,quantity))+"0 \n")
    file.write("--------------------------------------------------------------------------------------------")
    file.close()

def recieptstudent(name,bookname,quantity,rate,uniqueid,days):
    books = storedList()
    book = books[bookname][0]

    file = open("Return_"+name+str(uniqueid)+".txt","w")
    file.write("-----------------------------------------|RETURN|-------------------------------------------"+"\n")
    file.write("Time: " +time + "\n")
    file.write("--------------------------------------------------------------------------------------------"+"\n")
    file.write("Name: " + name +"\n")
    file.write("Returned: ")
    file.write(str(book) + ", " + str(quantity) + "," + rate +"\n")
    file.write("Amount: $"+str(transcation(bookname,quantity) + overdued(days,quantity))+"0 \n")
    file.write("--------------------------------------------------------------------------------------------")
    file.close()

def getRate(bookname=0):
    return storedList()[bookname][3]

def returnBook(name,bookname,quantity):
    books = storedList()
    book = books[bookname][0]

    q = int(books[bookname][2])
    r = q + quantity
    books[bookname][2] = r

    file= open(".//resource//stock.txt","w")
    for i in books:
        file.writelines(str(i[0]) +","+ str(i[1])+ "," + str(i[2]) + ","+ str(i[3])+"\n")
    file.close()

def transcation(bookname,quantity):
    rate = float(getRate(bookname).replace("$",""))
    total = quantity * rate
    return total

def display():
    print("--------------------------------------------------------------------------------------------")
    print("Book Name         Author               Quantity   Price(per 10 day)")
    print("--------------------------------------------------------------------------------------------")

    file = open(".//resource//stock.txt","r")
    for line in file:
        print(line.replace(",","  \t  "))

    print("--------------------------------------------------------------------------------------------")

def getUniqeID():
    file = open(".//resource//uniqueid.txt","r")
    uniqueid = int(file.readline())
    file.close
    return uniqueid

def UpdateUniqueID(uniqueid):
    file = open(".//resource//uniqueid.txt","w")
    file.write(str(uniqueid))
    file.close

def chkBookname(bookname):
    books = storedList()
    for i in books:
        if bookname in i[0]:
            return True
    else:
        return False

def overdued(days,quantity):
    if days > 10:
        return int(0.50*(days - 10)*quantity)
    else:
        return int(0)


import argparse
import os
import hashlib
import json

parser = argparse.ArgumentParser()
parser.add_argument("--file", help='to wrap file')
args = parser.parse_args()

create_folder = "file/"
file_size = os.path.getsize(args.file)
# 100 바이트 이하면 가치없다고 판단하고 종료
if file_size < 100:
    print("파일이 너무 작아서 가치가 없습니다.")
    exit()

# 파일을 10분할로 하되, 마지막 chunk는 그 나머지를 취하도록 구성
one_file_size = int(file_size / 10)

file_size_list = [
    one_file_size,
    one_file_size,
    one_file_size,
    one_file_size,
    one_file_size,
    one_file_size,
    one_file_size,
    one_file_size,
    one_file_size,
    file_size - one_file_size * 9
]

# byte sequence가 주어지면, md5 값 계산
def getHashName(byte_seq):
    hasher = hashlib.md5()
    hasher.update(byte_seq)
    return hasher.hexdigest()

# 해당 파일의 md5 계산
def checkHash(path, blocksize):
    hash = hashlib.md5()
    with open(create_folder + path, 'rb') as file:
        buf = file.read(blocksize)
        hash.update(buf)
    file.close()
    return hash.hexdigest()

# simpletorrent 파일 형식(json)
json_result = {
    "actual_name": "",
    "hash_list": "",
    "size": ""
}

# hash_list는 md5 해쉬값이 들어간다.
list = []

# 해당 파일을 읽고 md5 파일 이름으로 file 폴더에 저장
with open(args.file, "rb") as f:
    for i in range(0, 10):
        byte = f.read(file_size_list[i])
        file_name = getHashName(byte)
        list.append(file_name)
        with open(create_folder + file_name, "wb") as output:
            output.write(byte)
            output.close()
        # print(file_name + " " + str(file_size_list[i]) + " " + checkHash(file_name, file_size_list[i]))

# simpletorrent file 포맷에 집어 넣음
json_result["actual_name"] = args.file
json_result["hash_list"] = list
json_result["size"] = file_size

# Dictionary를 json.dumps를 이용하여 string으로 바꿈
json_object = json.dumps(json_result, indent=2)

# 해당 이름.simpletorrent 파일로 저장(w를 이용하여 스트링으로 저장)
with open(args.file + ".simpletorrent", "w") as output:
    output.write(json_object)
    output.close()